From 85173cadbe47c90117a749fea45e679b74f3cca3 Mon Sep 17 00:00:00 2001 From: Alexliu Date: Sat, 5 Sep 2026 00:54:17 +0800 Subject: [PATCH 1/9] fix(agent): stop grounding rule from licensing labelled fabrication The rule used to require an up-front disclaimer for unsourced figures but otherwise let the model present them, and scoped that disclaimer duty to figures and numbers specifically. A model could satisfy it by inventing rows and appending a label copied near-verbatim from the rule text. Replace the disclosure duty with a default prohibition: unsourced values of any enumerated kind (numbers, person or organization names, identifiers, dates, statuses, table rows) must be omitted and reported as missing. The only exception is a current user request that explicitly asks for a template or sample, and even then the model must state the nature of the content before presenting it rather than appending a caveat afterward. The exemption for wording the model must compose itself (search queries, code, document text) is unconditional so all four call sites -- including the three forced-answer sites that previously lacked any prompt covering this -- carry it, while facts written literally inside such composed text remain subject to the sourcing rule. Also fix the module and function docstrings, which claimed disclosure was the instructed default; that stopped being true once fabrication without a source became a prohibition rather than a labelling obligation. --- src/xagent/core/agent/grounding.py | 74 ++++++---- tests/core/agent/test_grounding.py | 223 ++++++++++++++++++++++++----- tests/core/agent/test_react.py | 10 +- 3 files changed, 238 insertions(+), 69 deletions(-) diff --git a/src/xagent/core/agent/grounding.py b/src/xagent/core/agent/grounding.py index bc5b6f89b..7ffd5c353 100644 --- a/src/xagent/core/agent/grounding.py +++ b/src/xagent/core/agent/grounding.py @@ -13,15 +13,20 @@ remedy -- ask the user, or finish reporting the gap -- belongs to the calling pattern, which owns the user-interaction policy this module cannot see. -This is the proposal-A mitigation from issue #1235. It reduces how often -unsourced figures are emitted and makes disclosure the instructed default, but -it cannot repair a session whose evidence compaction already discarded. +This is the proposal-A mitigation from issue #1235. It forbids unsourced values +by default and makes reporting the gap the instructed response, but it cannot +repair a session whose evidence compaction already discarded. Proposals B (evidence-preserving compaction) and C (provenance tracking and a data-source gate) remain open. """ from __future__ import annotations +VALUE_KINDS = ( + "a number, a person or organization name, an identifier or reference " + "code, a date, a status, or a row of a table" +) + def grounding_rule(*, can_call_tools: bool = True) -> str: """Return the grounding rule for answer text and, optionally, tool arguments. @@ -36,13 +41,14 @@ def grounding_rule(*, can_call_tools: bool = True) -> str: possible. Returns: - A prompt fragment forbidding unsupported claims and unsourced - quantitative data, and requiring up-front disclosure of any - illustrative figures. When ``can_call_tools`` is true it also forbids - supplying a fact-carrying tool-call argument that no source provides, - while leaving arguments the model is expected to compose untouched -- - except for a fact value written literally inside composed code or - document text, which the sourcing requirement still covers. + A prompt fragment forbidding unsupported claims and unsourced values of + every kind it enumerates, requiring the gap be reported instead, and + confining unsourced content to a current request that explicitly asks + for a template or sample. When ``can_call_tools`` is true it also + forbids supplying a fact-carrying tool-call argument that no source + provides, while leaving arguments the model is expected to compose + untouched -- except for a fact value written literally inside composed + code or document text, which the sourcing requirement still covers. """ insufficient_context_rule = ( "If available context is insufficient, say so or use an appropriate " @@ -61,17 +67,15 @@ def grounding_rule(*, can_call_tools: bool = True) -> str: "messages, the retrieved context, or a value an earlier tool result " "actually returned; never guess one, never substitute a " "plausible-looking placeholder for one the user has not given, and " - "never carry one over from a different record. This does not restrict " - "values you are expected to compose yourself, such as a search query, " - "code or a command you write to do the work, a message or answer you " - "write to the user, or document text you were asked to produce. A fact " - "value written literally inside such composed code or text is still " - "subject to the sourcing rule above. The rule also does not reach a " - "default or inferred parameter value such as a page size or result " - "limit, which you are expected to decide yourself. Treat " - "a fact-carrying value you cannot source as " - "missing information rather than inventing it, and omit it when the " - "tool allows it to be omitted." + "never carry one over from a different record. The answer you write " + "to the user reaches you as an argument too; it is wording you " + "compose, so this argument standard does not reach it, while the " + "sourcing rule above still governs every fact inside it. This clause " + "does not reach a default or inferred parameter value such as a page " + "size or result limit, which you are expected to decide yourself. " + "Treat a fact-carrying value you cannot source as missing information " + "rather than inventing it, and omit it when the tool allows it to be " + "omitted." if can_call_tools else "" ) @@ -81,12 +85,26 @@ def grounding_rule(*, can_call_tools: bool = True) -> str: "statistics, percentages, table rows, or time series) that are not " "supported by the conversation, retrieved context, or tool results. " f"{insufficient_context_rule}" - "Never invent figures to fill a gap, and never present invented numbers " - "as real data; produce unsupported figures only when the user explicitly " - "asked for a template, mockup, or illustrative example. Labeling is " - "required either way: if the answer ends up containing any figure that no " - "tool result or provided context supports, whether or not the user asked " - "for one, say so up front, before presenting it, and state that those " - "figures are illustrative placeholders not drawn from any data source." + f"Never fill a gap with an invented value, whether it is {VALUE_KINDS}: " + "when nothing in this conversation, the provided context, or a tool " + "result supports a value the answer needs, leave that value out and " + "say plainly that it is missing, rather than supplying one that " + "looks right. This does not restrict the wording you compose -- how " + "you phrase your reply, a search query, code or a command you write " + "to do the work, or document text you were asked to produce -- it " + "restricts every fact asserted inside that wording. A fact value " + "written literally inside such composed code or text is still " + "subject to the sourcing rule above: the text you compose is yours; " + f"{VALUE_KINDS} that you place inside it is not. The only case in " + "which content that no source supports may appear in the answer is a " + "current user request that explicitly asks you to write a template, " + "a sample, or content that is not meant to be real; in that case, " + "before any of that content appears in the answer, state that the " + "request asked for content that is not real and that none of it " + "comes from a data source, and keep such content to what the request " + "asked for. Outside that case a caveat does not make an invented " + "value acceptable: if you find yourself about to add a note " + "explaining that some values are not real, remove those values and " + "report the gap instead." f"{tool_argument_rule}" ) diff --git a/tests/core/agent/test_grounding.py b/tests/core/agent/test_grounding.py index ea77ea583..9384c4129 100644 --- a/tests/core/agent/test_grounding.py +++ b/tests/core/agent/test_grounding.py @@ -1,6 +1,13 @@ from __future__ import annotations -from xagent.core.agent.grounding import grounding_rule +import xagent.core.agent.grounding as grounding +from xagent.core.agent.grounding import VALUE_KINDS, grounding_rule + +# The sole sentence that may appear inside the answer without a source: it +# names the exception explicitly and is unique to this rule's wording. +TEMPLATE_EXCEPTION_MARKER = ( + "a current user request that explicitly asks you to write a template" +) def test_grounding_rule_covers_quantitative_data() -> None: @@ -22,10 +29,10 @@ def test_grounding_rule_covers_quantitative_data() -> None: ): assert term in rule assert "use an appropriate tool to verify" in rule - assert "illustrative placeholders" in rule + assert TEMPLATE_EXCEPTION_MARKER in rule # Pin the concatenation: a dropped trailing space would still satisfy # every membership assertion above. - assert "verify. Never invent figures" in rule + assert "verify. Never fill a gap" in rule def test_grounding_rule_without_tools_omits_tool_verification() -> None: @@ -34,8 +41,8 @@ def test_grounding_rule_without_tools_omits_tool_verification() -> None: assert "use an appropriate tool" not in rule assert "invented values" in rule assert "quantitative data" in rule - assert "illustrative placeholders" in rule - assert "invented values. Never invent figures" in rule + assert TEMPLATE_EXCEPTION_MARKER in rule + assert "invented values. Never fill a gap" in rule def test_grounding_rule_covers_fact_carrying_tool_arguments() -> None: @@ -59,7 +66,7 @@ def test_grounding_rule_covers_fact_carrying_tool_arguments() -> None: ): assert phrase in rule # Pin the concatenation onto the answer rules that precede it. - assert "illustrative placeholders not drawn from any data source. The same" in rule + assert "report the gap instead. The same standard applies" in rule def test_grounding_rule_exempts_values_the_model_must_compose() -> None: @@ -68,23 +75,43 @@ def test_grounding_rule_exempts_values_the_model_must_compose() -> None: ReAct runs with ``tool_choice="required"``, and the answer it writes is itself a tool argument. Without this exemption the rule would be violated on every turn, which would drain the authority of the answer rules sharing - the same prompt. + the same prompt. The exemption covers only wording, not the facts + asserted inside it, and it is unconditional so the three forced-answer + prompts -- which have no tool-argument clause of their own -- still tell + the model that composing its answer is not itself a violation. + """ + for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): + assert "This does not restrict the wording you compose" in rule + for example in ( + "how you phrase your reply", + "a search query", + "code or a command you write to do the work", + "document text you were asked to produce", + ): + assert example in rule + assert "it restricts every fact asserted inside that wording" in rule + # The answer itself is not named as an example of exempt wording: + # naming it here would read as a self-exemption for the very text + # this rule constrains. + assert "a message or answer you write to the user" not in rule + + +def test_grounding_rule_scopes_the_answer_as_argument_exemption() -> None: + """The answer reaches the model as a tool argument too, on the tools path. + + Without this clause, the tool-argument standard (identifiers, dates, + quantities...) would appear to also govern the answer's own wording, + which is a different, already-covered concern. This clause exempts only + the wording standard, not the sourcing rule above it. """ rule = grounding_rule() - assert "This does not restrict values you are expected to compose yourself" in rule - for example in ( - "a search query", - "code or a command you write to do the work", - "a message or answer you write to the user", - "document text you were asked to produce", - ): - assert example in rule - # A page size the model picks is not a claim about the world, so pausing - # for it would be the over-asking failure the exemption exists to prevent. - assert ( - "does not reach a default or inferred parameter value such as a page " - "size or result limit" in rule + assert "The answer you write to the user reaches you as an argument too" in rule + assert "while the sourcing rule above still governs every fact inside it" in rule + # This scoping sentence lives in the tool-argument clause, so it has + # nothing to say on the no-tools path. + assert "The answer you write to the user reaches you" not in grounding_rule( + can_call_tools=False ) @@ -95,14 +122,16 @@ def test_grounding_rule_keeps_literal_facts_inside_composed_text_sourced() -> No ``write_file`` one ``content`` argument. Read at whole-argument granularity, the exemption would clear an invented identifier for delivery into an external system as long as it rode inside composed code - or document text -- the same outcome the rule exists to prevent. + or document text -- the same outcome the rule exists to prevent. The + same value-kind list used for the top-level prohibition is reused here + so the two statements cannot silently drift apart. """ - rule = grounding_rule() - - assert ( - "A fact value written literally inside such composed code or text is " - "still subject to the sourcing rule above." in rule - ) + for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): + assert ( + "still subject to the sourcing rule above: the text you compose " + "is yours" in rule + ) + assert rule.count(VALUE_KINDS) == 2 def test_grounding_rule_without_tools_omits_tool_argument_clause() -> None: @@ -111,18 +140,140 @@ def test_grounding_rule_without_tools_omits_tool_argument_clause() -> None: assert "tool-call arguments that assert facts" not in rule assert "never guess one" not in rule - assert "compose yourself" not in rule - assert "still subject to the sourcing rule above" not in rule + assert ( + "This clause does not reach a default or inferred parameter value" not in rule + ) assert "a default or inferred parameter value" not in rule + # The compose exemption and the sourcing rule over composed text are + # unconditional, so the no-tools variant still carries them. + assert "This does not restrict the wording you compose" in rule + assert "still subject to the sourcing rule above" in rule + + +def test_grounding_rule_offers_no_reusable_disclaimer_phrasing() -> None: + """The rule must not hand the model a disclaimer phrase it can paste in. + + The #1235 incident's fabricated answer copied its own disclaimer + near-verbatim from the rule that was supposed to prevent fabrication. + """ + for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): + for phrase in ( + "illustrative placeholder", + "illustrative placeholders", + "illustrative example", + "mockup", + "mock data", + "not drawn from any data source", + "for demonstration purposes", + ): + assert phrase not in rule.lower() + # "plausible-looking placeholder" names a fabricated tool-argument value, + # not an answer-text disclaimer, and is kept deliberately. + assert "plausible-looking placeholder" in grounding_rule() + + +def test_grounding_rule_prohibits_every_unsourced_value_kind() -> None: + """The prohibition must not be scoped to numbers alone. + + The #1235 incident fabricated person names and reference codes, neither + of which the old wording ("figures", "numbers") named. + """ + for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): + for kind in ( + "a number", + "a person or organization name", + "an identifier or reference code", + "a date", + "a status", + "a row of a table", + ): + assert kind in rule -def test_grounding_rule_requires_labeling_regardless_of_request() -> None: - """Disclosure must not be conditional on the user asking for a template. +def test_grounding_rule_does_not_license_labelled_fabrication() -> None: + """Disclosure is no longer an unconditional duty independent of request. - The #1235 session asked for a real KPI report, so a disclosure duty gated - on "user asked for an example" would not have applied to it. + The old wording ("Labeling is required either way ... whether or not the + user asked for one") read as permission to fabricate as long as the + fabrication was labelled, which is exactly what the #1235 incident did. """ for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): - assert "Labeling is required either way" in rule - assert "whether or not the user asked for one" in rule - assert "produce unsupported figures only when the user explicitly" in rule + for phrase in ( + "Labeling is required either way", + "whether or not the user asked", + "either way", + ): + assert phrase not in rule + + +def test_grounding_rule_exception_requires_an_explicit_current_request() -> None: + """The sole exception is scoped to the current turn's own request. + + A request from an earlier turn must not license fabrication now: the + #1235 session's request was for a real report, so nothing said in an + earlier turn should have been able to relax that. + """ + for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): + assert ( + "The only case in which content that no source supports may " + "appear in the answer is a current user request that explicitly " + "asks" in rule + ) + assert ( + "Outside that case a caveat does not make an invented value " + "acceptable" in rule + ) + + +def test_grounding_rule_states_sample_nature_before_presenting_it() -> None: + """On the exception path, the disclosure must precede the content.""" + for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): + assert "before any of that content appears in the answer, state" in rule + assert rule.index("before any of that content appears") < rule.index( + "keep such content to what the request asked for" + ) + + +def test_grounding_rule_rejects_caveat_as_a_substitute_for_omission() -> None: + """A caveat must not be used to launder an otherwise-forbidden value. + + The #1235 incident's fabricated answer paired unsourced rows with a note + explaining they were illustrative -- the exact pattern this closes. + """ + for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): + assert "a caveat does not make an invented value acceptable" in rule + assert "remove those values and report the gap instead" in rule + + +def test_grounding_rule_keeps_the_prohibition_free_of_routing_terms() -> None: + """The rule states the prohibition without prescribing a remedy. + + Remedies differ by call site (ReAct can retry with a tool, Auto can + route to react, DAG has neither); the shared rule leaves that choice to + the calling pattern, per this module's own docstring. + """ + for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): + assert "leave that value out and say plainly that it is missing" in rule + for phrase in ( + "choose react", + "existing_context_sufficient", + "re-query", + ): + assert phrase not in rule + + +def test_grounding_module_docstring_states_the_default_as_a_prohibition() -> None: + """The module docstring must not claim disclosure is the instructed default. + + That claim stopped being true once the rule started forbidding unsourced + values by default and confining disclosure to the explicit-template + exception. + """ + doc = grounding.__doc__ or "" + assert "instructed default" not in doc + normalized_doc = " ".join(doc.split()) + assert ( + "Proposals B (evidence-preserving compaction) and C (provenance " + "tracking and a data-source gate) remain open." in normalized_doc + ) + assert "illustrative" not in (grounding_rule.__doc__ or "") diff --git a/tests/core/agent/test_react.py b/tests/core/agent/test_react.py index 51a5ebb00..080ffd8b4 100644 --- a/tests/core/agent/test_react.py +++ b/tests/core/agent/test_react.py @@ -1515,16 +1515,16 @@ def test_react_grounding_rule_present_in_both_answer_paths() -> None: for prompt in (tool_prompt, lookup_tool_prompt, forced_prompt): assert "quantitative data" in prompt - assert "illustrative placeholders" in prompt + assert ( + "a current user request that explicitly asks you to write a template" + in prompt + ) + assert "This does not restrict the wording you compose" in prompt assert "use an appropriate tool to verify" in tool_prompt assert "use an appropriate tool" not in forced_prompt for prompt in (tool_prompt, lookup_tool_prompt): assert "tool-call arguments that assert facts" in prompt assert "never guess one" in prompt - assert ( - "This does not restrict values you are expected to compose yourself" - in prompt - ) assert "tool-call arguments that assert facts" not in forced_prompt assert "## FINAL DELIVERABLE FILE REFERENCES" not in tool_prompt assert "exact markdown_link" in tool_prompt From 140393e5f61c937caae41da86cde570a422e7da1 Mon Sep 17 00:00:00 2001 From: Alexliu Date: Sat, 5 Sep 2026 00:56:21 +0800 Subject: [PATCH 2/9] fix(agent): align auto and dag sibling sentences with the grounding rule Both patterns follow their grounding_rule() call with a local sentence that named the old wording ("unsupported specifics", "illustrative placeholder"). The rule no longer uses that vocabulary, so those sentences pointed at wording the model would never see. Auto's routing remedy is reworded to name the value kinds the rule forbids without changing its own job (route to react so a tool can supply the value). DAG's assessment sentence is reworded to describe leaving out a value rather than labelling an illustrative placeholder, matching the rule's gap-reporting instruction. --- src/xagent/core/agent/pattern/auto/auto.py | 7 ++++--- src/xagent/core/agent/pattern/dag/dag.py | 5 ++--- tests/core/agent/test_auto.py | 18 ++++++++++++++++-- tests/core/agent/test_dag.py | 11 ++++++++++- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/xagent/core/agent/pattern/auto/auto.py b/src/xagent/core/agent/pattern/auto/auto.py index 42dd0f76c..574fdfbae 100644 --- a/src/xagent/core/agent/pattern/auto/auto.py +++ b/src/xagent/core/agent/pattern/auto/auto.py @@ -1288,9 +1288,10 @@ def _decision_prompt( "answer field in the same tool call. Put action before answer in the " "tool arguments. " f"When writing that answer field: {grounding_rule(can_call_tools=False)} " - "If the answer would need such unsupported specifics, set " - "existing_context_sufficient=false and choose react so the agent can " - "verify them with tools.\n\n" + "If the answer would need any value the rule above forbids you to " + "supply -- a number, a name, an identifier, a date, or a table row " + "that no source here supports -- set existing_context_sufficient=false " + "and choose react, so the agent can obtain it with tools.\n\n" f"{final_deliverable_file_reference_instructions(can_lookup=False)}\n\n" "You must classify whether " "the latest request requires current or external facts, and whether " diff --git a/src/xagent/core/agent/pattern/dag/dag.py b/src/xagent/core/agent/pattern/dag/dag.py index ac3062375..05a12c292 100644 --- a/src/xagent/core/agent/pattern/dag/dag.py +++ b/src/xagent/core/agent/pattern/dag/dag.py @@ -1583,9 +1583,8 @@ def _completion_assessment_messages(self, context: Any) -> list[dict[str, Any]]: "over from candidate_output or step_results: " f"{grounding_rule(can_call_tools=False)}\n\n" f"{final_deliverable_file_reference_instructions(can_lookup=False)}\n\n" - "If the answer presents any figure as an illustrative " - "placeholder because no step produced the underlying data, " - "name that unsourced data in reason even when you choose " + "If the answer leaves out a value because no step produced " + "it, name that missing data in reason even when you choose " "status=completed. " f"{final_answer_language_rule(subject='output_language_policy field')}" ), diff --git a/tests/core/agent/test_auto.py b/tests/core/agent/test_auto.py index dc0e1526c..32c096c95 100644 --- a/tests/core/agent/test_auto.py +++ b/tests/core/agent/test_auto.py @@ -1051,12 +1051,26 @@ async def test_auto_decision_prompt_includes_grounding_rule() -> None: assert result["success"] is True decision_prompt = llm.calls[0]["messages"][-1]["content"] assert "quantitative data" in decision_prompt - assert "illustrative placeholders" in decision_prompt + assert ( + "a current user request that explicitly asks you to write a template" + in decision_prompt + ) assert "invented values" in decision_prompt assert decision_prompt.count("## FINAL DELIVERABLE FILE REFERENCES") == 1 assert decision_prompt.index( - "If the answer would need such unsupported specifics" + "If the answer would need any value the rule above forbids" ) < decision_prompt.index("## FINAL DELIVERABLE FILE REFERENCES") + # The routing remedy stays specific to auto's own decision, so it is + # worded independently of the shared rule's neutral gap-reporting text. + assert ( + "set existing_context_sufficient=false and choose react, so the agent " + "can obtain it with tools" in decision_prompt + ) + assert ( + "a number, a name, an identifier, a date, or a table row that no " + "source here supports" in decision_prompt + ) + assert "such unsupported specifics" not in decision_prompt assert "get_workspace_output_files" not in decision_prompt assert "You must classify whether" in decision_prompt assert "You must also classify whether" not in decision_prompt diff --git a/tests/core/agent/test_dag.py b/tests/core/agent/test_dag.py index aedeca1f7..28c27b3d6 100644 --- a/tests/core/agent/test_dag.py +++ b/tests/core/agent/test_dag.py @@ -407,10 +407,19 @@ def test_dag_completion_assessment_prompt_includes_grounding_rule() -> None: system_prompt = messages[0]["content"] assert "quantitative data" in system_prompt - assert "illustrative placeholders" in system_prompt + assert ( + "a current user request that explicitly asks you to write a template" + in system_prompt + ) assert "use an appropriate tool" not in system_prompt assert system_prompt.count("## FINAL DELIVERABLE FILE REFERENCES") == 1 assert "get_workspace_output_files" not in system_prompt + # The DAG-local sentence that follows the shared rule must stay in step + # with the rule's own gap-reporting wording, not the disclosure wording + # the rule no longer uses. + assert "name that missing data in reason" in system_prompt + assert "illustrative" not in system_prompt + assert "report that value as unavailable" not in system_prompt answer_description = pattern._completion_assessment_tool_schema()["function"][ "parameters" From 0e0f7771f57b47c6e53e027d4b2aff0a1b201c85 Mon Sep 17 00:00:00 2001 From: Alexliu Date: Sat, 5 Sep 2026 01:00:15 +0800 Subject: [PATCH 3/9] fix(agent): stop compaction summaries from instructing the next call The compaction summary prompt asked the summarizer to "name the next action needed" and told it the next call should continue "without redoing completed tool calls." Both fabrication-incident summaries put a do-not-call-tools instruction in that slot; the summarizer has no visibility into what tools the next call will actually have. Remove the next-action instruction entirely and state explicitly that the summary must not instruct the next call on tool use. Require fact-carrying values for the records the current request points at (names, identifiers, statuses, dates, counts) to be copied character for character rather than paraphrased or invented to complete a pattern, with credentials and unrelated personal information excluded even when they would otherwise qualify. Forbid claiming a dataset is complete unless the history shows every item was both returned and is still described in the summary -- the incident's dataset was fully returned across nine tool calls, but four of those calls' raw payloads had already been dropped by an earlier compaction. Give the smallest output budget an explicit priority order for what survives, and cap the prompt itself so it cannot silently outgrow the budget it exists to fit inside. The trailer appended after the summary is unchanged; it already carries a correctly-conditioned "re-read or re-query the source" instruction and is not part of this text. --- src/xagent/core/agent/context/execution.py | 48 ++++-- tests/core/agent/test_context.py | 169 ++++++++++++++++++++- 2 files changed, 203 insertions(+), 14 deletions(-) diff --git a/src/xagent/core/agent/context/execution.py b/src/xagent/core/agent/context/execution.py index 4adb94e91..c4c703c06 100644 --- a/src/xagent/core/agent/context/execution.py +++ b/src/xagent/core/agent/context/execution.py @@ -1509,17 +1509,38 @@ def _build_llm_compact_prompt( "tool calls, tool observations, files or URLs mentioned, " "decisions made, and open work. Drop duplicated search noise, " "irrelevant raw payloads, and verbose intermediate text. " - "Preserve exact reusable artifact handles, including file_id " - "values, file: references, markdown file links, URLs, relative " - "paths, absolute paths, output_path, image_path, video_path, " - "artifact filenames, and any other path-like result fields; do " - "not replace machine-usable handles with only descriptive " - "filenames. Clearly separate completed work from remaining work " - "and name the next action needed. " - "Preserve the language of user-facing requests and constraints; " - "if the history is multilingual, keep important details in their " - "original language instead of translating them. " - "Return only the compact summary." + "Preserve exact reusable artifact handles -- file_id values, " + "file: references, markdown links, URLs, paths, output_path, " + "image_path, video_path, artifact filenames, any other " + "path-like field -- never a descriptive filename instead. " + "Preserve, character for character, the values a tool result " + "returned for the records the request points at: their names, " + "the people, organizations or teams they belong to, their " + "identifiers and reference codes, their statuses, dates, " + "counts and totals. Copy such a value or omit it; never " + "paraphrase, substitute, or invent one to complete a pattern. " + "Dropping a raw payload does not license dropping these " + "values; they are not the bulk that instruction covers. " + "Never copy, in whole or in part, a credential, token, key, " + "password, or other authentication material, or personal " + "information the request does not point at; note only that " + "such a value was present and was omitted. If a value is both " + "an identifier the request points at and authentication " + "material, the exclusion wins: omit it. If your budget cannot " + "hold all of this, keep, in this order: first state what is " + "missing and not listed here, with counts; artifact handles; " + "the identifiers and names the request points at; statuses " + "and dates; then the rest. Separate completed work from " + "remaining work. Report only what happened and what is " + "missing: never call a dataset complete, fully retrieved, or " + "fully processed unless the history shows every item was " + "returned and every one is still described here; say which " + "parts survive as prose only. Write no instruction to the " + "next call about tool use or whether to answer: that decision " + "is not yours and its tools are unknown to you. Preserve the " + "language of user-facing requests and constraints; keep " + "multilingual details in their original language. Return only " + "the compact summary." ), }, { @@ -1528,8 +1549,9 @@ def _build_llm_compact_prompt( "Conversation history to compact:\n" f"{transcript}\n\n" "Write a concise but complete continuity summary for the next " - "LLM call. The next LLM call should be able to continue without " - "redoing completed tool calls." + "LLM call. Record which tool calls already completed and what " + "they returned, so the next call can judge for itself what " + "still needs doing." ), }, ] diff --git a/tests/core/agent/test_context.py b/tests/core/agent/test_context.py index 963184ac2..f096a9a86 100644 --- a/tests/core/agent/test_context.py +++ b/tests/core/agent/test_context.py @@ -1068,7 +1068,9 @@ class CompactLLM: assert "completed work from remaining work" in prompt[0]["content"] prompt_text = prompt[1]["content"] assert "Tool read_file returned" in str(prompt_text) - assert "without redoing completed tool calls" in str(prompt_text) + assert "so the next call can judge for itself what still needs doing" in str( + prompt_text + ) result = ctx.compact_with_llm_response( { @@ -1098,6 +1100,171 @@ class CompactLLM: assert ctx.messages[1].content == "current request" +def _build_llm_compact_prompt_texts() -> tuple[str, str]: + ctx = ExecutionContext() + ctx.compact_config.threshold = 1 + ctx.add_user_message("Build a KPI report") + ctx.add_assistant_message( + "", + tool_calls=[ + {"id": "call-1", "type": "function", "function": {"name": "read_file"}}, + ], + ) + ctx.add_tool_result("read_file", {"output": "x" * 200}, tool_call_id="call-1") + request = ctx.build_llm_compact_request_if_needed() + assert request is not None + prompt = request["messages"] + return prompt[0]["content"], str(prompt[1]["content"]) + + +def test_compact_prompt_forbids_instructing_the_next_call() -> None: + """The summary must not tell the next call what to do. + + Both observed fabrication-incident summaries put a "do not call tools + again" or "output the final answer" instruction in their own Next + Action slot; the next call's tool set is not known to the summarizer. + """ + system, user = _build_llm_compact_prompt_texts() + + assert "Write no instruction to the next call" in system + for phrase in ( + "name the next action needed", + "without redoing completed tool calls", + "without making additional tool calls", + "do not call tools", + "produce the final answer", + ): + assert phrase not in system + assert phrase not in user + + +def test_compact_prompt_forbids_unearned_completeness_claims() -> None: + """A dataset must not be called complete unless every part still is. + + The fabrication incident's second turn claimed "the complete dataset of + 443 clients" when four of the nine fetched pages' raw payloads had + already been dropped by an earlier compaction; the pages were returned, + but were no longer described anywhere in the summary. + """ + system, _ = _build_llm_compact_prompt_texts() + + assert ( + "never call a dataset complete, fully retrieved, or fully processed" in system + ) + assert ( + "unless the history shows every item was returned and every one is " + "still described here" in system + ) + assert "say which parts survive as prose only" in system + + +def test_compact_prompt_requires_verbatim_values_for_the_requested_records() -> None: + """Fact-carrying values for records the request points at must be copied. + + The fabrication incident's second turn dropped every team name, client + name, and client code from its summary while still claiming the + underlying dataset was complete. + """ + system, _ = _build_llm_compact_prompt_texts() + + assert "character for character" in system + assert "for the records the request points at" in system + for kind in ( + "their names", + "identifiers and reference codes", + "their statuses, dates, counts and totals", + ): + assert kind in system + + +def test_compact_prompt_excludes_credentials_and_unrelated_personal_data() -> None: + """Verbatim retention must not extend to credentials or stray PII.""" + system, _ = _build_llm_compact_prompt_texts() + + assert "Never copy, in whole or in part" in system + for kind in ("credential", "token", "key", "password", "authentication material"): + assert kind in system + assert "personal information the request does not point at" in system + assert "note only that such a value was present and was omitted" in system + + +def test_compact_prompt_excludes_credentials_even_when_also_an_identifier() -> None: + """A value that is both a requested identifier and a credential is excluded. + + Verbatim retention (records the request points at) and the credential + exclusion can both apply to the same value, e.g. an API key listed + alongside a connector's identifier; the prompt must say which one wins. + """ + system, _ = _build_llm_compact_prompt_texts() + + assert ( + "If a value is both an identifier the request points at and " + "authentication material, the exclusion wins: omit it." in system + ) + + +def test_compact_prompt_forbids_inventing_values_to_complete_a_pattern() -> None: + """The summarizer must not pattern-complete a paginated list. + + The fabrication incident's invented rows were patterned: sequential + reference codes, alphabetically ordered names. Prohibiting pattern + completion at the summarizer addresses the same failure one layer + earlier than the answering model. + """ + system, _ = _build_llm_compact_prompt_texts() + + assert "never paraphrase, substitute, or invent one to complete a pattern" in system + + +def test_compact_prompt_subordinates_payload_dropping_to_value_preservation() -> None: + """Dropping a raw payload must not be read as licensing dropping its values. + + The instruction to drop "irrelevant raw payloads" and the instruction to + preserve fact-carrying values sit two sentences apart; this states which + one governs a value the request points at. + """ + system, _ = _build_llm_compact_prompt_texts() + + assert "Dropping a raw payload does not license dropping these values" in system + assert "they are not the bulk that instruction covers" in system + assert "irrelevant raw payloads" in system + + +def test_compact_prompt_ranks_what_to_keep_when_the_budget_is_short() -> None: + """When the budget is too small for everything, priority order is explicit. + + At the smallest fallback budget (``COMPACT_SUMMARY_MIN_TOKENS``), a + silent partial summary is the same defect as the incident: a claim of + completeness with no signal that anything was left out. + """ + system, _ = _build_llm_compact_prompt_texts() + + assert "keep, in this order:" in system + tail = system[system.index("keep, in this order:") :] + order = [ + "first state what is missing and not listed here, with counts", + "artifact handles", + "the identifiers and names the request points at", + "statuses and dates", + "then the rest", + ] + positions = [tail.index(item) for item in order] + assert positions == sorted(positions) + + +def test_compact_prompt_stays_within_the_smallest_budget() -> None: + """The prompt itself must not be longer than the smallest summary it asks for. + + ``COMPACT_SUMMARY_MIN_TOKENS`` is 256 tokens, roughly 190 English words; + 310 words already left zero margin for the next required addition, so + the cap is 330 -- room for one more sentence before the next change must + make an explicit trade-off instead of silently growing the prompt. + """ + system, _ = _build_llm_compact_prompt_texts() + + assert len(system.split()) <= 330 + + def test_compact_with_llm_reports_dropped_tool_results_by_name() -> None: ctx = ExecutionContext() ctx.compact_config.threshold = 1 From 5795c8724e8239667f40eaab62b6d773d090285a Mon Sep 17 00:00:00 2001 From: Alexliu Date: Sat, 5 Sep 2026 01:19:30 +0800 Subject: [PATCH 4/9] test(agent): tighten grounding rule test names and drop a subsumed assertion --- tests/core/agent/test_grounding.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/core/agent/test_grounding.py b/tests/core/agent/test_grounding.py index 9384c4129..f9db942d7 100644 --- a/tests/core/agent/test_grounding.py +++ b/tests/core/agent/test_grounding.py @@ -69,7 +69,7 @@ def test_grounding_rule_covers_fact_carrying_tool_arguments() -> None: assert "report the gap instead. The same standard applies" in rule -def test_grounding_rule_exempts_values_the_model_must_compose() -> None: +def test_grounding_rule_exempts_the_wording_the_model_composes() -> None: """The prohibition must not reach arguments the model is meant to author. ReAct runs with ``tool_choice="required"``, and the answer it writes is @@ -140,9 +140,6 @@ def test_grounding_rule_without_tools_omits_tool_argument_clause() -> None: assert "tool-call arguments that assert facts" not in rule assert "never guess one" not in rule - assert ( - "This clause does not reach a default or inferred parameter value" not in rule - ) assert "a default or inferred parameter value" not in rule # The compose exemption and the sourcing rule over composed text are # unconditional, so the no-tools variant still carries them. From b54fa0c9437f97691967f5c2ec2a2c959b0f435c Mon Sep 17 00:00:00 2001 From: Alexliu Date: Mon, 7 Sep 2026 01:12:45 +0800 Subject: [PATCH 5/9] fix(agent): reference the shared value-kind list in auto's routing remedy The auto pattern's routing-remedy sentence and its test assertion each hand-wrote a five-item value-kind list that had drifted from the grounding module's six-item VALUE_KINDS constant (missing "a status"). Both now interpolate VALUE_KINDS directly, so the routing remedy always matches the same value kinds the grounding rule itself forbids, with no second literal copy left to fall out of sync. --- src/xagent/core/agent/pattern/auto/auto.py | 8 ++++---- tests/core/agent/test_auto.py | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/xagent/core/agent/pattern/auto/auto.py b/src/xagent/core/agent/pattern/auto/auto.py index 574fdfbae..8bdb12c2d 100644 --- a/src/xagent/core/agent/pattern/auto/auto.py +++ b/src/xagent/core/agent/pattern/auto/auto.py @@ -26,7 +26,7 @@ build_load_skill_tool, ) from ...frame import ExecutionFrame, ExecutionSnapshot, ExecutionStatus -from ...grounding import grounding_rule +from ...grounding import VALUE_KINDS, grounding_rule from ...language import ( final_answer_language_rule, reset_metadata_output_language, @@ -1289,9 +1289,9 @@ def _decision_prompt( "tool arguments. " f"When writing that answer field: {grounding_rule(can_call_tools=False)} " "If the answer would need any value the rule above forbids you to " - "supply -- a number, a name, an identifier, a date, or a table row " - "that no source here supports -- set existing_context_sufficient=false " - "and choose react, so the agent can obtain it with tools.\n\n" + f"supply -- {VALUE_KINDS} that no source here supports -- set " + "existing_context_sufficient=false and choose react, so the agent " + "can obtain it with tools.\n\n" f"{final_deliverable_file_reference_instructions(can_lookup=False)}\n\n" "You must classify whether " "the latest request requires current or external facts, and whether " diff --git a/tests/core/agent/test_auto.py b/tests/core/agent/test_auto.py index 32c096c95..5ede7081e 100644 --- a/tests/core/agent/test_auto.py +++ b/tests/core/agent/test_auto.py @@ -21,6 +21,7 @@ ReActPattern, ) from xagent.core.agent.context.enrichment import MEMORY_CONTEXT_METADATA_KEY +from xagent.core.agent.grounding import VALUE_KINDS from xagent.core.agent.language import ( OUTPUT_LANGUAGE_METADATA_KEY, OUTPUT_LANGUAGE_SOURCE_METADATA_KEY, @@ -1066,10 +1067,10 @@ async def test_auto_decision_prompt_includes_grounding_rule() -> None: "set existing_context_sufficient=false and choose react, so the agent " "can obtain it with tools" in decision_prompt ) - assert ( - "a number, a name, an identifier, a date, or a table row that no " - "source here supports" in decision_prompt - ) + # The value kinds are not auto's own wording: the sibling sentence + # interpolates the shared constant, so this pins the reference rather + # than restating the list. + assert f"{VALUE_KINDS} that no source here supports" in decision_prompt assert "such unsupported specifics" not in decision_prompt assert "get_workspace_output_files" not in decision_prompt assert "You must classify whether" in decision_prompt From c4352cd52b315ca864c58a91122f63569a8b2614 Mon Sep 17 00:00:00 2001 From: Alexliu Date: Tue, 8 Sep 2026 11:53:06 +0800 Subject: [PATCH 6/9] fix(agent): widen the compaction trailers to the shared value-kind list --- src/xagent/core/agent/context/execution.py | 11 +++++---- tests/core/agent/test_context.py | 27 +++++++++++++++++----- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/xagent/core/agent/context/execution.py b/src/xagent/core/agent/context/execution.py index c4c703c06..63c63204b 100644 --- a/src/xagent/core/agent/context/execution.py +++ b/src/xagent/core/agent/context/execution.py @@ -25,6 +25,7 @@ format_tool_result_for_observation, sanitize_tool_result_for_public_context, ) +from ..grounding import VALUE_KINDS from ..language import ( effective_output_language, render_dag_step_language_reference, @@ -1295,7 +1296,7 @@ def compact_with_llm_response( "explicitly asks to restart, revise, or regenerate them, or the detail " "you need was lost in compaction. This summary is a lossy paraphrase of " "the raw history, not the history itself: when the answer needs an exact " - "value, figure, statistic, table row, quotation, or identifier that this " + f"statistic, quotation, or other value -- {VALUE_KINDS} -- that this " "summary does not literally contain, re-read or re-query the source " "instead of reconstructing the value from this summary or from memory. " "Only re-run tools that read; if the value came from a tool that writes, " @@ -1453,7 +1454,7 @@ def _dropped_tool_results_notice(counts: dict[str, int]) -> str: """Describe the tool observations this compaction removes from context. Without this, the summary silently replaces every retrieved value and - the agent cannot tell a remembered figure from an invented one. + the agent cannot tell a remembered value from an invented one. """ if not counts: return "" @@ -1462,9 +1463,9 @@ def _dropped_tool_results_notice(counts: dict[str, int]) -> str: prefix = ( f"Raw observations from {total} tool {call_label} dropped by this " "compaction. Their exact values are no longer in context; only the " - "summary above describes them. Treat any figure not literally present in " - "that summary as unavailable rather than recalled. Tools whose results " - "were dropped:\n" + "summary above describes them. Treat any value not literally present in " + f"that summary -- {VALUE_KINDS} -- as unavailable rather than recalled. " + "Tools whose results were dropped:\n" ) # Tool names can come from dynamic MCP server config, so bound both the # per-name length and the total notice size the way the sibling diff --git a/tests/core/agent/test_context.py b/tests/core/agent/test_context.py index f096a9a86..cc76ee81a 100644 --- a/tests/core/agent/test_context.py +++ b/tests/core/agent/test_context.py @@ -22,6 +22,7 @@ enrich_context_with_memory, ) from xagent.core.agent.context.execution import CLOCK_TIMEZONE_METADATA_KEY +from xagent.core.agent.grounding import VALUE_KINDS from xagent.core.agent.language import ( OUTPUT_LANGUAGE_METADATA_KEY, detect_prose_script_mismatch, @@ -1092,6 +1093,13 @@ class CompactLLM: assert "current execution state" in ctx.messages[0].content assert "do not repeat completed tool calls" in ctx.messages[0].content assert "lost in compaction" in ctx.messages[0].content + # The trailer's own value-kind scope has to be the rule's, not a + # narrower list of its own: this is the text the next call reads when + # deciding whether to re-fetch a value or recall it. + assert ( + f"exact statistic, quotation, or other value -- {VALUE_KINDS} --" + in ctx.messages[0].content + ) assert "re-read or re-query the source" in ctx.messages[0].content assert "Only re-run tools that read" in ctx.messages[0].content assert "- read_file" in ctx.messages[0].content @@ -1252,13 +1260,16 @@ def test_compact_prompt_ranks_what_to_keep_when_the_budget_is_short() -> None: assert positions == sorted(positions) -def test_compact_prompt_stays_within_the_smallest_budget() -> None: - """The prompt itself must not be longer than the smallest summary it asks for. +def test_compact_prompt_does_not_grow_past_its_measured_ceiling() -> None: + """The prompt must not grow a sentence at a time without an explicit trade. - ``COMPACT_SUMMARY_MIN_TOKENS`` is 256 tokens, roughly 190 English words; - 310 words already left zero margin for the next required addition, so - the cap is 330 -- room for one more sentence before the next change must - make an explicit trade-off instead of silently growing the prompt. + 330 is a growth ceiling, not a derived limit: the prompt measured 327 + words when the cap was set, and the cap sits three words above that, + deliberately less than one sentence, so the next sentence added here + hits the cap and has to drop something to fit. Nothing enforces a + prompt length at runtime. ``COMPACT_SUMMARY_MIN_TOKENS`` does not: it + bounds the summary the model writes (``_llm_compact_max_tokens`` passes + it as ``max_tokens``), never the length of the prompt asking for it. """ system, _ = _build_llm_compact_prompt_texts() @@ -1297,6 +1308,10 @@ def test_compact_with_llm_reports_dropped_tool_results_by_name() -> None: assert "4 tool calls were dropped" in notice assert "- web_search x3" in notice assert "- read_file" in notice + assert ( + f"Treat any value not literally present in that summary -- {VALUE_KINDS} --" + in notice + ) assert "unavailable rather than recalled" in notice assert result.metadata["dropped_tool_result_count"] == 4 assert result.metadata["dropped_tool_results_by_name"] == { From 28292169415b67e3490a6731bc4dda037a82e98d Mon Sep 17 00:00:00 2001 From: Alexliu Date: Tue, 8 Sep 2026 11:53:34 +0800 Subject: [PATCH 7/9] fix(agent): let the template exception reach content written to a tool --- src/xagent/core/agent/grounding.py | 21 ++++++------ tests/core/agent/test_dag.py | 3 ++ tests/core/agent/test_grounding.py | 52 +++++++++++++++++++++++++++--- 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/src/xagent/core/agent/grounding.py b/src/xagent/core/agent/grounding.py index 7ffd5c353..3e2500448 100644 --- a/src/xagent/core/agent/grounding.py +++ b/src/xagent/core/agent/grounding.py @@ -48,7 +48,8 @@ def grounding_rule(*, can_call_tools: bool = True) -> str: forbids supplying a fact-carrying tool-call argument that no source provides, while leaving arguments the model is expected to compose untouched -- except for a fact value written literally inside composed - code or document text, which the sourcing requirement still covers. + code or document text, which the sourcing requirement still covers + unless the request explicitly asked for a template or a sample. """ insufficient_context_rule = ( "If available context is insufficient, say so or use an appropriate " @@ -95,14 +96,16 @@ def grounding_rule(*, can_call_tools: bool = True) -> str: "restricts every fact asserted inside that wording. A fact value " "written literally inside such composed code or text is still " "subject to the sourcing rule above: the text you compose is yours; " - f"{VALUE_KINDS} that you place inside it is not. The only case in " - "which content that no source supports may appear in the answer is a " - "current user request that explicitly asks you to write a template, " - "a sample, or content that is not meant to be real; in that case, " - "before any of that content appears in the answer, state that the " - "request asked for content that is not real and that none of it " - "comes from a data source, and keep such content to what the request " - "asked for. Outside that case a caveat does not make an invented " + f"a value you place inside it -- {VALUE_KINDS} -- is not. The only " + "case in which content that no source supports may appear -- in the " + "answer, or inside document text or other content the request asks " + "you to write and hand to a tool -- is a current user request that " + "explicitly asks you to write a template or a sample, meaning " + "content that is not meant to be real; in that case, before any of " + "that content appears, state in your reply that the request asked " + "for content that is not real and that none of it comes from a data " + "source, and keep such content to what the request asked for. " + "Outside that case a caveat does not make an invented " "value acceptable: if you find yourself about to add a note " "explaining that some values are not real, remove those values and " "report the gap instead." diff --git a/tests/core/agent/test_dag.py b/tests/core/agent/test_dag.py index 28c27b3d6..ff1278667 100644 --- a/tests/core/agent/test_dag.py +++ b/tests/core/agent/test_dag.py @@ -419,6 +419,9 @@ def test_dag_completion_assessment_prompt_includes_grounding_rule() -> None: # the rule no longer uses. assert "name that missing data in reason" in system_prompt assert "illustrative" not in system_prompt + # A forward marker, not a regression guard: this phrasing was the first + # design draft's remedy wording and was replaced, so it has never been + # in the prompt. It goes red if someone reaches for it again. assert "report that value as unavailable" not in system_prompt answer_description = pattern._completion_assessment_tool_schema()["function"][ diff --git a/tests/core/agent/test_grounding.py b/tests/core/agent/test_grounding.py index f9db942d7..573414515 100644 --- a/tests/core/agent/test_grounding.py +++ b/tests/core/agent/test_grounding.py @@ -65,6 +65,12 @@ def test_grounding_rule_covers_fact_carrying_tool_arguments() -> None: "omit it when the tool allows", ): assert phrase in rule + # A page size the model picks is not a claim about the world, so pausing + # for it would be the over-asking failure the exemption exists to prevent. + assert ( + "does not reach a default or inferred parameter value such as a page " + "size or result limit" in rule + ) # Pin the concatenation onto the answer rules that precede it. assert "report the gap instead. The same standard applies" in rule @@ -131,6 +137,9 @@ def test_grounding_rule_keeps_literal_facts_inside_composed_text_sourced() -> No "still subject to the sourcing rule above: the text you compose " "is yours" in rule ) + # The list is set off before "is not" so the qualifier reads against + # the whole list, not against its last member alone. + assert f"a value you place inside it -- {VALUE_KINDS} -- is not" in rule assert rule.count(VALUE_KINDS) == 2 @@ -213,8 +222,9 @@ def test_grounding_rule_exception_requires_an_explicit_current_request() -> None for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): assert ( "The only case in which content that no source supports may " - "appear in the answer is a current user request that explicitly " - "asks" in rule + "appear -- in the answer, or inside document text or other " + "content the request asks you to write and hand to a tool -- is " + "a current user request that explicitly asks" in rule ) assert ( "Outside that case a caveat does not make an invented value " @@ -225,10 +235,39 @@ def test_grounding_rule_exception_requires_an_explicit_current_request() -> None def test_grounding_rule_states_sample_nature_before_presenting_it() -> None: """On the exception path, the disclosure must precede the content.""" for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): - assert "before any of that content appears in the answer, state" in rule - assert rule.index("before any of that content appears") < rule.index( - "keep such content to what the request asked for" + assert "before any of that content appears, state in your reply" in rule + + +def test_grounding_rule_exception_reaches_content_bound_for_a_tool_argument() -> None: + """A sample the user asked for is often written into a file, not the answer. + + "Write a sample invoice and save it to sample-invoice.md" delivers the + requested content through a tool argument. An exception scoped to the + answer alone would leave the request unanswerable: the compose exemption + hands every fact inside that content back to the sourcing rule, so the + exception has to reach the same destination the content goes to. + """ + for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): + assert ( + "may appear -- in the answer, or inside document text or other " + "content the request asks you to write and hand to a tool --" in rule + ) + + +def test_grounding_rule_exception_trigger_qualifies_what_a_sample_means() -> None: + """The trigger must say what a sample is, not offer a third alternative. + + As a third alternative it could never constrain the second: a request + classified as "a sample" satisfied the trigger before that phrase was + read, so "make me a sample table of last quarter's real refunds" opened + the exception. + """ + for rule in (grounding_rule(), grounding_rule(can_call_tools=False)): + assert ( + "write a template or a sample, meaning content that is not meant " + "to be real" in rule ) + assert "a sample, or content that is not meant to be real" not in rule def test_grounding_rule_rejects_caveat_as_a_substitute_for_omission() -> None: @@ -269,6 +308,9 @@ def test_grounding_module_docstring_states_the_default_as_a_prohibition() -> Non doc = grounding.__doc__ or "" assert "instructed default" not in doc normalized_doc = " ".join(doc.split()) + # A denial of one phrasing is evaded by any synonym, so pin the claim + # the docstring must positively make. + assert "makes reporting the gap the instructed response" in normalized_doc assert ( "Proposals B (evidence-preserving compaction) and C (provenance " "tracking and a data-source gate) remain open." in normalized_doc From 5b8c3a5f78aae8b7fcd37d7d5871aca5e540ad78 Mon Sep 17 00:00:00 2001 From: Alexliu Date: Tue, 8 Sep 2026 20:24:54 +0800 Subject: [PATCH 8/9] test(agent): state what the forward marker in the dag prompt test guards --- tests/core/agent/test_dag.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/agent/test_dag.py b/tests/core/agent/test_dag.py index ff1278667..df23c4115 100644 --- a/tests/core/agent/test_dag.py +++ b/tests/core/agent/test_dag.py @@ -419,9 +419,9 @@ def test_dag_completion_assessment_prompt_includes_grounding_rule() -> None: # the rule no longer uses. assert "name that missing data in reason" in system_prompt assert "illustrative" not in system_prompt - # A forward marker, not a regression guard: this phrasing was the first - # design draft's remedy wording and was replaced, so it has never been - # in the prompt. It goes red if someone reaches for it again. + # A forward marker, not a regression guard: this phrasing has never been + # in the prompt. It goes red if someone reaches for "report as + # unavailable" instead of naming the missing data in reason. assert "report that value as unavailable" not in system_prompt answer_description = pattern._completion_assessment_tool_schema()["function"][ From 57e455282ae43ba8c4ab4860dde448a748791108 Mon Sep 17 00:00:00 2001 From: Alexliu Date: Tue, 8 Sep 2026 23:15:59 +0800 Subject: [PATCH 9/9] fix(agent): keep compaction notices whole and their scope unambiguous The dropped-tool notice prefix spells out the shared value-kind list, and at 1024 characters that prefix crowded real tool names out of the very list of lost evidence the notice exists to give; 1152 holds a full page of names at the length an MCP server produces. Punctuate auto's routing remedy so "that no source here supports" qualifies the value the answer would need rather than the last kind in the list, and tell the compaction trailer's reader to report a value as unavailable when no tool can supply it at all. In the compaction prompt, the credential exclusion now covers a handle the request points at as well as an identifier, and the sentence forbidding instructions to the next call sits with the other rules about what the summary may say. Tests take the single-dropped-observation setup from one shared helper. --- src/xagent/core/agent/context/execution.py | 27 +++++--- src/xagent/core/agent/pattern/auto/auto.py | 2 +- tests/core/agent/test_auto.py | 2 +- tests/core/agent/test_context.py | 78 +++++++++++++++++----- 4 files changed, 80 insertions(+), 29 deletions(-) diff --git a/src/xagent/core/agent/context/execution.py b/src/xagent/core/agent/context/execution.py index 63c63204b..4451dcfc5 100644 --- a/src/xagent/core/agent/context/execution.py +++ b/src/xagent/core/agent/context/execution.py @@ -103,7 +103,9 @@ COMPACT_SUMMARY_FALLBACK_BUDGETS = (4096, 2048, 1024, COMPACT_SUMMARY_MIN_TOKENS) COMPACT_CONTEXT_REF_MAX_TOKENS = 2048 COMPACT_DROPPED_REF_NOTICE_MAX_CHARS = 2048 -COMPACT_DROPPED_TOOL_NOTICE_MAX_CHARS = 1024 +# Sized so the notice prefix, which spells out the shared VALUE_KINDS list, +# leaves room for the full name list rather than crowding names out of it. +COMPACT_DROPPED_TOOL_NOTICE_MAX_CHARS = 1152 COMPACT_DROPPED_TOOL_NAME_MAX_CHARS = 64 # load_skill retrieves guidance, not evidence, and re-running it restores @@ -1299,6 +1301,8 @@ def compact_with_llm_response( f"statistic, quotation, or other value -- {VALUE_KINDS} -- that this " "summary does not literally contain, re-read or re-query the source " "instead of reconstructing the value from this summary or from memory. " + "If no tool can supply it, report it as unavailable rather than " + "reconstructing it. " "Only re-run tools that read; if the value came from a tool that writes, " "sends, executes, or otherwise changes state, do not re-run it -- re-read " "the artifact it produced, or report the value as unavailable. " @@ -1526,19 +1530,20 @@ def _build_llm_compact_prompt( "password, or other authentication material, or personal " "information the request does not point at; note only that " "such a value was present and was omitted. If a value is both " - "an identifier the request points at and authentication " - "material, the exclusion wins: omit it. If your budget cannot " - "hold all of this, keep, in this order: first state what is " - "missing and not listed here, with counts; artifact handles; " - "the identifiers and names the request points at; statuses " - "and dates; then the rest. Separate completed work from " - "remaining work. Report only what happened and what is " + "an identifier or handle the request points at and " + "authentication material, the exclusion wins: omit it. If " + "your budget cannot hold all of this, keep, in this order: " + "first state what is missing and not listed here, with " + "counts; artifact handles; the identifiers and names the " + "request points at; statuses and dates; then the rest. " + "Separate completed work from remaining work. Write no " + "instruction to the next call about tool use or whether to " + "answer: that decision is not yours and its tools are " + "unknown to you. Report only what happened and what is " "missing: never call a dataset complete, fully retrieved, or " "fully processed unless the history shows every item was " "returned and every one is still described here; say which " - "parts survive as prose only. Write no instruction to the " - "next call about tool use or whether to answer: that decision " - "is not yours and its tools are unknown to you. Preserve the " + "parts survive as prose only. Preserve the " "language of user-facing requests and constraints; keep " "multilingual details in their original language. Return only " "the compact summary." diff --git a/src/xagent/core/agent/pattern/auto/auto.py b/src/xagent/core/agent/pattern/auto/auto.py index 8bdb12c2d..d9b6adf50 100644 --- a/src/xagent/core/agent/pattern/auto/auto.py +++ b/src/xagent/core/agent/pattern/auto/auto.py @@ -1289,7 +1289,7 @@ def _decision_prompt( "tool arguments. " f"When writing that answer field: {grounding_rule(can_call_tools=False)} " "If the answer would need any value the rule above forbids you to " - f"supply -- {VALUE_KINDS} that no source here supports -- set " + f"supply -- {VALUE_KINDS} -- that no source here supports, set " "existing_context_sufficient=false and choose react, so the agent " "can obtain it with tools.\n\n" f"{final_deliverable_file_reference_instructions(can_lookup=False)}\n\n" diff --git a/tests/core/agent/test_auto.py b/tests/core/agent/test_auto.py index 5ede7081e..08b2a3377 100644 --- a/tests/core/agent/test_auto.py +++ b/tests/core/agent/test_auto.py @@ -1070,7 +1070,7 @@ async def test_auto_decision_prompt_includes_grounding_rule() -> None: # The value kinds are not auto's own wording: the sibling sentence # interpolates the shared constant, so this pins the reference rather # than restating the list. - assert f"{VALUE_KINDS} that no source here supports" in decision_prompt + assert f"-- {VALUE_KINDS} -- that no source here supports" in decision_prompt assert "such unsupported specifics" not in decision_prompt assert "get_workspace_output_files" not in decision_prompt assert "You must classify whether" in decision_prompt diff --git a/tests/core/agent/test_context.py b/tests/core/agent/test_context.py index cc76ee81a..5d2fa4af5 100644 --- a/tests/core/agent/test_context.py +++ b/tests/core/agent/test_context.py @@ -21,7 +21,10 @@ _lookup_relevant_memories_with_context, enrich_context_with_memory, ) -from xagent.core.agent.context.execution import CLOCK_TIMEZONE_METADATA_KEY +from xagent.core.agent.context.execution import ( + CLOCK_TIMEZONE_METADATA_KEY, + COMPACT_DROPPED_TOOL_NOTICE_MAX_NAMES, +) from xagent.core.agent.grounding import VALUE_KINDS from xagent.core.agent.language import ( OUTPUT_LANGUAGE_METADATA_KEY, @@ -1043,13 +1046,16 @@ def test_compact_truncate_preserves_tool_call_pair_boundary() -> None: assert ctx.messages[2].tool_call_id == "call-2" -def test_compact_with_llm_summarizes_history_and_preserves_current_user() -> None: - class CompactLLM: - model_name = "compact-test" +def _ctx_with_one_dropped_tool_result(user_message: str) -> ExecutionContext: + """Return a context holding exactly one compactable tool observation. + The threshold of 1 makes the next compaction fire, and the single + ``read_file`` result is the only evidence it drops, so both the summary + trailer and the compaction prompt can be read off the same setup. + """ ctx = ExecutionContext() ctx.compact_config.threshold = 1 - ctx.add_user_message("current request") + ctx.add_user_message(user_message) ctx.add_assistant_message( "", tool_calls=[ @@ -1057,6 +1063,14 @@ class CompactLLM: ], ) ctx.add_tool_result("read_file", {"output": "x" * 200}, tool_call_id="call-1") + return ctx + + +def test_compact_with_llm_summarizes_history_and_preserves_current_user() -> None: + class CompactLLM: + model_name = "compact-test" + + ctx = _ctx_with_one_dropped_tool_result("current request") llm = CompactLLM() request = ctx.build_llm_compact_request_if_needed() @@ -1109,16 +1123,7 @@ class CompactLLM: def _build_llm_compact_prompt_texts() -> tuple[str, str]: - ctx = ExecutionContext() - ctx.compact_config.threshold = 1 - ctx.add_user_message("Build a KPI report") - ctx.add_assistant_message( - "", - tool_calls=[ - {"id": "call-1", "type": "function", "function": {"name": "read_file"}}, - ], - ) - ctx.add_tool_result("read_file", {"output": "x" * 200}, tool_call_id="call-1") + ctx = _ctx_with_one_dropped_tool_result("Build a KPI report") request = ctx.build_llm_compact_request_if_needed() assert request is not None prompt = request["messages"] @@ -1206,7 +1211,7 @@ def test_compact_prompt_excludes_credentials_even_when_also_an_identifier() -> N system, _ = _build_llm_compact_prompt_texts() assert ( - "If a value is both an identifier the request points at and " + "If a value is both an identifier or handle the request points at and " "authentication material, the exclusion wins: omit it." in system ) @@ -1481,6 +1486,47 @@ def test_compact_with_llm_caps_and_clamps_dropped_tool_names() -> None: assert result.metadata["dropped_tool_result_count"] == len(tool_names) +def test_compact_with_llm_lists_a_full_page_of_long_tool_names() -> None: + """The char budget has to hold the names, not just the notice prefix. + + That prefix spells out the shared value-kind list, and an MCP server + contributes names much longer than a builtin tool's. A budget sized + without that headroom drops names the run actually used while the + per-name cap is nowhere near reached. + """ + ctx = ExecutionContext() + ctx.compact_config.threshold = 1 + ctx.add_user_message("Run many MCP tools") + tool_names = [ + f"mcp_analytics_server_report_row_{index:02d}" + for index in range(COMPACT_DROPPED_TOOL_NOTICE_MAX_NAMES) + ] + # 34 is the longest name the budget fits a full page of; shorter names + # would still fit a budget that left no room for the prefix. + assert all(len(name) == 34 for name in tool_names) + for index, tool_name in enumerate(tool_names): + call_id = f"call-{index}" + ctx.add_assistant_message( + "", + tool_calls=[ + { + "id": call_id, + "type": "function", + "function": {"name": tool_name}, + } + ], + ) + ctx.add_tool_result(tool_name, {"output": f"rows {index}"}, call_id) + + result = ctx.compact_with_llm_response({"content": "Ran many MCP tools."}) + + notice = ctx.messages[0].content + for tool_name in tool_names: + assert f"- {tool_name}" in notice + assert "additional" not in notice + assert result.metadata["dropped_tool_result_count"] == len(tool_names) + + def test_compact_with_llm_orders_ref_notice_before_tool_notice() -> None: ctx = ExecutionContext() ctx.compact_config.threshold = 1