Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 41 additions & 18 deletions src/xagent/core/agent/context/execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
format_tool_result_for_observation,
sanitize_tool_result_for_public_context,
)
from ..grounding import VALUE_KINDS
from ..language import (
effective_output_language,
render_dag_step_language_reference,
Expand Down Expand Up @@ -1295,7 +1296,7 @@ def compact_with_llm_response(
"explicitly asks to restart, revise, or regenerate them, or the detail "
"you need was lost in compaction. This summary is a lossy paraphrase of "
"the raw history, not the history itself: when the answer needs an exact "
"value, figure, statistic, table row, quotation, or identifier that this "
f"statistic, quotation, or other value -- {VALUE_KINDS} -- that this "
"summary does not literally contain, re-read or re-query the source "
"instead of reconstructing the value from this summary or from memory. "
"Only re-run tools that read; if the value came from a tool that writes, "
Comment thread
AlexLiu190625 marked this conversation as resolved.
Expand Down Expand Up @@ -1453,7 +1454,7 @@ def _dropped_tool_results_notice(counts: dict[str, int]) -> str:
"""Describe the tool observations this compaction removes from context.

Without this, the summary silently replaces every retrieved value and
the agent cannot tell a remembered figure from an invented one.
the agent cannot tell a remembered value from an invented one.
"""
if not counts:
return ""
Expand All @@ -1462,9 +1463,9 @@ def _dropped_tool_results_notice(counts: dict[str, int]) -> str:
prefix = (
f"Raw observations from {total} tool {call_label} dropped by this "
"compaction. Their exact values are no longer in context; only the "
"summary above describes them. Treat any figure not literally present in "
"that summary as unavailable rather than recalled. Tools whose results "
"were dropped:\n"
"summary above describes them. Treat any value not literally present in "
f"that summary -- {VALUE_KINDS} -- as unavailable rather than recalled. "
Comment thread
AlexLiu190625 marked this conversation as resolved.
"Tools whose results were dropped:\n"
)
# Tool names can come from dynamic MCP server config, so bound both the
# per-name length and the total notice size the way the sibling
Expand Down Expand Up @@ -1509,17 +1510,38 @@ def _build_llm_compact_prompt(
"tool calls, tool observations, files or URLs mentioned, "
"decisions made, and open work. Drop duplicated search noise, "
"irrelevant raw payloads, and verbose intermediate text. "
"Preserve exact reusable artifact handles, including file_id "
"values, file: references, markdown file links, URLs, relative "
"paths, absolute paths, output_path, image_path, video_path, "
"artifact filenames, and any other path-like result fields; do "
"not replace machine-usable handles with only descriptive "
"filenames. Clearly separate completed work from remaining work "
"and name the next action needed. "
"Preserve the language of user-facing requests and constraints; "
"if the history is multilingual, keep important details in their "
"original language instead of translating them. "
"Return only the compact summary."
"Preserve exact reusable artifact handles -- file_id values, "
"file: references, markdown links, URLs, paths, output_path, "
"image_path, video_path, artifact filenames, any other "
"path-like field -- never a descriptive filename instead. "
"Preserve, character for character, the values a tool result "
"returned for the records the request points at: their names, "
Comment thread
AlexLiu190625 marked this conversation as resolved.
"the people, organizations or teams they belong to, their "
"identifiers and reference codes, their statuses, dates, "
"counts and totals. Copy such a value or omit it; never "
"paraphrase, substitute, or invent one to complete a pattern. "
"Dropping a raw payload does not license dropping these "
"values; they are not the bulk that instruction covers. "
"Never copy, in whole or in part, a credential, token, key, "
"password, or other authentication material, or personal "
"information the request does not point at; note only that "
"such a value was present and was omitted. If a value is both "
"an identifier the request points at and authentication "
"material, the exclusion wins: omit it. If your budget cannot "
"hold all of this, keep, in this order: first state what is "
"missing and not listed here, with counts; artifact handles; "
"the identifiers and names the request points at; statuses "
"and dates; then the rest. Separate completed work from "
Comment thread
AlexLiu190625 marked this conversation as resolved.
Outdated
"remaining work. Report only what happened and what is "
"missing: never call a dataset complete, fully retrieved, or "
"fully processed unless the history shows every item was "
"returned and every one is still described here; say which "
"parts survive as prose only. Write no instruction to the "
Comment thread
AlexLiu190625 marked this conversation as resolved.
Outdated
"next call about tool use or whether to answer: that decision "
"is not yours and its tools are unknown to you. Preserve the "
"language of user-facing requests and constraints; keep "
"multilingual details in their original language. Return only "
"the compact summary."
),
},
{
Expand All @@ -1528,8 +1550,9 @@ def _build_llm_compact_prompt(
"Conversation history to compact:\n"
f"{transcript}\n\n"
"Write a concise but complete continuity summary for the next "
"LLM call. The next LLM call should be able to continue without "
"redoing completed tool calls."
"LLM call. Record which tool calls already completed and what "
"they returned, so the next call can judge for itself what "
"still needs doing."
),
},
]
Expand Down
77 changes: 49 additions & 28 deletions src/xagent/core/agent/grounding.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,20 @@
remedy -- ask the user, or finish reporting the gap -- belongs to the calling
pattern, which owns the user-interaction policy this module cannot see.

This is the proposal-A mitigation from issue #1235. It reduces how often
unsourced figures are emitted and makes disclosure the instructed default, but
it cannot repair a session whose evidence compaction already discarded.
This is the proposal-A mitigation from issue #1235. It forbids unsourced values
by default and makes reporting the gap the instructed response, but it cannot
repair a session whose evidence compaction already discarded.
Proposals B (evidence-preserving compaction) and C (provenance tracking and a
data-source gate) remain open.
"""

from __future__ import annotations

VALUE_KINDS = (
Comment thread
AlexLiu190625 marked this conversation as resolved.
"a number, a person or organization name, an identifier or reference "
"code, a date, a status, or a row of a table"
)


def grounding_rule(*, can_call_tools: bool = True) -> str:
"""Return the grounding rule for answer text and, optionally, tool arguments.
Expand All @@ -36,13 +41,15 @@ def grounding_rule(*, can_call_tools: bool = True) -> str:
possible.

Returns:
A prompt fragment forbidding unsupported claims and unsourced
quantitative data, and requiring up-front disclosure of any
illustrative figures. When ``can_call_tools`` is true it also forbids
supplying a fact-carrying tool-call argument that no source provides,
while leaving arguments the model is expected to compose untouched --
except for a fact value written literally inside composed code or
document text, which the sourcing requirement still covers.
A prompt fragment forbidding unsupported claims and unsourced values of
every kind it enumerates, requiring the gap be reported instead, and
confining unsourced content to a current request that explicitly asks
for a template or sample. When ``can_call_tools`` is true it also
forbids supplying a fact-carrying tool-call argument that no source
provides, while leaving arguments the model is expected to compose
untouched -- except for a fact value written literally inside composed
code or document text, which the sourcing requirement still covers
unless the request explicitly asked for a template or a sample.
"""
insufficient_context_rule = (
"If available context is insufficient, say so or use an appropriate "
Expand All @@ -61,17 +68,15 @@ def grounding_rule(*, can_call_tools: bool = True) -> str:
"messages, the retrieved context, or a value an earlier tool result "
"actually returned; never guess one, never substitute a "
"plausible-looking placeholder for one the user has not given, and "
"never carry one over from a different record. This does not restrict "
"values you are expected to compose yourself, such as a search query, "
"code or a command you write to do the work, a message or answer you "
"write to the user, or document text you were asked to produce. A fact "
"value written literally inside such composed code or text is still "
"subject to the sourcing rule above. The rule also does not reach a "
"default or inferred parameter value such as a page size or result "
"limit, which you are expected to decide yourself. Treat "
"a fact-carrying value you cannot source as "
"missing information rather than inventing it, and omit it when the "
"tool allows it to be omitted."
"never carry one over from a different record. The answer you write "
"to the user reaches you as an argument too; it is wording you "
"compose, so this argument standard does not reach it, while the "
"sourcing rule above still governs every fact inside it. This clause "
"does not reach a default or inferred parameter value such as a page "
Comment thread
AlexLiu190625 marked this conversation as resolved.
"size or result limit, which you are expected to decide yourself. "
"Treat a fact-carrying value you cannot source as missing information "
"rather than inventing it, and omit it when the tool allows it to be "
"omitted."
if can_call_tools
else ""
)
Expand All @@ -81,12 +86,28 @@ def grounding_rule(*, can_call_tools: bool = True) -> str:
"statistics, percentages, table rows, or time series) that are not "
"supported by the conversation, retrieved context, or tool results. "
f"{insufficient_context_rule}"
"Never invent figures to fill a gap, and never present invented numbers "
"as real data; produce unsupported figures only when the user explicitly "
"asked for a template, mockup, or illustrative example. Labeling is "
"required either way: if the answer ends up containing any figure that no "
"tool result or provided context supports, whether or not the user asked "
"for one, say so up front, before presenting it, and state that those "
"figures are illustrative placeholders not drawn from any data source."
f"Never fill a gap with an invented value, whether it is {VALUE_KINDS}: "
Comment thread
AlexLiu190625 marked this conversation as resolved.
"when nothing in this conversation, the provided context, or a tool "
"result supports a value the answer needs, leave that value out and "
"say plainly that it is missing, rather than supplying one that "
"looks right. This does not restrict the wording you compose -- how "
"you phrase your reply, a search query, code or a command you write "
"to do the work, or document text you were asked to produce -- it "
"restricts every fact asserted inside that wording. A fact value "
"written literally inside such composed code or text is still "
Comment thread
AlexLiu190625 marked this conversation as resolved.
"subject to the sourcing rule above: the text you compose is yours; "
f"a value you place inside it -- {VALUE_KINDS} -- is not. The only "
"case in which content that no source supports may appear -- in the "
"answer, or inside document text or other content the request asks "
"you to write and hand to a tool -- is a current user request that "
"explicitly asks you to write a template or a sample, meaning "
"content that is not meant to be real; in that case, before any of "
"that content appears, state in your reply that the request asked "
"for content that is not real and that none of it comes from a data "
"source, and keep such content to what the request asked for. "
"Outside that case a caveat does not make an invented "
"value acceptable: if you find yourself about to add a note "
"explaining that some values are not real, remove those values and "
"report the gap instead."
f"{tool_argument_rule}"
)
9 changes: 5 additions & 4 deletions src/xagent/core/agent/pattern/auto/auto.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
build_load_skill_tool,
)
from ...frame import ExecutionFrame, ExecutionSnapshot, ExecutionStatus
from ...grounding import grounding_rule
from ...grounding import VALUE_KINDS, grounding_rule
from ...language import (
final_answer_language_rule,
reset_metadata_output_language,
Expand Down Expand Up @@ -1288,9 +1288,10 @@ def _decision_prompt(
"answer field in the same tool call. Put action before answer in the "
"tool arguments. "
f"When writing that answer field: {grounding_rule(can_call_tools=False)} "
"If the answer would need such unsupported specifics, set "
"existing_context_sufficient=false and choose react so the agent can "
"verify them with tools.\n\n"
"If the answer would need any value the rule above forbids you to "
f"supply -- {VALUE_KINDS} that no source here supports -- set "
Comment thread
AlexLiu190625 marked this conversation as resolved.
Outdated
"existing_context_sufficient=false and choose react, so the agent "
"can obtain it with tools.\n\n"
f"{final_deliverable_file_reference_instructions(can_lookup=False)}\n\n"
"You must classify whether "
"the latest request requires current or external facts, and whether "
Expand Down
5 changes: 2 additions & 3 deletions src/xagent/core/agent/pattern/dag/dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -1583,9 +1583,8 @@ def _completion_assessment_messages(self, context: Any) -> list[dict[str, Any]]:
"over from candidate_output or step_results: "
f"{grounding_rule(can_call_tools=False)}\n\n"
f"{final_deliverable_file_reference_instructions(can_lookup=False)}\n\n"
"If the answer presents any figure as an illustrative "
"placeholder because no step produced the underlying data, "
"name that unsourced data in reason even when you choose "
"If the answer leaves out a value because no step produced "
"it, name that missing data in reason even when you choose "
"status=completed. "
f"{final_answer_language_rule(subject='output_language_policy field')}"
),
Expand Down
19 changes: 17 additions & 2 deletions tests/core/agent/test_auto.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
ReActPattern,
)
from xagent.core.agent.context.enrichment import MEMORY_CONTEXT_METADATA_KEY
from xagent.core.agent.grounding import VALUE_KINDS
from xagent.core.agent.language import (
OUTPUT_LANGUAGE_METADATA_KEY,
OUTPUT_LANGUAGE_SOURCE_METADATA_KEY,
Expand Down Expand Up @@ -1051,12 +1052,26 @@ async def test_auto_decision_prompt_includes_grounding_rule() -> None:
assert result["success"] is True
decision_prompt = llm.calls[0]["messages"][-1]["content"]
assert "quantitative data" in decision_prompt
assert "illustrative placeholders" in decision_prompt
assert (
"a current user request that explicitly asks you to write a template"
in decision_prompt
)
assert "invented values" in decision_prompt
assert decision_prompt.count("## FINAL DELIVERABLE FILE REFERENCES") == 1
assert decision_prompt.index(
"If the answer would need such unsupported specifics"
"If the answer would need any value the rule above forbids"
) < decision_prompt.index("## FINAL DELIVERABLE FILE REFERENCES")
# The routing remedy stays specific to auto's own decision, so it is
# worded independently of the shared rule's neutral gap-reporting text.
assert (
"set existing_context_sufficient=false and choose react, so the agent "
"can obtain it with tools" in decision_prompt
)
# The value kinds are not auto's own wording: the sibling sentence
# interpolates the shared constant, so this pins the reference rather
# than restating the list.
assert f"{VALUE_KINDS} that no source here supports" in decision_prompt
assert "such unsupported specifics" not in decision_prompt
assert "get_workspace_output_files" not in decision_prompt
assert "You must classify whether" in decision_prompt
assert "You must also classify whether" not in decision_prompt
Expand Down
Loading