Skip to content

Commit 9975617

Browse files
[openai-agents] Map GuardrailSpanData to the semconv run_guardrail span
Adds GuardrailInvocation to opentelemetry-util-genai and stops dropping agents.tracing.span_data.GuardrailSpanData in GenAITracingProcessor. gen_ai.guardrail.verdict.type is computed when the span ends because the Agents SDK sets GuardrailSpanData.triggered after the span starts. gen_ai.guardrail.target.type is not emitted from the openai-agents processor because the SDK span data does not record direction. Assisted-by: GPT-5 Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Signed-off-by: FugoP <264910004+AgentGymLeader@users.noreply.github.com>
1 parent 1f99a17 commit 9975617

8 files changed

Lines changed: 259 additions & 1 deletion

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Record OpenAI Agents SDK guardrail tracing data as ``run_guardrail`` spans.

instrumentation/opentelemetry-instrumentation-genai-openai-agents/src/opentelemetry/instrumentation/genai/openai_agents/processor.py

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
from agents.tracing.span_data import (
4242
AgentSpanData,
4343
FunctionSpanData,
44+
GuardrailSpanData,
4445
)
4546

4647
from opentelemetry.semconv._incubating.attributes import (
@@ -52,6 +53,7 @@
5253
from opentelemetry.util.genai.handler import TelemetryHandler
5354
from opentelemetry.util.genai.invocation import (
5455
GenAIInvocation,
56+
GuardrailInvocation,
5557
ToolInvocation,
5658
)
5759
from opentelemetry.util.genai.types import Error
@@ -126,8 +128,17 @@ def on_span_start(self, span: Span[Any]) -> None:
126128
)
127129
self._invocations[span] = invocation
128130
return
131+
if isinstance(span_data, GuardrailSpanData):
132+
# GuardrailSpanData does not record whether an input or output guardrail ran.
133+
invocation = self._handler.guardrail(
134+
span_data.name,
135+
provider=self._provider,
136+
target_type=None,
137+
)
138+
self._invocations[span] = invocation
139+
return
129140
# Other span_data types (GenerationSpanData, ResponseSpanData,
130-
# HandoffSpanData, GuardrailSpanData, Speech/TranscriptionSpanData)
141+
# HandoffSpanData, Speech/TranscriptionSpanData)
131142
# are intentionally ignored. LLM-level spans come from the openai
132143
# instrumentation; the rest have no semconv yet.
133144

@@ -148,6 +159,10 @@ def on_span_end(self, span: Span[Any]) -> None:
148159
invocation.tool_result = (
149160
output if isinstance(output, str) else str(output)
150161
)
162+
if isinstance(invocation, GuardrailInvocation) and isinstance(
163+
span.span_data, GuardrailSpanData
164+
):
165+
invocation.triggered = span.span_data.triggered
151166
# SpanError is a mapping, not a raised exception, so it never
152167
# reaches util-genai's exception path on its own.
153168
span_error = getattr(span, "error", None)

instrumentation/opentelemetry-instrumentation-genai-openai-agents/tests/test_processor.py

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
AgentSpanData,
1313
FunctionSpanData,
1414
GenerationSpanData,
15+
GuardrailSpanData,
1516
HandoffSpanData,
1617
ResponseSpanData,
1718
)
@@ -20,6 +21,7 @@
2021
GenAITracingProcessor,
2122
)
2223
from opentelemetry.sdk.trace import TracerProvider
24+
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
2325
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
2426
InMemorySpanExporter,
2527
)
@@ -84,6 +86,36 @@ def test_agent_span_creates_invoke_local_agent() -> None:
8486
handler.invoke_local_agent.return_value.stop.assert_called_once_with()
8587

8688

89+
@pytest.mark.parametrize(
90+
("triggered", "verdict"),
91+
[(False, "allow"), (True, "deny")],
92+
)
93+
def test_guardrail_span_creates_guardrail_invocation(
94+
triggered: bool, verdict: str
95+
) -> None:
96+
span_exporter = InMemorySpanExporter()
97+
tracer_provider = TracerProvider()
98+
tracer_provider.add_span_processor(
99+
SimpleSpanProcessor(span_exporter)
100+
)
101+
handler = TelemetryHandler(tracer_provider=tracer_provider)
102+
processor = GenAITracingProcessor(handler, provider="openai")
103+
span = _Span(GuardrailSpanData(name="content_filter", triggered=False))
104+
105+
processor.on_span_start(span)
106+
span.span_data.triggered = triggered
107+
processor.on_span_end(span)
108+
109+
finished_span = span_exporter.get_finished_spans()[0]
110+
assert finished_span.name == "run_guardrail content_filter"
111+
assert finished_span.attributes == {
112+
"gen_ai.operation.name": "run_guardrail",
113+
"gen_ai.guardrail.component.name": "content_filter",
114+
"gen_ai.provider.name": "openai",
115+
"gen_ai.guardrail.verdict.type": verdict,
116+
}
117+
118+
87119
def test_function_span_creates_tool_invocation_and_sets_provider_metric() -> (
88120
None
89121
):
@@ -344,6 +376,24 @@ def test_tool_span_error_sets_error_status_and_type(
344376
assert tool_span.attributes["error.type"] == "_OTHER"
345377

346378

379+
def test_guardrail_span_error_sets_error_status_and_type(
380+
tracer_provider: TracerProvider,
381+
span_exporter: InMemorySpanExporter,
382+
) -> None:
383+
handler = TelemetryHandler(tracer_provider=tracer_provider)
384+
processor = GenAITracingProcessor(handler, provider="openai")
385+
span = _Span(GuardrailSpanData(name="content_filter", triggered=False))
386+
span.error = {"message": "Error running guardrail", "data": {}}
387+
388+
processor.on_span_start(span)
389+
processor.on_span_end(span)
390+
391+
(guardrail_span,) = span_exporter.get_finished_spans()
392+
assert guardrail_span.status.status_code is StatusCode.ERROR
393+
assert guardrail_span.attributes is not None
394+
assert guardrail_span.attributes["error.type"] == "_OTHER"
395+
396+
347397
def test_agent_span_error_sets_error_status_and_type(
348398
tracer_provider: TracerProvider,
349399
span_exporter: InMemorySpanExporter,
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add ``GuardrailInvocation`` for recording semantic-convention ``run_guardrail`` spans.
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Copyright The OpenTelemetry Authors
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
from __future__ import annotations
5+
6+
from opentelemetry._logs import Logger
7+
from opentelemetry.semconv._incubating.attributes.gen_ai_attributes import (
8+
GEN_AI_OPERATION_NAME,
9+
GEN_AI_PROVIDER_NAME,
10+
)
11+
from opentelemetry.trace import SpanKind, Tracer
12+
from opentelemetry.util.genai._invocation import Error, GenAIInvocation
13+
from opentelemetry.util.genai.completion_hook import CompletionHook
14+
from opentelemetry.util.genai.metrics import InvocationMetricsRecorder
15+
from opentelemetry.util.types import AttributeValue
16+
17+
# semconv-genai#427 (unreleased)
18+
# TODO: switch to the semconv module once #427 merges.
19+
_GEN_AI_OPERATION_NAME_RUN_GUARDRAIL = "run_guardrail"
20+
# semconv-genai#427 (unreleased)
21+
# TODO: switch to the semconv module once #427 merges.
22+
_GEN_AI_GUARDRAIL_COMPONENT_NAME = "gen_ai.guardrail.component.name"
23+
# semconv-genai#427 (unreleased)
24+
# TODO: switch to the semconv module once #427 merges.
25+
_GEN_AI_GUARDRAIL_VERDICT_TYPE = "gen_ai.guardrail.verdict.type"
26+
# semconv-genai#427 (unreleased)
27+
# TODO: switch to the semconv module once #427 merges.
28+
_GEN_AI_GUARDRAIL_TARGET_TYPE = "gen_ai.guardrail.target.type"
29+
30+
31+
class GuardrailInvocation(GenAIInvocation):
32+
"""Represents a guardrail invocation for run_guardrail span tracking.
33+
34+
Use handler.guardrail(name) rather than constructing this directly.
35+
36+
Reference: https://github.com/open-telemetry/semantic-conventions-genai/pull/427
37+
38+
Semantic convention attributes for run_guardrail spans:
39+
- gen_ai.operation.name: "run_guardrail"
40+
- gen_ai.guardrail.component.name: Name of the guardrail
41+
- gen_ai.provider.name: Name of the GenAI provider
42+
- gen_ai.guardrail.verdict.type: "deny" if triggered, otherwise "allow"
43+
- gen_ai.guardrail.target.type: "input" or "output". Required by
44+
semconv-genai#427 and must be supplied by instrumentations that know the
45+
guardrail direction.
46+
"""
47+
48+
def __init__(
49+
self,
50+
tracer: Tracer,
51+
metrics_recorder: InvocationMetricsRecorder,
52+
logger: Logger,
53+
completion_hook: CompletionHook,
54+
name: str,
55+
*,
56+
provider: str,
57+
target_type: str | None = None,
58+
) -> None:
59+
"""Use handler.guardrail(name) instead of calling this directly."""
60+
operation_name = _GEN_AI_OPERATION_NAME_RUN_GUARDRAIL
61+
super().__init__(
62+
tracer,
63+
metrics_recorder,
64+
logger,
65+
completion_hook,
66+
operation_name=operation_name,
67+
span_name=f"{operation_name} {name}" if name else operation_name,
68+
span_kind=SpanKind.INTERNAL,
69+
)
70+
self._name: str = name
71+
self._provider: str = provider
72+
self.triggered: bool = False
73+
self.target_type: str | None = target_type
74+
self._start(self._get_start_attributes())
75+
76+
def _get_start_attributes(self) -> dict[str, AttributeValue]:
77+
"""Return sampling-relevant attributes available at span creation time."""
78+
return {
79+
GEN_AI_OPERATION_NAME: self._operation_name,
80+
_GEN_AI_GUARDRAIL_COMPONENT_NAME: self._name,
81+
GEN_AI_PROVIDER_NAME: self._provider,
82+
}
83+
84+
def _get_metric_attributes(self) -> dict[str, AttributeValue]:
85+
attrs: dict[str, AttributeValue] = {
86+
GEN_AI_OPERATION_NAME: self._operation_name,
87+
GEN_AI_PROVIDER_NAME: self._provider,
88+
}
89+
attrs.update(self.metric_attributes)
90+
return attrs
91+
92+
def _apply_finish(self, error: Error | None = None) -> None:
93+
if error is not None:
94+
self._apply_error_attributes(error)
95+
optional_attrs = (
96+
(_GEN_AI_GUARDRAIL_TARGET_TYPE, self.target_type),
97+
)
98+
attributes: dict[str, AttributeValue] = {
99+
_GEN_AI_GUARDRAIL_VERDICT_TYPE: (
100+
"deny" if self.triggered else "allow"
101+
),
102+
**{key: value for key, value in optional_attrs if value is not None},
103+
}
104+
attributes.update(self.attributes)
105+
self.span.set_attributes(attributes)
106+
self._metrics_recorder.record(self)

util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
from opentelemetry.util.genai.invocation import (
5959
EmbeddingInvocation,
6060
FetchResponseInvocation,
61+
GuardrailInvocation,
6162
InferenceInvocation,
6263
RetrievalInvocation,
6364
ToolInvocation,
@@ -435,6 +436,34 @@ def tool(
435436
tool_description=tool_description,
436437
)
437438

439+
def guardrail(
440+
self,
441+
name: str,
442+
*,
443+
provider: str,
444+
target_type: str | None = None,
445+
) -> GuardrailInvocation:
446+
"""Returns a Guardrail invocation. Starts span when called.
447+
448+
Returned object can be used as a ContextManager which automatically calls `stop` or `fail`
449+
to finalize the span upon exiting. If not used as a ContextManager, the caller is
450+
responsible for calling `stop` or `fail` to finalize the span.
451+
452+
``target_type`` is required by semconv-genai#427 and must be supplied
453+
by instrumentations that know the guardrail direction.
454+
455+
Only set data attributes on the invocation object, do not modify the span or context.
456+
"""
457+
return GuardrailInvocation(
458+
self._tracer,
459+
self._metrics_recorder,
460+
self._logger,
461+
self._completion_hook,
462+
name,
463+
provider=provider,
464+
target_type=target_type,
465+
)
466+
438467
def start_invoke_local_agent(
439468
self,
440469
*,

util/opentelemetry-util-genai/src/opentelemetry/util/genai/invocation.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
from opentelemetry.util.genai._fetch_response_invocation import (
2323
FetchResponseInvocation,
2424
)
25+
from opentelemetry.util.genai._guardrail_invocation import GuardrailInvocation
2526
from opentelemetry.util.genai._inference_invocation import InferenceInvocation
2627
from opentelemetry.util.genai._invocation import (
2728
ContextToken,
@@ -39,6 +40,7 @@
3940
"Error",
4041
"FetchResponseInvocation",
4142
"GenAIInvocation",
43+
"GuardrailInvocation",
4244
"InferenceInvocation",
4345
"RetrievalInvocation",
4446
"ToolInvocation",
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Copyright The OpenTelemetry Authors
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
from opentelemetry.sdk.trace import TracerProvider
5+
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
6+
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
7+
InMemorySpanExporter,
8+
)
9+
from opentelemetry.trace import SpanKind
10+
from opentelemetry.util.genai.handler import TelemetryHandler
11+
12+
13+
def _make_span_exporter_and_handler() -> (
14+
tuple[InMemorySpanExporter, TelemetryHandler]
15+
):
16+
span_exporter = InMemorySpanExporter()
17+
tracer_provider = TracerProvider()
18+
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
19+
return span_exporter, TelemetryHandler(tracer_provider=tracer_provider)
20+
21+
22+
def test_guardrail_span_attributes_and_name() -> None:
23+
span_exporter, handler = _make_span_exporter_and_handler()
24+
invocation = handler.guardrail(
25+
"content_filter",
26+
provider="openai",
27+
)
28+
invocation.stop()
29+
30+
span = span_exporter.get_finished_spans()[0]
31+
assert span.name == "run_guardrail content_filter"
32+
assert span.kind == SpanKind.INTERNAL
33+
assert span.attributes == {
34+
"gen_ai.operation.name": "run_guardrail",
35+
"gen_ai.guardrail.component.name": "content_filter",
36+
"gen_ai.provider.name": "openai",
37+
"gen_ai.guardrail.verdict.type": "allow",
38+
}
39+
assert "gen_ai.guardrail.target.type" not in span.attributes
40+
41+
42+
def test_guardrail_target_type_and_deny_verdict() -> None:
43+
span_exporter, handler = _make_span_exporter_and_handler()
44+
invocation = handler.guardrail(
45+
"jailbreak_filter",
46+
provider="openai",
47+
target_type="input",
48+
)
49+
invocation.triggered = True
50+
invocation.stop()
51+
52+
span = span_exporter.get_finished_spans()[0]
53+
assert span.attributes["gen_ai.guardrail.verdict.type"] == "deny"
54+
assert span.attributes["gen_ai.guardrail.target.type"] == "input"

0 commit comments

Comments
 (0)