Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions dotnet/src/Grafana.Agento11y/Agento11yClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ public sealed partial class Agento11yClient : IAsyncDisposable
internal const string SpanAttrToolCallId = "gen_ai.tool.call.id";
internal const string SpanAttrToolType = "gen_ai.tool.type";
internal const string SpanAttrToolDescription = "gen_ai.tool.description";
internal const string SpanAttrSkillName = "agento11y.skill.name";
internal const string SpanAttrToolCallArguments = "gen_ai.tool.call.arguments";
internal const string SpanAttrToolCallResult = "gen_ai.tool.call.result";
internal const string SpanAttrTagPrefix = "agento11y.tag.";
Expand Down Expand Up @@ -1441,6 +1442,8 @@ internal static void ApplyToolSpanAttributes(Activity activity, ToolExecutionSta
activity.SetTag(SpanAttrToolDescription, tool.ToolDescription);
}

ApplySkillNameSpanAttribute(activity, tool.SkillName);

if (!string.IsNullOrWhiteSpace(tool.ConversationId))
{
activity.SetTag(SpanAttrConversationId, tool.ConversationId);
Expand Down Expand Up @@ -1470,6 +1473,15 @@ internal static void ApplyToolSpanAttributes(Activity activity, ToolExecutionSta
}
}

private static void ApplySkillNameSpanAttribute(Activity activity, string? skillName)
{
var projected = skillName?.Trim();
if (!string.IsNullOrEmpty(projected))
{
activity.SetTag(SpanAttrSkillName, projected);
}
}

internal static string OperationName(Generation generation)
{
if (!string.IsNullOrWhiteSpace(generation.OperationName))
Expand Down
1 change: 1 addition & 0 deletions dotnet/src/Grafana.Agento11y/Models.cs
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,7 @@ public sealed class ToolExecutionStart
public string ToolCallId { get; set; } = string.Empty;
public string ToolType { get; set; } = string.Empty;
public string ToolDescription { get; set; } = string.Empty;
public string? SkillName { get; set; }
public string ConversationId { get; set; } = string.Empty;
public string ConversationTitle { get; set; } = string.Empty;
public string AgentName { get; set; } = string.Empty;
Expand Down
69 changes: 69 additions & 0 deletions dotnet/tests/Grafana.Agento11y.Tests/ConformanceTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,7 @@ public async Task ToolExecutionSemantics()
ToolName = "weather",
ToolCallId = "call-weather-1",
ToolType = "function",
SkillName = " code-review ",
RequestProvider = "openai",
RequestModel = "gpt-5",
ContentCapture = ContentCaptureMode.Full,
Expand All @@ -486,6 +487,7 @@ public async Task ToolExecutionSemantics()
Assert.Equal("weather", span.GetTagItem("gen_ai.tool.name"));
Assert.Equal("call-weather-1", span.GetTagItem("gen_ai.tool.call.id"));
Assert.Equal("function", span.GetTagItem("gen_ai.tool.type"));
Assert.Equal("code-review", span.GetTagItem("agento11y.skill.name")?.ToString());
Assert.Contains("Paris", span.GetTagItem("gen_ai.tool.call.arguments")?.ToString());
Assert.Contains("sunny", span.GetTagItem("gen_ai.tool.call.result")?.ToString());
Assert.Equal("openai", span.GetTagItem("gen_ai.provider.name")?.ToString());
Expand All @@ -495,6 +497,73 @@ public async Task ToolExecutionSemantics()
Assert.Equal("v-context", span.GetTagItem("gen_ai.agent.version")?.ToString());
Assert.Contains("gen_ai.client.operation.duration", env.MetricNames);
Assert.DoesNotContain("gen_ai.client.time_to_first_token", env.MetricNames);
var durationMeasurements = env.MetricMeasurements
.Where(measurement =>
string.Equals(measurement.Name, "gen_ai.client.operation.duration", StringComparison.Ordinal)
&& measurement.Tags.TryGetValue("gen_ai.operation.name", out var operation)
&& string.Equals(operation?.ToString(), "execute_tool", StringComparison.Ordinal))
.ToList();
Assert.NotEmpty(durationMeasurements);
Assert.All(durationMeasurements, measurement =>
Assert.False(measurement.Tags.ContainsKey("agento11y.skill.name")));
}

[Theory]
[InlineData(null, null, false)]
[InlineData(" \t\r\n ", null, false)]
[InlineData(" failure-recovery ", "failure-recovery", true)]
public async Task ToolExecutionSkillMarkerSemantics(string? skillName, string? expected, bool failed)
{
await using var env = new ConformanceEnv();
var start = new ToolExecutionStart
{
ToolName = "skill-marker-tool",
};
if (skillName != null)
{
start.SkillName = skillName;
}

var recorder = env.Client.StartToolExecution(start);
if (failed)
{
recorder.SetExecutionError(new InvalidOperationException("tool failed"));
}
recorder.End();

await env.ShutdownAsync();

var span = env.OperationSpan("execute_tool");
Assert.Equal("execute_tool", span.GetTagItem("gen_ai.operation.name"));
if (expected == null)
{
Assert.DoesNotContain("agento11y.skill.name", span.TagObjects.Select(tag => tag.Key));
}
else
{
Assert.Equal(expected, span.GetTagItem("agento11y.skill.name")?.ToString());
}
Assert.Equal(failed ? ActivityStatusCode.Error : ActivityStatusCode.Ok, span.Status);
}

[Fact]
public async Task ToolExecutionStartDeepCloneIsolatesSkillName()
{
await using var env = new ConformanceEnv();
var start = new ToolExecutionStart
{
ToolName = "isolated-tool",
SkillName = "original-skill",
};

var recorder = env.Client.StartToolExecution(start);
start.SkillName = "mutated-after-start";
recorder.End();

await env.ShutdownAsync();

var span = env.OperationSpan("execute_tool");
Assert.Equal("original-skill", span.GetTagItem("agento11y.skill.name")?.ToString());
}

[Fact]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1381,6 +1381,7 @@ public async Task StrippedModes_ToolSpan_OmitsContentAttrs(ContentCaptureMode mo
{
ToolName = "weather",
ToolCallId = "call_1",
SkillName = " weather-lookup ",
ConversationTitle = "Sensitive tool title",
ToolDescription = "Get weather: free-form provider-supplied text",
IncludeContent = true,
Expand All @@ -1402,6 +1403,7 @@ public async Task StrippedModes_ToolSpan_OmitsContentAttrs(ContentCaptureMode mo
Assert.Null(span.GetTagItem("gen_ai.tool.description"));
// Identity attributes still emitted.
Assert.Equal("weather", span.GetTagItem("gen_ai.tool.name")?.ToString());
Assert.Equal("weather-lookup", span.GetTagItem("agento11y.skill.name")?.ToString());
}

// Tools have no proto export; under both stripped modes the raw provider
Expand Down
9 changes: 9 additions & 0 deletions go/agento11y/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,7 @@ const (
spanAttrToolCallID = "gen_ai.tool.call.id"
spanAttrToolType = "gen_ai.tool.type"
spanAttrToolDescription = contentcapture.ToolDescriptionAttributeKey
spanAttrSkillName = "agento11y.skill.name"
spanAttrToolCallArguments = contentcapture.ToolCallArgumentsAttributeKey
spanAttrToolCallResult = contentcapture.ToolCallResultAttributeKey
spanAttrTagPrefix = "agento11y.tag."
Expand Down Expand Up @@ -2476,6 +2477,7 @@ func toolSpanAttributes(start ToolExecutionStart) []attribute.KeyValue {
attribute.String(spanAttrToolName, start.ToolName),
attribute.String(sdkMetadataKeyName, sdkName),
}
attrs = appendSkillNameAttribute(attrs, start.SkillName)

if callID := strings.TrimSpace(start.ToolCallID); callID != "" {
attrs = append(attrs, attribute.String(spanAttrToolCallID, callID))
Expand Down Expand Up @@ -2508,6 +2510,13 @@ func toolSpanAttributes(start ToolExecutionStart) []attribute.KeyValue {
return attrs
}

func appendSkillNameAttribute(attrs []attribute.KeyValue, skillName string) []attribute.KeyValue {
if skillName = strings.TrimSpace(skillName); skillName != "" {
attrs = append(attrs, attribute.String(spanAttrSkillName, skillName))
}
return attrs
}

func serializeToolContent(value any) (string, error) {
if value == nil {
return "", nil
Expand Down
48 changes: 47 additions & 1 deletion go/agento11y/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1176,6 +1176,45 @@ func TestStartToolExecutionSetsExecuteToolAttributes(t *testing.T) {
}
}

const testSkillNameAttributeKey = "agento11y.skill.name"

func TestToolExecutionSkillNameAttribute(t *testing.T) {
tests := []struct {
name string
skillName string
want string
}{
{name: "populated is trimmed", skillName: " code-review ", want: "code-review"},
{name: "omitted"},
{name: "whitespace omitted", skillName: " \t\n "},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
client, recorder, _ := newTestClient(t, Config{})
_, toolRecorder := client.StartToolExecution(context.Background(), ToolExecutionStart{
ToolName: "weather",
SkillName: tc.skillName,
})
toolRecorder.End()
if err := toolRecorder.Err(); err != nil {
t.Fatalf("end tool execution: %v", err)
}

attrs := spanAttributeMap(onlyToolSpan(t, recorder.Ended()))
if tc.want == "" {
if _, ok := attrs[testSkillNameAttributeKey]; ok {
t.Fatalf("did not expect %s for skill name %q", testSkillNameAttributeKey, tc.skillName)
}
return
}
if got := attrs[testSkillNameAttributeKey].AsString(); got != tc.want {
t.Fatalf("expected %s=%q, got %q", testSkillNameAttributeKey, tc.want, got)
}
})
}
}

func TestToolExecutionRecorderContentCapture(t *testing.T) {
// Backward compat: Config{} with IncludeContent controls tool content.
client, recorder, _ := newTestClient(t, Config{})
Expand Down Expand Up @@ -1233,7 +1272,8 @@ func TestToolExecutionRecorderContentCapture(t *testing.T) {
func TestToolExecutionRecorderErrorSetsStatusAndType(t *testing.T) {
client, recorder, _ := newTestClient(t, Config{})
_, toolRecorder := client.StartToolExecution(context.Background(), ToolExecutionStart{
ToolName: "weather",
ToolName: "weather",
SkillName: " code-review ",
})

toolRecorder.SetExecError(errors.New("tool failed"))
Expand All @@ -1251,6 +1291,12 @@ func TestToolExecutionRecorderErrorSetsStatusAndType(t *testing.T) {
if attrs[spanAttrErrorType].AsString() != "tool_execution_error" {
t.Fatalf("expected error.type=tool_execution_error")
}
if attrs[spanAttrErrorCategory].AsString() != "sdk_error" {
t.Fatalf("expected error.category=sdk_error")
}
if attrs[testSkillNameAttributeKey].AsString() != "code-review" {
t.Fatalf("expected %s=code-review on failed tool", testSkillNameAttributeKey)
}
}

func TestToolExecutionRecorderEndIsIdempotent(t *testing.T) {
Expand Down
9 changes: 9 additions & 0 deletions go/agento11y/conformance_helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ const (
spanAttrToolCallID = "gen_ai.tool.call.id"
spanAttrToolType = "gen_ai.tool.type"
spanAttrToolDescription = "gen_ai.tool.description"
spanAttrSkillName = "agento11y.skill.name"
spanAttrToolCallArguments = "gen_ai.tool.call.arguments"
spanAttrToolCallResult = "gen_ai.tool.call.result"
spanAttrResponseID = "gen_ai.response.id"
Expand Down Expand Up @@ -655,6 +656,14 @@ func histogramPointMatches(attrs attribute.Set, want map[string]string) bool {
return true
}

func requireMetricAttrAbsent(t *testing.T, attrs attribute.Set, key string) {
t.Helper()

if value, ok := (&attrs).Value(attribute.Key(key)); ok {
t.Fatalf("did not expect metric attribute %q to be present (value %q)", key, value.Emit())
}
}

func requireProtoMetadata(t *testing.T, generation *agento11yv1.Generation, key, want string) {
t.Helper()

Expand Down
47 changes: 46 additions & 1 deletion go/agento11y/conformance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1015,6 +1015,7 @@ func TestConformance_ToolExecution(t *testing.T) {
ToolCallID: "call-weather",
ToolType: "function",
ToolDescription: "Get weather",
SkillName: " code-review ",
RequestModel: conformanceModel.Name,
RequestProvider: conformanceModel.Provider,
IncludeContent: true,
Expand Down Expand Up @@ -1042,13 +1043,14 @@ func TestConformance_ToolExecution(t *testing.T) {

metrics := env.CollectMetrics(t)
duration := findHistogram[float64](t, metrics, metricOperationDuration)
requireHistogramPointWithAttrs(t, duration, map[string]string{
durationPoint := requireHistogramPointWithAttrs(t, duration, map[string]string{
spanAttrOperationName: conformanceToolOperation,
spanAttrProviderName: conformanceModel.Provider,
spanAttrRequestModel: conformanceModel.Name,
spanAttrToolName: "weather",
spanAttrAgentName: "agent-tools",
})
requireMetricAttrAbsent(t, durationPoint.Attributes, spanAttrSkillName)

env.Shutdown(t)

Expand All @@ -1063,6 +1065,7 @@ func TestConformance_ToolExecution(t *testing.T) {
requireSpanAttr(t, attrs, spanAttrToolCallID, "call-weather")
requireSpanAttr(t, attrs, spanAttrToolType, "function")
requireSpanAttr(t, attrs, spanAttrToolDescription, "Get weather")
requireSpanAttr(t, attrs, spanAttrSkillName, "code-review")
requireSpanAttr(t, attrs, spanAttrConversationID, "conv-tool")
requireSpanAttr(t, attrs, spanAttrConversationTitle, "Weather lookup")
requireSpanAttr(t, attrs, spanAttrAgentName, "agent-tools")
Expand All @@ -1074,6 +1077,48 @@ func TestConformance_ToolExecution(t *testing.T) {
requireSpanAttrPresent(t, attrs, spanAttrToolCallResult)
}

func TestConformance_ToolSkillMarkerSurvivesStrippedModes(t *testing.T) {
tests := []struct {
name string
mode agento11y.ContentCaptureMode
}{
{name: "metadata_only", mode: agento11y.ContentCaptureModeMetadataOnly},
{name: "full_with_metadata_spans", mode: agento11y.ContentCaptureModeFullWithMetadataSpans},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
env := newConformanceEnv(t, withConformanceConfig(func(cfg *agento11y.Config) {
cfg.ContentCapture = tc.mode
}))

_, recorder := env.Client.StartToolExecution(context.Background(), agento11y.ToolExecutionStart{
ToolName: "weather",
ToolDescription: "Get sensitive weather",
ConversationTitle: "Sensitive tool conversation",
SkillName: " code-review ",
IncludeContent: true,
})
recorder.SetResult(agento11y.ToolExecutionEnd{
Arguments: map[string]any{"city": "Paris"},
Result: map[string]any{"temp_c": 18},
})
recorder.End()
if err := recorder.Err(); err != nil {
t.Fatalf("record tool execution: %v", err)
}

span := findSpan(t, env.Spans.Ended(), conformanceToolOperation)
attrs := spanAttrs(span)
requireSpanAttr(t, attrs, spanAttrSkillName, "code-review")
requireSpanAttrAbsent(t, attrs, spanAttrToolDescription)
requireSpanAttrAbsent(t, attrs, spanAttrConversationTitle)
requireSpanAttrAbsent(t, attrs, spanAttrToolCallArguments)
requireSpanAttrAbsent(t, attrs, spanAttrToolCallResult)
})
}
}

func TestConformance_Embedding(t *testing.T) {
env := newConformanceEnv(t)

Expand Down
1 change: 1 addition & 0 deletions go/agento11y/contentcapture/contentcapture_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,7 @@ func TestIsTraceContentAttribute(t *testing.T) {
{key: "gen_ai.retrieval.documents", want: true},
{key: "agento11y.generation.raw_artifacts", want: true},
{key: "gen_ai.tool.name", want: false},
{key: "agento11y.skill.name", want: false},
{key: "gen_ai.usage.input_tokens", want: false},
{key: contentcapture.ErrorCategoryAttributeKey, want: false},
{key: contentcapture.MetadataKeyCallError, want: false},
Expand Down
1 change: 1 addition & 0 deletions go/agento11y/tool.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import "time"
// ToolExecutionStart seeds a tool execution span before the tool call runs.
type ToolExecutionStart struct {
ToolName string
SkillName string
ToolCallID string
ToolType string
ToolDescription string
Expand Down
8 changes: 8 additions & 0 deletions go/otelgenai/genai.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ const (
genAIResponseStatusKey attribute.Key = "gen_ai.response.status"
)

// agento11ySkillNameKey is a vendor attribute and is intentionally separate
// from the pinned semantic-convention registry.
const agento11ySkillNameKey attribute.Key = "agento11y.skill.name"

// EndHook observes a finished invocation before its span closes. It returns
// extra span attributes, and may transform the invocation itself, which is how
// content redaction plugs in.
Expand Down Expand Up @@ -434,6 +438,10 @@ func (h *Handler) End(ctx context.Context, inv *Invocation) {
if span != nil {
span.SetName(inv.spanName())
attrs := h.requestAttributes(inv)
// End hooks can change the operation, but attributes set by Start cannot be removed.
if skillName := strings.TrimSpace(inv.SkillName); inv.operation() == OperationExecuteTool && skillName != "" {
attrs = append(attrs, agento11ySkillNameKey.String(skillName))
}
attrs = append(attrs, h.responseAttributes(inv)...)
attrs = append(attrs, h.contentAttributes(inv, capture)...)
attrs = append(attrs, inv.Attributes...)
Expand Down
Loading
Loading