test(plugins/agento11y): drive the real Claude Code and Codex CLIs against a fake model

.github/workflows/ci.yml

@@ -166,6 +166,24 @@ jobs:
       - name: Test
         run: cd ${{ matrix.module }} && GOWORK=off go test ./...
 
+      # TestAgentIntegration drives the real agent CLIs against a fake model.
+      # testdata/agent-clis pins them, so a new agent release arrives as its
+      # own dependency bump and fails there instead of on an unrelated PR.
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        if: matrix.module == 'plugins/agento11y'
+        with:
+          node-version: '24'
+
+      - name: Agent CLI integration
+        if: matrix.module == 'plugins/agento11y'
+        env:
+          GOWORK: "off"
+          AGENT_CLI_TESTS: claude,codex
+        run: |
+          cd ${{ matrix.module }}/internal/entry
+          npm ci --prefix testdata/agent-clis
+          PATH="$PWD/testdata/agent-clis/node_modules/.bin:$PATH" go test . -run TestAgentIntegration -count=1 -v
+
       - name: Build agento11y CLI
         if: matrix.module == 'plugins/agento11y'
         env:

js/scripts/check-js-dependency-pinning.mjs

@@ -16,6 +16,7 @@ const publishedManifests = [
 const privateManifests = [
   "package.json",
   "plugins/agento11y/package.json",
+  "plugins/agento11y/internal/entry/testdata/agent-clis/package.json",
   "examples/experiments/typescript/package.json",
   "examples/getting-started/typescript/package.json",
   "examples/getting-started/typescript-hooks/package.json",

plugins/agento11y/internal/agents/claudecode/hook.go

@@ -6,8 +6,10 @@ package claudecode
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"log"
 	"os"
 	"strings"
@@ -172,7 +174,8 @@ func Hook(ctx context.Context, stdin io.Reader, stdout io.Writer, logger *log.Lo
 	}
 	defer func() { _ = otelProviders.Shutdown(hookCtx) }()
 
-	lines, safeOffset, rawCount := readTranscriptSettled(hookCtx, input.TranscriptPath, st.Offset, logger)
+	stop := strings.TrimSpace(input.HookEventName) == "Stop"
+	lines, safeOffset, rawCount := readTranscriptSettled(hookCtx, input.TranscriptPath, st.Offset, stop, logger)
 	if rawCount == 0 {
 		return nil
 	}
@@ -314,13 +317,19 @@ func handleUserPromptSubmit(ctx context.Context, stdout io.Writer, input *hookIn
 // a trailing tool_result, a partial assistant line, or a lone prompt awaiting
 // its first assistant reply triggers the bounded wait.
 //
+// A Stop means the turn is over, so on Stop the read also waits until the last
+// complete assistant turn ended the turn. Without CLAUDE_CODE_EAGER_FLUSH, a
+// headless `claude -p` run fires Stop before its transcript writes land: the
+// file may not exist yet, or may end at the turn that called a tool, and the
+// SessionEnd that would catch up is cancelled when -p exits.
+//
 // Returns the coalesced lines, the safe offset, and the raw line count so the
 // caller can distinguish "nothing to read" from "read but nothing complete".
-func readTranscriptSettled(ctx context.Context, path string, offset int64, logger *log.Logger) ([]transcript.Line, int64, int) {
+func readTranscriptSettled(ctx context.Context, path string, offset int64, stop bool, logger *log.Logger) ([]transcript.Line, int64, int) {
 	deadline := time.Now().Add(transcriptSettleWindow)
 	for {
 		raw, _, err := transcript.Read(path, offset)
-		if err != nil {
+		if err != nil && (!stop || !errors.Is(err, fs.ErrNotExist)) {
 			logger.Printf("read transcript: %v", err)
 			return nil, 0, 0
 		}
@@ -331,7 +340,13 @@ func readTranscriptSettled(ctx context.Context, path string, offset int64, logge
 		// flushing. An empty read (redundant Stop/SessionEnd after a prior
 		// export) has nothing to wait for, and tailNeedsSettle decides the rest.
 		settled := len(raw) == 0 || !tailNeedsSettle(raw[len(raw)-1], safeOffset)
+		if stop {
+			settled = settled && endsTurn(coalesced)
+		}
 		if settled || !time.Now().Before(deadline) {
+			if err != nil {
+				logger.Printf("read transcript: %v", err)
+			}
 			return coalesced, safeOffset, len(raw)
 		}
 
@@ -345,6 +360,16 @@ func readTranscriptSettled(ctx context.Context, path string, offset int64, logge
 	}
 }
 
+// endsTurn reports whether the last coalesced line is an assistant turn that
+// ended the turn rather than stopping to call a tool.
+func endsTurn(lines []transcript.Line) bool {
+	if len(lines) == 0 || lines[len(lines)-1].Type != "assistant" {
+		return false
+	}
+	var msg transcript.AssistantMessage
+	return json.Unmarshal(lines[len(lines)-1].Message, &msg) == nil && msg.StopReason != "tool_use"
+}
+
 // tailNeedsSettle reports whether the last raw transcript line indicates an
 // assistant turn that Claude Code is still flushing, so re-reading may recover
 // it. safeOffset is the end of the last complete assistant turn (from Coalesce).

plugins/agento11y/internal/agents/claudecode/hook_test.go

@@ -592,7 +592,7 @@ func TestReadTranscriptSettled_CapturesLateFinalTurn(t *testing.T) {
 	}()
 
 	logs := log.New(io.Discard, "", 0)
-	lines, safeOffset, rawCount := readTranscriptSettled(context.Background(), path, 0, logs)
+	lines, safeOffset, rawCount := readTranscriptSettled(context.Background(), path, 0, false, logs)
 
 	if rawCount != 3 {
 		t.Fatalf("rawCount = %d, want 3 (tool-use turn, tool_result, final turn)", rawCount)
@@ -606,6 +606,87 @@ func TestReadTranscriptSettled_CapturesLateFinalTurn(t *testing.T) {
 	}
 }
 
+// TestReadTranscriptSettled_StopWaitsForTranscriptCreatedLate reproduces a
+// headless `claude -p` run whose Stop fires before Claude Code has created the
+// transcript: the read waits for the file instead of failing on it.
+func TestReadTranscriptSettled_StopWaitsForTranscriptCreatedLate(t *testing.T) {
+	prev := transcriptSettleWindow
+	transcriptSettleWindow = 2 * time.Second
+	t.Cleanup(func() { transcriptSettleWindow = prev })
+
+	path := filepath.Join(t.TempDir(), "transcript.jsonl")
+	go func() {
+		time.Sleep(150 * time.Millisecond)
+		_ = os.WriteFile(path, []byte(buildHookAssistantJSONL("late-file", "req_a", "end_turn", "done", 5)+"\n"), 0o644)
+	}()
+
+	lines, _, rawCount := readTranscriptSettled(context.Background(), path, 0, true, log.New(io.Discard, "", 0))
+	if rawCount != 1 || len(lines) != 1 {
+		t.Fatalf("rawCount=%d len(lines)=%d, want 1/1 (the turn written after Stop)", rawCount, len(lines))
+	}
+}
+
+// TestReadTranscriptSettled_StopWaitsPastToolUseTail reproduces the same race
+// one step later: at Stop the transcript ends at the turn that called a tool,
+// complete but not the end of the turn, and the tool result and closing reply
+// land a moment later.
+func TestReadTranscriptSettled_StopWaitsPastToolUseTail(t *testing.T) {
+	prev := transcriptSettleWindow
+	transcriptSettleWindow = 2 * time.Second
+	t.Cleanup(func() { transcriptSettleWindow = prev })
+
+	path := filepath.Join(t.TempDir(), "transcript.jsonl")
+	sessionID := "tool-use-tail"
+	if err := os.WriteFile(path, []byte(buildHookAssistantJSONL(sessionID, "req_a", "tool_use", "calling tool", 10)+"\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	go func() {
+		time.Sleep(150 * time.Millisecond)
+		f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644)
+		if err != nil {
+			return
+		}
+		defer func() { _ = f.Close() }()
+		_, _ = f.WriteString(buildHookToolResultJSONL(sessionID, "tu_1", "tool ok") + "\n" +
+			buildHookAssistantJSONL(sessionID, "req_b", "end_turn", "all done", 5) + "\n")
+	}()
+
+	lines, _, rawCount := readTranscriptSettled(context.Background(), path, 0, true, log.New(io.Discard, "", 0))
+	if rawCount != 3 {
+		t.Fatalf("rawCount = %d, want 3 (tool-use turn, tool_result, final turn)", rawCount)
+	}
+	if last := lines[len(lines)-1]; last.RequestID != "req_b" {
+		t.Fatalf("last coalesced line RequestID = %q, want req_b (the closing turn)", last.RequestID)
+	}
+}
+
+// TestHook_OnlyStopWaitsForMissingTranscript pins which event waits for a
+// transcript Claude Code has not created yet. A Stop always follows a turn, so
+// it waits out the settle window. A SessionEnd can close a session that never
+// wrote a transcript, so it returns at once instead of delaying the exit.
+func TestHook_OnlyStopWaitsForMissingTranscript(t *testing.T) {
+	prev := transcriptSettleWindow
+	transcriptSettleWindow = time.Second
+	t.Cleanup(func() { transcriptSettleWindow = prev })
+	t.Setenv("SIGIL_ENDPOINT", "http://127.0.0.1:9000")
+	t.Setenv("SIGIL_AUTH_TENANT_ID", "")
+	t.Setenv("SIGIL_AUTH_TOKEN", "")
+	t.Setenv("SIGIL_OTEL_EXPORTER_OTLP_ENDPOINT", "")
+	t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")
+
+	missing := filepath.Join(t.TempDir(), "never-written.jsonl")
+	for _, tc := range []struct {
+		event    string
+		wantWait bool
+	}{{"Stop", true}, {"SessionEnd", false}} {
+		start := time.Now()
+		runHookForTest(t, hookInput{HookEventName: tc.event, SessionID: "missing-transcript", TranscriptPath: missing})
+		if waited := time.Since(start) >= transcriptSettleWindow; waited != tc.wantWait {
+			t.Errorf("%s waited out the settle window = %v, want %v", tc.event, waited, tc.wantWait)
+		}
+	}
+}
+
 // TestReadTranscriptSettled_ReturnsImmediatelyWhenTerminal confirms the common
 // case adds no latency: when the tail is already a complete assistant turn the
 // function returns on the first read without waiting out the settle window.
@@ -623,7 +704,7 @@ func TestReadTranscriptSettled_ReturnsImmediatelyWhenTerminal(t *testing.T) {
 	}
 
 	start := time.Now()
-	lines, safeOffset, rawCount := readTranscriptSettled(context.Background(), path, 0, log.New(io.Discard, "", 0))
+	lines, safeOffset, rawCount := readTranscriptSettled(context.Background(), path, 0, true, log.New(io.Discard, "", 0))
 	if elapsed := time.Since(start); elapsed > time.Second {
 		t.Fatalf("settled read took %s; expected near-immediate return", elapsed)
 	}
@@ -655,7 +736,7 @@ func TestReadTranscriptSettled_EmptyReadReturnsImmediately(t *testing.T) {
 	// Read from end-of-file: a prior export already consumed everything.
 	eof := int64(len(content))
 	start := time.Now()
-	lines, safeOffset, rawCount := readTranscriptSettled(context.Background(), path, eof, log.New(io.Discard, "", 0))
+	lines, safeOffset, rawCount := readTranscriptSettled(context.Background(), path, eof, false, log.New(io.Discard, "", 0))
 	if elapsed := time.Since(start); elapsed > time.Second {
 		t.Fatalf("empty read took %s; expected immediate return without waiting out the settle window", elapsed)
 	}
@@ -684,7 +765,7 @@ func TestReadTranscriptSettled_TrailingPromptAfterCompleteTurn(t *testing.T) {
 	}
 
 	start := time.Now()
-	lines, safeOffset, rawCount := readTranscriptSettled(context.Background(), path, 0, log.New(io.Discard, "", 0))
+	lines, safeOffset, rawCount := readTranscriptSettled(context.Background(), path, 0, false, log.New(io.Discard, "", 0))
 	if elapsed := time.Since(start); elapsed > time.Second {
 		t.Fatalf("trailing-prompt read took %s; expected immediate return (completed turn already present)", elapsed)
 	}
@@ -728,7 +809,7 @@ func TestReadTranscriptSettled_LonePromptWaitsForReply(t *testing.T) {
 		_, _ = f.WriteString(buildHookAssistantJSONL(sessionID, "req_a", "end_turn", "hello", 4) + "\n")
 	}()
 
-	lines, safeOffset, rawCount := readTranscriptSettled(context.Background(), path, 0, log.New(io.Discard, "", 0))
+	lines, safeOffset, rawCount := readTranscriptSettled(context.Background(), path, 0, false, log.New(io.Discard, "", 0))
 	if rawCount != 2 {
 		t.Fatalf("rawCount = %d, want 2 (prompt + late assistant reply)", rawCount)
 	}