fix(tool): avoid UTF-8 misdetection at encoding sample boundaries

internal/tool/builtin/grep.go

@@ -14,6 +14,7 @@ import (
 	"regexp"
 	"strings"
 	"time"
+	"unicode/utf8"
 
 	"golang.org/x/text/transform"
 
@@ -179,7 +180,7 @@ func (g grepTool) runNative(ctx context.Context, pattern, path string, info os.F
 		// Peek the first 8 KiB to reject binaries cheaply without reading
 		// the entire file into memory. Check BOM first (UTF-16 files have
 		// 0x00 for ASCII), then NUL.
-		n, _ := io.ReadFull(f, peekBuf)
+		n, rerr := io.ReadFull(f, peekBuf)
 		peek := peekBuf[:n]
 
 		bomKind := fileenc.DetectQuick(peek)
@@ -189,11 +190,9 @@ func (g grepTool) runNative(ctx context.Context, pattern, path string, info os.F
 			}
 		}
 
-		// Detect encoding from the peek alone — sufficient for the
-		// UTF-8 vs GB18030 distinction (utf8.Valid on 8 KiB is reliable).
-		// Then stream the rest through a decoder so the 200-match cap can
-		// stop reading early instead of buffering the entire file.
-		enc, _ := fileenc.Detect(peek)
+		// An 8 KiB peek can end mid multi-byte sequence; choose a character-safe
+		// detection sample without discarding evidence that the file is GB18030.
+		enc, _ := fileenc.Detect(detectSample(peek, rerr == nil))
 
 		var src io.Reader
 		if enc == fileenc.UTF16LE || enc == fileenc.UTF16BE {
@@ -268,6 +267,31 @@ func (g grepTool) runNative(ctx context.Context, pattern, path string, info os.F
 	return formatGrep(ctx, out, truncated, to), nil
 }
 
+// detectSample returns a character-safe encoding-detection sample. A bounded
+// read can end mid UTF-8 sequence and still be valid GB18030, so prefer a
+// strict UTF-8 prefix after dropping at most one maximum-width tail. If that
+// is impossible, retain a prefix that is demonstrably GB18030 instead of
+// trimming back to an ASCII-only line and losing the encoding evidence.
+func detectSample(peek []byte, more bool) []byte {
+	if !more {
+		return peek
+	}
+	maxTrim := min(3, len(peek))
+	for trim := 0; trim <= maxTrim; trim++ {
+		candidate := peek[:len(peek)-trim]
+		if utf8.Valid(candidate) {
+			return candidate
+		}
+	}
+	for trim := 0; trim <= maxTrim; trim++ {
+		candidate := peek[:len(peek)-trim]
+		if enc, _ := fileenc.Detect(candidate); enc == fileenc.GB18030 {
+			return candidate
+		}
+	}
+	return peek
+}
+
 // runRipgrep delegates the search to ripgrep, which already emits
 // path:line:text with these flags and honors .gitignore. Output is streamed and
 // capped at grepMaxMatches so a flood of hits can't blow up memory.

internal/tool/builtin/grep_encoding_test.go

@@ -0,0 +1,80 @@
+package builtin
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	fileenc "reasonix/internal/fileutil/encoding"
+)
+
+// TestGrepUTF8ChinesePeekBoundary reproduces the misdetection where the first
+// 8 KiB sample ends mid multi-byte sequence: a truncated UTF-8 tail is nearly
+// always "valid" GB18030, so fileenc.Detect tagged the file GB18030 and grep
+// decoded it into mojibake (Chinese patterns stopped matching).
+func TestGrepUTF8ChinesePeekBoundary(t *testing.T) {
+	line := strings.Repeat("中", 100) + "\n" // 301 bytes
+	var sb strings.Builder
+	for i := 0; i < 27; i++ {
+		sb.WriteString(line) // 8127 bytes
+	}
+	// 8192-byte boundary lands mid "中" (E4 B8 …), a truncated UTF-8 tail.
+	sb.WriteString(strings.Repeat("中", 21))
+	sb.WriteString("中后续目标行统一战线内容\n")
+	content := sb.String()
+	if !utf8.ValidString(content) {
+		t.Fatal("test fixture must be valid UTF-8")
+	}
+	if utf8.ValidString(content[:8*1024]) {
+		t.Fatal("test fixture must end the 8 KiB sample mid UTF-8 sequence")
+	}
+
+	path := filepath.Join(t.TempDir(), "cn.md")
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	out := runTool(t, grepTool{}, map[string]any{"pattern": "统一战线", "path": path})
+	if !strings.Contains(out, "统一战线") {
+		t.Fatalf("grep on UTF-8 Chinese file (peek boundary cut) missed the pattern:\n%s", out)
+	}
+}
+
+// TestGrepGB18030LongLineAfterASCIIHeader ensures making a UTF-8 sample
+// character-safe does not throw away the non-ASCII evidence needed to retain
+// GB18030 detection. The 8-byte header keeps the 8 KiB sample on a complete
+// two-byte GB18030 character boundary.
+func TestGrepGB18030LongLineAfterASCIIHeader(t *testing.T) {
+	content := "header!\n" + strings.Repeat("中", 5000) + "统一战线目标\n"
+	raw := fileenc.Encode(content, fileenc.GB18030)
+	if utf8.Valid(raw) {
+		t.Fatal("test fixture must not be valid UTF-8")
+	}
+
+	path := filepath.Join(t.TempDir(), "long-gb18030.md")
+	if err := os.WriteFile(path, raw, 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	out := runTool(t, grepTool{}, map[string]any{"pattern": "统一战线", "path": path})
+	if !strings.Contains(out, "统一战线") {
+		t.Fatalf("grep misdetected GB18030 after an ASCII-only header:\n%s", out)
+	}
+}
+
+// TestGrepUTF8ChineseLongSingleLine covers the same boundary cut in a file
+// with no newline in the sample, where trimming to '\n' is not possible.
+func TestGrepUTF8ChineseLongSingleLine(t *testing.T) {
+	content := strings.Repeat("中", 2730) + "统一战线目标" + strings.Repeat("中", 7000)
+	path := filepath.Join(t.TempDir(), "long.md")
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	out := runTool(t, grepTool{}, map[string]any{"pattern": "统一战线", "path": path})
+	if !strings.Contains(out, "统一战线") {
+		t.Fatalf("grep on long single-line UTF-8 Chinese file missed the pattern:\n%s", out)
+	}
+}

internal/tool/builtin/readfile.go

@@ -190,16 +190,9 @@ func (r readFile) Execute(ctx context.Context, args json.RawMessage) (string, er
 		peekEOF = merr != nil
 	}
 
-	// Detect from a char-safe slice: when more file follows, trim to the last
-	// newline so the sample never ends mid multi-byte sequence (UTF-8 and GB18030
-	// are ASCII-transparent, so '\n' is always a clean boundary).
-	sample := head
-	if !peekEOF {
-		if i := bytes.LastIndexByte(head, '\n'); i >= 0 {
-			sample = head[:i+1]
-		}
-	}
-	enc, _ := fileenc.Detect(sample)
+	// detectSample trims the head to a char-safe boundary when more file
+	// follows: a cut mid multi-byte sequence misdetects UTF-8 as GB18030.
+	enc, _ := fileenc.Detect(detectSample(head, !peekEOF))
 
 	src := io.MultiReader(bytes.NewReader(head), f)
 	if dec := fileenc.Decoder(enc); dec != nil {

internal/tool/builtin/readfile_encoding_test.go

@@ -0,0 +1,44 @@
+package builtin
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	fileenc "reasonix/internal/fileutil/encoding"
+)
+
+// TestReadFileUTF8ChineseLongSingleLine guards the 256 KiB sampling boundary
+// cut: a sample with no newline ending mid multi-byte sequence misdetects as
+// GB18030 (the truncated UTF-8 tail is "valid" there), and read_file would
+// show the whole file as mojibake.
+func TestReadFileUTF8ChineseLongSingleLine(t *testing.T) {
+	content := strings.Repeat("中", 90000) // 270000 bytes, no newline, > sample
+	path := filepath.Join(t.TempDir(), "long.md")
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	out := runTool(t, readFile{}, map[string]any{"path": path})
+	if !strings.HasPrefix(out, "1→中") {
+		t.Fatalf("read_file garbled a UTF-8 Chinese long single line:\n%.120s", out)
+	}
+}
+
+// TestReadFileGB18030LongLineAfterASCIIHeader guards the inverse boundary:
+// making the 256 KiB sample safe for UTF-8 must retain the non-ASCII bytes
+// that distinguish a long GB18030 line from an ASCII-only prefix.
+func TestReadFileGB18030LongLineAfterASCIIHeader(t *testing.T) {
+	content := "header!\n统一战线目标" + strings.Repeat("中", 132000)
+	raw := fileenc.Encode(content, fileenc.GB18030)
+	path := filepath.Join(t.TempDir(), "long-gb18030.md")
+	if err := os.WriteFile(path, raw, 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	out := runTool(t, readFile{}, map[string]any{"path": path, "offset": 1, "limit": 1})
+	if !strings.Contains(out, "统一战线目标") {
+		t.Fatalf("read_file misdetected GB18030 after an ASCII-only header:\n%.120s", out)
+	}
+}