diff --git a/src-tauri/src/agent/tools.rs b/src-tauri/src/agent/tools.rs index 67be466d5..3870961ec 100644 --- a/src-tauri/src/agent/tools.rs +++ b/src-tauri/src/agent/tools.rs @@ -1941,7 +1941,10 @@ fn provider_payload_error(provider: &str, value: &Value) -> Option { .and_then(Value::as_str) .filter(|message| !message.trim().is_empty()) .unwrap_or("unknown API error"); - return Some(format!("{provider} failed (code {}): {message}", code.unwrap_or(0))); + return Some(format!( + "{provider} failed (code {}): {message}", + code.unwrap_or(0) + )); } } if provider == "Brave Search" && value.get("web").is_none() { diff --git a/src/lib/index-generator.test.ts b/src/lib/index-generator.test.ts new file mode 100644 index 000000000..72ed42e9d --- /dev/null +++ b/src/lib/index-generator.test.ts @@ -0,0 +1,151 @@ +import { describe, it, expect } from "vitest" +import { + generateIndexMd, + extractIndexDescription, + buildLogEntry, + type IndexInputPage, +} from "./index-generator" + +function page(relativePath: string, frontmatter: Record, body: string): IndexInputPage { + const fm = Object.entries(frontmatter) + .map(([k, v]) => `${k}: ${v}`) + .join("\n") + return { relativePath, content: `---\n${fm}\n---\n\n${body}` } +} + +describe("generateIndexMd", () => { + it("groups pages by type in a stable display order", () => { + const pages: IndexInputPage[] = [ + page("wiki/concepts/cot.md", { type: "concept", title: "Chain of Thought" }, "# Chain of Thought\n\nA prompting technique."), + page("wiki/entities/openai.md", { type: "entity", title: "OpenAI" }, "# OpenAI\n\nAn AI research company."), + page("wiki/sources/wei-2022.md", { type: "source", title: "Wei 2022" }, "# Wei 2022\n\nA paper on CoT."), + ] + const out = generateIndexMd(pages, { date: "2026-06-27" }) + // Entities before Concepts before Sources + const entIdx = out.indexOf("## Entities") + const conIdx = out.indexOf("## Concepts") + const srcIdx = out.indexOf("## Sources") + expect(entIdx).toBeGreaterThan(-1) + expect(entIdx).toBeLessThan(conIdx) + expect(conIdx).toBeLessThan(srcIdx) + }) + + it("emits `- [[slug]] — description` entries using the first body sentence", () => { + const pages = [ + page("wiki/entities/openai.md", { type: "entity", title: "OpenAI" }, "# OpenAI\n\nAn AI research company. Founded in 2015."), + ] + const out = generateIndexMd(pages, { date: "2026-06-27" }) + expect(out).toContain("- [[openai]] — An AI research company.") + }) + + it("includes a deterministic overview-typed frontmatter header", () => { + const out = generateIndexMd([], { date: "2026-06-27" }) + expect(out).toMatch(/^---\ntype: overview\ntitle: Wiki Index\n/) + expect(out).toContain("created: 2026-06-27") + expect(out).toContain("updated: 2026-06-27") + expect(out).toContain("# Wiki Index") + }) + + it("carries an existing created date forward, refreshing only updated", () => { + const out = generateIndexMd([], { date: "2026-06-27", created: "2026-01-05" }) + expect(out).toContain("created: 2026-01-05") + expect(out).toContain("updated: 2026-06-27") + }) + + it("falls back to the ingest date when created is absent or blank", () => { + expect(generateIndexMd([], { date: "2026-06-27", created: " " })).toContain( + "created: 2026-06-27", + ) + }) + + it("excludes index.md, log.md, and overview.md themselves", () => { + const pages = [ + page("wiki/index.md", { type: "overview", title: "Wiki Index" }, "# Wiki Index"), + page("wiki/log.md", { type: "overview", title: "Log" }, "# Log"), + page("wiki/overview.md", { type: "overview", title: "Overview" }, "# Overview\n\nSummary."), + page("wiki/entities/foo.md", { type: "entity", title: "Foo" }, "# Foo\n\nA thing."), + ] + const out = generateIndexMd(pages, { date: "2026-06-27" }) + expect(out).toContain("- [[foo]]") + expect(out).not.toContain("[[index]]") + expect(out).not.toContain("[[log]]") + expect(out).not.toContain("[[overview]]") + }) + + it("infers type from path when frontmatter type is missing", () => { + const pages: IndexInputPage[] = [ + { relativePath: "wiki/entities/bar.md", content: "---\ntitle: Bar\n---\n\n# Bar\n\nAn entity." }, + ] + const out = generateIndexMd(pages, { date: "2026-06-27" }) + expect(out).toContain("## Entities") + expect(out).toContain("- [[bar]] — An entity.") + }) + + it("sorts entries within a type alphabetically by slug", () => { + const pages = [ + page("wiki/entities/zebra.md", { type: "entity", title: "Zebra" }, "# Zebra\n\nLast."), + page("wiki/entities/alpha.md", { type: "entity", title: "Alpha" }, "# Alpha\n\nFirst."), + ] + const out = generateIndexMd(pages, { date: "2026-06-27" }) + expect(out.indexOf("[[alpha]]")).toBeLessThan(out.indexOf("[[zebra]]")) + }) + + it("falls back to a placeholder when there are no pages", () => { + const out = generateIndexMd([], { date: "2026-06-27" }) + expect(out).toContain("_No pages yet._") + }) + + it("places custom/unknown types after known types, alphabetically", () => { + const pages = [ + page("wiki/entities/foo.md", { type: "entity", title: "Foo" }, "# Foo\n\nE."), + page("wiki/people/jane.md", { type: "people", title: "Jane" }, "# Jane\n\nP."), + ] + const out = generateIndexMd(pages, { date: "2026-06-27" }) + expect(out.indexOf("## Entities")).toBeLessThan(out.indexOf("## People")) + }) +}) + +describe("extractIndexDescription", () => { + it("takes the first sentence of the body", () => { + expect(extractIndexDescription("# Title\n\nFirst sentence. Second sentence.", "Title", "slug")).toBe("First sentence.") + }) + + it("skips headings, comments, blockquotes, and lists", () => { + const body = "# Heading\n\n\n\n> placeholder\n\n- a list item\n\nReal prose here." + expect(extractIndexDescription(body, "Title", "slug")).toBe("Real prose here.") + }) + + it("flattens wikilinks and markdown to plain text", () => { + const body = "# T\n\nSee [[other-page|the other page]] and **bold** `code`." + expect(extractIndexDescription(body, "T", "slug")).toBe("See the other page and bold code.") + }) + + it("falls back to title when the body has no prose", () => { + expect(extractIndexDescription("# Only Heading", "My Title", "slug")).toBe("My Title") + }) + + it("falls back to a de-slugified slug when title is empty too", () => { + expect(extractIndexDescription("", "", "my-page-slug")).toBe("my page slug") + }) + + it("truncates very long sentences", () => { + const long = "x".repeat(300) + "." + const out = extractIndexDescription(`# T\n\n${long}`, "T", "slug") + expect(out.length).toBeLessThanOrEqual(200) + expect(out.endsWith("…")).toBe(true) + }) +}) + +describe("buildLogEntry", () => { + it("formats a log line with date and title", () => { + expect(buildLogEntry("2026-06-27", "My Source")).toBe("## [2026-06-27] ingest | My Source") + }) + + it("collapses whitespace in the title", () => { + expect(buildLogEntry("2026-06-27", " My Source ")).toBe("## [2026-06-27] ingest | My Source") + }) + + it("uses a placeholder for an empty title", () => { + expect(buildLogEntry("2026-06-27", "")).toBe("## [2026-06-27] ingest | (untitled source)") + }) +}) diff --git a/src/lib/index-generator.ts b/src/lib/index-generator.ts new file mode 100644 index 000000000..265e36cb0 --- /dev/null +++ b/src/lib/index-generator.ts @@ -0,0 +1,256 @@ +/** + * Deterministic wiki/index.md and wiki/log.md generation. + * + * Why this exists: ingest used to ask the LLM to emit an updated + * index.md (full file) and a log.md entry on every wave. Because the + * Claude/Codex CLI transports are stateless and the HTTP providers see + * only a snapshot of the current index, the model routinely dropped + * existing index entries, re-referenced deleted pages, or echoed the + * entire log (which the append-only writer then duplicated). The index + * is fully derivable from each page's frontmatter, so we generate it + * from disk instead — the LLM never writes it. The log is append-only + * and its single new line is composed here from the ingest date + the + * source title rather than trusted to the model. + * + * These functions are pure (string in, string out); the caller in + * ingest.ts is responsible for reading every page off disk and writing + * the result back inside the project lock. + */ + +import { parseFrontmatter } from "@/lib/frontmatter" +import { inferWikiTypeFromPath, wikiTypeLabel } from "@/lib/wiki-page-types" +import { normalizePath, getFileName } from "@/lib/path-utils" + +export interface IndexInputPage { + /** Wiki-root-relative path, e.g. "wiki/entities/foo.md". */ + relativePath: string + content: string +} + +/** Display order for known page types; unknown/custom types follow, + * sorted alphabetically. Mirrors WIKI_TYPE_DIRS ordering in + * wiki-page-types.ts so the rendered index is stable run-to-run. */ +const TYPE_DISPLAY_ORDER = [ + "entity", + "concept", + "source", + "query", + "comparison", + "synthesis", + "finding", + "thesis", + "methodology", +] + +// Aggregate / structural files that are never themselves index entries. +// schema.md and purpose.md are project-scaffolding pages (some projects +// place them under wiki/); excluding them by basename keeps stray +// `[[schema]]` / `[[purpose]]` entries out of the index. +const EXCLUDED_BASENAMES = new Set([ + "index.md", + "log.md", + "overview.md", + "schema.md", + "purpose.md", +]) + +interface IndexEntry { + slug: string + type: string + description: string +} + +/** + * Build the complete wiki/index.md content deterministically from the + * frontmatter + first body sentence of every content page. + * + * Pages are grouped by type (entity, concept, …) in a stable order; + * each entry is `- [[slug]] — description`. The slug is the page's + * filename without extension so it resolves as an Obsidian wikilink. + * + * `opts.created` carries the existing index's creation date forward: + * the file is rewritten wholesale every wave, so without it `created` + * would be stamped to the current date each run and never mean + * anything but "last ingest". Omit it for a brand-new index, where + * `opts.date` is the creation date. + */ +export function generateIndexMd( + pages: IndexInputPage[], + opts: { date: string; created?: string }, +): string { + const entries: IndexEntry[] = [] + for (const page of pages) { + const rel = normalizePath(page.relativePath) + const baseName = getFileName(rel).toLowerCase() + if (EXCLUDED_BASENAMES.has(baseName)) continue + if (!rel.endsWith(".md")) continue + // Source summary pages live under wiki/sources/ and ARE indexed + // (type "source"); only the three aggregate files above are skipped. + + const { frontmatter, body } = parseFrontmatter(page.content) + const slug = getFileName(rel).replace(/\.md$/i, "") + if (!slug) continue + + const fmType = + typeof frontmatter?.type === "string" ? frontmatter.type.trim() : "" + const type = fmType || inferWikiTypeFromPath(rel) || "other" + // overview-typed stray pages (other than overview.md, already + // excluded) shouldn't appear as index entries either. + if (type === "overview") continue + + const title = + typeof frontmatter?.title === "string" ? frontmatter.title.trim() : "" + const description = extractIndexDescription(body, title, slug) + entries.push({ slug, type, description }) + } + + // Group by type. + const byType = new Map() + for (const entry of entries) { + const bucket = byType.get(entry.type) ?? [] + bucket.push(entry) + byType.set(entry.type, bucket) + } + + const orderedTypes = [...byType.keys()].sort(compareTypes) + + const sections: string[] = [] + for (const type of orderedTypes) { + const bucket = byType.get(type) + if (!bucket || bucket.length === 0) continue + bucket.sort((a, b) => a.slug.localeCompare(b.slug)) + const heading = pluralizeTypeLabel(type) + const lines = bucket.map((entry) => + entry.description + ? `- [[${entry.slug}]] — ${entry.description}` + : `- [[${entry.slug}]]`, + ) + sections.push(`## ${heading}\n\n${lines.join("\n")}`) + } + + const frontmatter = [ + "---", + "type: overview", + "title: Wiki Index", + "tags: []", + "related: []", + `created: ${opts.created?.trim() || opts.date}`, + `updated: ${opts.date}`, + "---", + ].join("\n") + + const body = + sections.length > 0 + ? sections.join("\n\n") + : "_No pages yet._" + + return `${frontmatter}\n\n# Wiki Index\n\n${body}\n` +} + +/** Order known types by TYPE_DISPLAY_ORDER, then unknown types A→Z. */ +function compareTypes(a: string, b: string): number { + const ia = TYPE_DISPLAY_ORDER.indexOf(a) + const ib = TYPE_DISPLAY_ORDER.indexOf(b) + if (ia !== -1 && ib !== -1) return ia - ib + if (ia !== -1) return -1 + if (ib !== -1) return 1 + return a.localeCompare(b) +} + +/** Section heading for a type: known labels pluralized, custom types + * title-cased via wikiTypeLabel. */ +function pluralizeTypeLabel(type: string): string { + switch (type) { + case "entity": + return "Entities" + case "concept": + return "Concepts" + case "source": + return "Sources" + case "query": + return "Queries" + case "comparison": + return "Comparisons" + case "synthesis": + return "Synthesis" + case "finding": + return "Findings" + case "thesis": + return "Theses" + case "methodology": + return "Methodologies" + case "other": + return "Other" + default: + return wikiTypeLabel(type) + } +} + +const MAX_DESCRIPTION_CHARS = 200 + +/** + * Derive a one-line description for an index entry: the first sentence + * of the page body, falling back to the title, then the slug. Markdown + * decoration (wikilinks, links, emphasis, inline code) is flattened to + * plain text so the index line stays readable. + */ +export function extractIndexDescription( + body: string, + title: string, + slug: string, +): string { + const firstSentence = firstBodySentence(body) + const chosen = firstSentence || title || slug.replace(/[-_]+/g, " ") + return truncate(flattenInlineMarkdown(chosen), MAX_DESCRIPTION_CHARS) +} + +/** Pull the first prose sentence from a markdown body, skipping the H1 + * heading, blank lines, HTML comments, and block markup. */ +function firstBodySentence(body: string): string { + const lines = body.split(/\r?\n/) + for (const raw of lines) { + const line = raw.trim() + if (!line) continue + if (line.startsWith("#")) continue // headings (incl. the H1) + if (line.startsWith("