feat: generate index.md and log.md deterministically, not via LLM

src/lib/index-generator.test.ts

@@ -0,0 +1,139 @@
+import { describe, it, expect } from "vitest"
+import {
+  generateIndexMd,
+  extractIndexDescription,
+  buildLogEntry,
+  type IndexInputPage,
+} from "./index-generator"
+
+function page(relativePath: string, frontmatter: Record<string, string>, body: string): IndexInputPage {
+  const fm = Object.entries(frontmatter)
+    .map(([k, v]) => `${k}: ${v}`)
+    .join("\n")
+  return { relativePath, content: `---\n${fm}\n---\n\n${body}` }
+}
+
+describe("generateIndexMd", () => {
+  it("groups pages by type in a stable display order", () => {
+    const pages: IndexInputPage[] = [
+      page("wiki/concepts/cot.md", { type: "concept", title: "Chain of Thought" }, "# Chain of Thought\n\nA prompting technique."),
+      page("wiki/entities/openai.md", { type: "entity", title: "OpenAI" }, "# OpenAI\n\nAn AI research company."),
+      page("wiki/sources/wei-2022.md", { type: "source", title: "Wei 2022" }, "# Wei 2022\n\nA paper on CoT."),
+    ]
+    const out = generateIndexMd(pages, { date: "2026-06-27" })
+    // Entities before Concepts before Sources
+    const entIdx = out.indexOf("## Entities")
+    const conIdx = out.indexOf("## Concepts")
+    const srcIdx = out.indexOf("## Sources")
+    expect(entIdx).toBeGreaterThan(-1)
+    expect(entIdx).toBeLessThan(conIdx)
+    expect(conIdx).toBeLessThan(srcIdx)
+  })
+
+  it("emits `- [[slug]] — description` entries using the first body sentence", () => {
+    const pages = [
+      page("wiki/entities/openai.md", { type: "entity", title: "OpenAI" }, "# OpenAI\n\nAn AI research company. Founded in 2015."),
+    ]
+    const out = generateIndexMd(pages, { date: "2026-06-27" })
+    expect(out).toContain("- [[openai]] — An AI research company.")
+  })
+
+  it("includes a deterministic overview-typed frontmatter header", () => {
+    const out = generateIndexMd([], { date: "2026-06-27" })
+    expect(out).toMatch(/^---\ntype: overview\ntitle: Wiki Index\n/)
+    expect(out).toContain("created: 2026-06-27")
+    expect(out).toContain("updated: 2026-06-27")
+    expect(out).toContain("# Wiki Index")
+  })
+
+  it("excludes index.md, log.md, and overview.md themselves", () => {
+    const pages = [
+      page("wiki/index.md", { type: "overview", title: "Wiki Index" }, "# Wiki Index"),
+      page("wiki/log.md", { type: "overview", title: "Log" }, "# Log"),
+      page("wiki/overview.md", { type: "overview", title: "Overview" }, "# Overview\n\nSummary."),
+      page("wiki/entities/foo.md", { type: "entity", title: "Foo" }, "# Foo\n\nA thing."),
+    ]
+    const out = generateIndexMd(pages, { date: "2026-06-27" })
+    expect(out).toContain("- [[foo]]")
+    expect(out).not.toContain("[[index]]")
+    expect(out).not.toContain("[[log]]")
+    expect(out).not.toContain("[[overview]]")
+  })
+
+  it("infers type from path when frontmatter type is missing", () => {
+    const pages: IndexInputPage[] = [
+      { relativePath: "wiki/entities/bar.md", content: "---\ntitle: Bar\n---\n\n# Bar\n\nAn entity." },
+    ]
+    const out = generateIndexMd(pages, { date: "2026-06-27" })
+    expect(out).toContain("## Entities")
+    expect(out).toContain("- [[bar]] — An entity.")
+  })
+
+  it("sorts entries within a type alphabetically by slug", () => {
+    const pages = [
+      page("wiki/entities/zebra.md", { type: "entity", title: "Zebra" }, "# Zebra\n\nLast."),
+      page("wiki/entities/alpha.md", { type: "entity", title: "Alpha" }, "# Alpha\n\nFirst."),
+    ]
+    const out = generateIndexMd(pages, { date: "2026-06-27" })
+    expect(out.indexOf("[[alpha]]")).toBeLessThan(out.indexOf("[[zebra]]"))
+  })
+
+  it("falls back to a placeholder when there are no pages", () => {
+    const out = generateIndexMd([], { date: "2026-06-27" })
+    expect(out).toContain("_No pages yet._")
+  })
+
+  it("places custom/unknown types after known types, alphabetically", () => {
+    const pages = [
+      page("wiki/entities/foo.md", { type: "entity", title: "Foo" }, "# Foo\n\nE."),
+      page("wiki/people/jane.md", { type: "people", title: "Jane" }, "# Jane\n\nP."),
+    ]
+    const out = generateIndexMd(pages, { date: "2026-06-27" })
+    expect(out.indexOf("## Entities")).toBeLessThan(out.indexOf("## People"))
+  })
+})
+
+describe("extractIndexDescription", () => {
+  it("takes the first sentence of the body", () => {
+    expect(extractIndexDescription("# Title\n\nFirst sentence. Second sentence.", "Title", "slug")).toBe("First sentence.")
+  })
+
+  it("skips headings, comments, blockquotes, and lists", () => {
+    const body = "# Heading\n\n<!-- comment -->\n\n> placeholder\n\n- a list item\n\nReal prose here."
+    expect(extractIndexDescription(body, "Title", "slug")).toBe("Real prose here.")
+  })
+
+  it("flattens wikilinks and markdown to plain text", () => {
+    const body = "# T\n\nSee [[other-page|the other page]] and **bold** `code`."
+    expect(extractIndexDescription(body, "T", "slug")).toBe("See the other page and bold code.")
+  })
+
+  it("falls back to title when the body has no prose", () => {
+    expect(extractIndexDescription("# Only Heading", "My Title", "slug")).toBe("My Title")
+  })
+
+  it("falls back to a de-slugified slug when title is empty too", () => {
+    expect(extractIndexDescription("", "", "my-page-slug")).toBe("my page slug")
+  })
+
+  it("truncates very long sentences", () => {
+    const long = "x".repeat(300) + "."
+    const out = extractIndexDescription(`# T\n\n${long}`, "T", "slug")
+    expect(out.length).toBeLessThanOrEqual(200)
+    expect(out.endsWith("…")).toBe(true)
+  })
+})
+
+describe("buildLogEntry", () => {
+  it("formats a log line with date and title", () => {
+    expect(buildLogEntry("2026-06-27", "My Source")).toBe("## [2026-06-27] ingest | My Source")
+  })
+
+  it("collapses whitespace in the title", () => {
+    expect(buildLogEntry("2026-06-27", "  My   Source  ")).toBe("## [2026-06-27] ingest | My Source")
+  })
+
+  it("uses a placeholder for an empty title", () => {
+    expect(buildLogEntry("2026-06-27", "")).toBe("## [2026-06-27] ingest | (untitled source)")
+  })
+})

src/lib/index-generator.ts

@@ -0,0 +1,250 @@
+/**
+ * Deterministic wiki/index.md and wiki/log.md generation.
+ *
+ * Why this exists: ingest used to ask the LLM to emit an updated
+ * index.md (full file) and a log.md entry on every wave. Because the
+ * Claude/Codex CLI transports are stateless and the HTTP providers see
+ * only a snapshot of the current index, the model routinely dropped
+ * existing index entries, re-referenced deleted pages, or echoed the
+ * entire log (which the append-only writer then duplicated). The index
+ * is fully derivable from each page's frontmatter, so we generate it
+ * from disk instead — the LLM never writes it. The log is append-only
+ * and its single new line is composed here from the ingest date + the
+ * source title rather than trusted to the model.
+ *
+ * These functions are pure (string in, string out); the caller in
+ * ingest.ts is responsible for reading every page off disk and writing
+ * the result back inside the project lock.
+ */
+
+import { parseFrontmatter } from "@/lib/frontmatter"
+import { inferWikiTypeFromPath, wikiTypeLabel } from "@/lib/wiki-page-types"
+import { normalizePath, getFileName } from "@/lib/path-utils"
+
+export interface IndexInputPage {
+  /** Wiki-root-relative path, e.g. "wiki/entities/foo.md". */
+  relativePath: string
+  content: string
+}
+
+/** Display order for known page types; unknown/custom types follow,
+ * sorted alphabetically. Mirrors WIKI_TYPE_DIRS ordering in
+ * wiki-page-types.ts so the rendered index is stable run-to-run. */
+const TYPE_DISPLAY_ORDER = [
+  "entity",
+  "concept",
+  "source",
+  "query",
+  "comparison",
+  "synthesis",
+  "finding",
+  "thesis",
+  "methodology",
+]
+
+// Aggregate / structural files that are never themselves index entries.
+// schema.md and purpose.md are project-scaffolding pages (some projects
+// place them under wiki/); excluding them by basename keeps stray
+// `[[schema]]` / `[[purpose]]` entries out of the index.
+const EXCLUDED_BASENAMES = new Set([
+  "index.md",
+  "log.md",
+  "overview.md",
+  "schema.md",
+  "purpose.md",
+])
+
+interface IndexEntry {
+  slug: string
+  type: string
+  description: string
+}
+
+/**
+ * Build the complete wiki/index.md content deterministically from the
+ * frontmatter + first body sentence of every content page.
+ *
+ * Pages are grouped by type (entity, concept, …) in a stable order;
+ * each entry is `- [[slug]] — description`. The slug is the page's
+ * filename without extension so it resolves as an Obsidian wikilink.
+ */
+export function generateIndexMd(
+  pages: IndexInputPage[],
+  opts: { date: string },
+): string {
+  const entries: IndexEntry[] = []
+  for (const page of pages) {
+    const rel = normalizePath(page.relativePath)
+    const baseName = getFileName(rel).toLowerCase()
+    if (EXCLUDED_BASENAMES.has(baseName)) continue
+    if (!rel.endsWith(".md")) continue
+    // Source summary pages live under wiki/sources/ and ARE indexed
+    // (type "source"); only the three aggregate files above are skipped.
+
+    const { frontmatter, body } = parseFrontmatter(page.content)
+    const slug = getFileName(rel).replace(/\.md$/i, "")
+    if (!slug) continue
+
+    const fmType =
+      typeof frontmatter?.type === "string" ? frontmatter.type.trim() : ""
+    const type = fmType || inferWikiTypeFromPath(rel) || "other"
+    // overview-typed stray pages (other than overview.md, already
+    // excluded) shouldn't appear as index entries either.
+    if (type === "overview") continue
+
+    const title =
+      typeof frontmatter?.title === "string" ? frontmatter.title.trim() : ""
+    const description = extractIndexDescription(body, title, slug)
+    entries.push({ slug, type, description })
+  }
+
+  // Group by type.
+  const byType = new Map<string, IndexEntry[]>()
+  for (const entry of entries) {
+    const bucket = byType.get(entry.type) ?? []
+    bucket.push(entry)
+    byType.set(entry.type, bucket)
+  }
+
+  const orderedTypes = [...byType.keys()].sort(compareTypes)
+
+  const sections: string[] = []
+  for (const type of orderedTypes) {
+    const bucket = byType.get(type)
+    if (!bucket || bucket.length === 0) continue
+    bucket.sort((a, b) => a.slug.localeCompare(b.slug))
+    const heading = pluralizeTypeLabel(type)
+    const lines = bucket.map((entry) =>
+      entry.description
+        ? `- [[${entry.slug}]] — ${entry.description}`
+        : `- [[${entry.slug}]]`,
+    )
+    sections.push(`## ${heading}\n\n${lines.join("\n")}`)
+  }
+
+  const frontmatter = [
+    "---",
+    "type: overview",
+    "title: Wiki Index",
+    "tags: []",
+    "related: []",
+    `created: ${opts.date}`,
+    `updated: ${opts.date}`,
+    "---",
+  ].join("\n")
+
+  const body =
+    sections.length > 0
+      ? sections.join("\n\n")
+      : "_No pages yet._"
+
+  return `${frontmatter}\n\n# Wiki Index\n\n${body}\n`
+}
+
+/** Order known types by TYPE_DISPLAY_ORDER, then unknown types A→Z. */
+function compareTypes(a: string, b: string): number {
+  const ia = TYPE_DISPLAY_ORDER.indexOf(a)
+  const ib = TYPE_DISPLAY_ORDER.indexOf(b)
+  if (ia !== -1 && ib !== -1) return ia - ib
+  if (ia !== -1) return -1
+  if (ib !== -1) return 1
+  return a.localeCompare(b)
+}
+
+/** Section heading for a type: known labels pluralized, custom types
+ * title-cased via wikiTypeLabel. */
+function pluralizeTypeLabel(type: string): string {
+  switch (type) {
+    case "entity":
+      return "Entities"
+    case "concept":
+      return "Concepts"
+    case "source":
+      return "Sources"
+    case "query":
+      return "Queries"
+    case "comparison":
+      return "Comparisons"
+    case "synthesis":
+      return "Synthesis"
+    case "finding":
+      return "Findings"
+    case "thesis":
+      return "Theses"
+    case "methodology":
+      return "Methodologies"
+    case "other":
+      return "Other"
+    default:
+      return wikiTypeLabel(type)
+  }
+}
+
+const MAX_DESCRIPTION_CHARS = 200
+
+/**
+ * Derive a one-line description for an index entry: the first sentence
+ * of the page body, falling back to the title, then the slug. Markdown
+ * decoration (wikilinks, links, emphasis, inline code) is flattened to
+ * plain text so the index line stays readable.
+ */
+export function extractIndexDescription(
+  body: string,
+  title: string,
+  slug: string,
+): string {
+  const firstSentence = firstBodySentence(body)
+  const chosen = firstSentence || title || slug.replace(/[-_]+/g, " ")
+  return truncate(flattenInlineMarkdown(chosen), MAX_DESCRIPTION_CHARS)
+}
+
+/** Pull the first prose sentence from a markdown body, skipping the H1
+ * heading, blank lines, HTML comments, and block markup. */
+function firstBodySentence(body: string): string {
+  const lines = body.split(/\r?\n/)
+  for (const raw of lines) {
+    const line = raw.trim()
+    if (!line) continue
+    if (line.startsWith("#")) continue // headings (incl. the H1)
+    if (line.startsWith("<!--")) continue // HTML comments
+    if (line.startsWith("---")) continue // stray fences / hrules
+    if (line.startsWith("```")) continue // code fences
+    if (line.startsWith(">")) continue // blockquote placeholders
+    if (/^[-*+]\s/.test(line)) continue // list items (rarely a good summary)
+    if (/^\|/.test(line)) continue // table rows
+    // Found a prose line. Take up to the first sentence terminator.
+    const sentence = line.match(/^(.+?[.!?])(?:\s|$)/)
+    return sentence ? sentence[1] : line
+  }
+  return ""
+}
+
+/** Strip inline markdown so a description renders as clean text. */
+function flattenInlineMarkdown(text: string): string {
+  return text
+    // [[target|alias]] → alias ; [[target]] → target
+    .replace(/\[\[([^\]|]+)\|([^\]]+)\]\]/g, "$2")
+    .replace(/\[\[([^\]]+)\]\]/g, "$1")
+    // [label](url) → label
+    .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
+    // **bold** / *italic* / _italic_ / `code`
+    .replace(/[*_`]+/g, "")
+    .replace(/\s+/g, " ")
+    .trim()
+}
+
+function truncate(text: string, max: number): string {
+  if (text.length <= max) return text
+  return `${text.slice(0, max - 1).trimEnd()}…`
+}
+
+/**
+ * Build the single log.md line for an ingest. Append-only: the caller
+ * concatenates this onto the existing log, never rewriting prior
+ * entries. Format matches the historical LLM-emitted shape so existing
+ * logs stay consistent: `## [YYYY-MM-DD] ingest | Title`.
+ */
+export function buildLogEntry(date: string, title: string): string {
+  const clean = title.trim().replace(/\s+/g, " ") || "(untitled source)"
+  return `## [${date}] ingest | ${clean}`
+}