docs: serve an llms.txt index of the documentation

pixi.toml

@@ -114,10 +114,14 @@ mkdocstrings = "*"
 mkdocstrings-python = "*"
 mkdocs-gen-files = "*"
 mkdocs-literate-nav = "*"
+nbsite = ">=0.10.0a0"
 
 [feature.doc.tasks]
 # Updated tasks to use zensical instead of mkdocs
-docs-build = { cmd = 'zensical build --clean', depends-on = ['install'] }
+_docs-build-site = { cmd = 'zensical build --clean', depends-on = ['install'] }
+# llms.txt is written into builtdocs, so it has to run after --clean has wiped it.
+_docs-markdown = 'python -m nbsite build-llms --config scripts/llms_config.py'
+docs-build = { depends-on = ['_docs-build-site', '_docs-markdown'] }
 docs-serve = { cmd = 'zensical serve', depends-on = ['install'] }
 docs-clean = { cmd = 'rm -rf builtdocs', depends-on = ['install'] }
 

scripts/llms_config.py

@@ -0,0 +1,82 @@
+"""Config for building Lumen markdown docs and llms.txt.
+
+The zensical nav is the single source of truth for which pages ship and what
+they are called: this reads it once and reuses it to build the llms.txt
+sections, instead of maintaining a second, parallel page list.
+"""
+
+import tomllib
+
+from pathlib import Path
+
+from nbsite.scripts import LlmsBuildConfig, LlmsSection, MarkdownSource
+
+ROOT = Path(__file__).parent.parent
+DOCS_DIR = ROOT / "docs"
+BUILTDOCS_DIR = ROOT / "builtdocs"
+OUTPUT_DIR = BUILTDOCS_DIR / "markdown"
+
+# Pages that carry no LLM code-gen value and should be excluded from the build.
+EXCLUDE_FILES = (
+    Path("releases.md"),
+    Path("contributing.md"),
+)
+
+
+def _flatten_nav(nav: list, trail: tuple[str, ...] = ()) -> list[tuple[tuple[str, ...], str, Path]]:
+    """Flatten zensical's nav into (group trail, label, path) triples."""
+    pages = []
+    for entry in nav:
+        for label, target in entry.items():
+            if isinstance(target, list):
+                pages.extend(_flatten_nav(target, trail + (label,)))
+            else:
+                pages.append((trail, label, Path(target)))
+    return pages
+
+
+_config = tomllib.loads((ROOT / "zensical.toml").read_text(encoding="utf-8"))["project"]
+NAV_PAGES = _flatten_nav(_config["nav"])
+LABELS = {path: label for _, label, path in NAV_PAGES}
+TRAILS = {path: trail for trail, _, path in NAV_PAGES}
+
+
+def _label(path: Path) -> str:
+    return LABELS.get(path, path.stem.replace("_", " "))
+
+
+def _section(title: str, description: str, *trail: str, group: str | None = None, under: bool = False) -> LlmsSection:
+    """One LlmsSection for a nav group, matched either exactly (*trail*) or by prefix (*under*)."""
+    matches_trail = (lambda path: TRAILS.get(path, ())[: len(trail)] == trail) if under else (lambda path: TRAILS.get(path) == trail)
+    return LlmsSection(
+        title=title,
+        description=description,
+        path_prefix=Path("."),
+        path_filter=matches_trail,
+        label_builder=_label,
+        group=group,
+    )
+
+
+CONFIG = LlmsBuildConfig(
+    project_title="Lumen",
+    project_description=(
+        "Lumen is an open-source and extensible agent framework for chatting with data "
+        "and for retrieval augmented generation. It turns natural language into SQL, data "
+        "transformation pipelines, visualizations and dashboards, and every step stays "
+        "inspectable, editable and reproducible."
+    ),
+    markdown_root=OUTPUT_DIR,
+    llms_output_path=BUILTDOCS_DIR / "llms.txt",
+    markdown_base_url="/markdown",
+    sources=(MarkdownSource(source_dir=DOCS_DIR, output_dir=OUTPUT_DIR, exclude_files=EXCLUDE_FILES),),
+    sections=(
+        _section("Overview", "Quick start, installation, and top-level pages."),
+        _section("Getting Started", "Launching Lumen, navigating the UI, and building your first app.", "Getting Started"),
+        _section("Tutorials", "Full walkthroughs building an AI-driven data exploration app end to end.", "Examples", "Tutorials", group="Examples"),
+        _section("Gallery", "Short example specs demonstrating individual sources, transforms, and views.", "Examples", "Gallery", group="Examples"),
+        _section("Configuration", "Top-level spec reference: sources, transforms, views, agents, and more.", "Configuration"),
+        _section("YAML Spec", "Detailed reference for writing a Lumen dashboard spec by hand.", "Configuration", "Specs"),
+        _section("API Reference", "Python API reference for Lumen's pipeline, sources, transforms, views, and AI components.", "Reference", under=True),
+    ),
+)

zensical.toml

@@ -26,9 +26,10 @@ nav = [
       {"SaaS Company Report Dashboard" = "examples/tutorials/saas_company_report_dashboard.md"},
       {"Census Data AI Explorer" = "examples/tutorials/census_data_ai_explorer.md"},
       {"Mesonet Weather Explorer" = "examples/tutorials/mesonet_weather_explorer.md"},
-      {"Weather API Explorer" = "examples/tutorials/weather_openapi_explorer.md"},
-      {"Massive Stock Explorer" = "examples/tutorials/massive_stock_explorer.md"},
-      {"Build Dashboard with Spec" = "examples/tutorials/penguins_dashboard_spec.md"}
+      {"Weather API Explorer (OpenAPI)" = "examples/tutorials/weather_openapi_explorer.md"},
+      {"Stock Market Explorer" = "examples/tutorials/massive_stock_explorer.md"},
+      {"Build Dashboard with Spec" = "examples/tutorials/penguins_dashboard_spec.md"},
+      {"SaaS Executive Dashboard" = "examples/tutorials/saas_metrics_reports.md"}
     ]},
     {"Gallery" = [
       {"Overview" = "examples/gallery/index.md"},