docs: serve markdown to AI agents

website/docusaurus.config.js

@@ -72,12 +72,14 @@ const config = {
   ],
   plugins: [
     [
-      'docusaurus-plugin-llms',
+      // docusaurus-plugin-llms plus <link rel="alternate"> to each page's markdown twin
+      require.resolve('./plugins/llms-markdown'),
       {
         title: 'SOQL Lib',
         description: 'Apex SOQL provides functional constructs for SOQL.',
         includeOrder: ['docs/*', 'soql/**', 'cache/**', 'evaluator/**'],
         pathTransformation: { ignorePaths: ['docs'] },
+        generateMarkdownFiles: true,
       },
     ],
     async function tailwindPlugin(context, options) {

website/middleware.js

@@ -0,0 +1,63 @@
+import { next, rewrite } from '@vercel/functions';
+
+// Vercel Routing Middleware (this directory is the project root): serves the markdown twin of a docs page
+// (/page.md, built by plugins/llms-markdown.js) to AI agents that send `Accept: text/markdown`. The home
+// page has no twin, so it answers with llms.txt.
+export const config = {
+  // Pages (with or without .html) and .md twins only; skips assets and any other file
+  matcher: '/((?!assets/)(?!.*\\.(?!(?:md|html)$)[^/.]+$).*)',
+};
+
+// Suffix of page URLs on this site: '' with cleanUrls, '.html' without
+const PAGE_SUFFIX = '';
+
+const quality = (accept, type) => {
+  for (const part of accept.split(',')) {
+    const [media, ...params] = part
+      .split(';')
+      .map((value) => value.trim().toLowerCase());
+    if (media !== type) continue;
+    const q = params.find((param) => param.startsWith('q='));
+    return q ? Number(q.slice(2)) || 0 : 1;
+  }
+  return 0;
+};
+
+const prefersMarkdown = (accept) => {
+  const markdown = quality(accept, 'text/markdown');
+  return markdown > 0 && markdown >= quality(accept, 'text/html');
+};
+
+export default async function middleware(request) {
+  const url = new URL(request.url);
+
+  if (url.pathname.endsWith('.md')) {
+    return next({
+      headers: {
+        Link: `<${url.origin}${url.pathname.slice(0, -3)}${PAGE_SUFFIX}>; rel="canonical"`,
+        Vary: 'Accept',
+      },
+    });
+  }
+  if (!prefersMarkdown(request.headers.get('accept') || '')) {
+    return next({ headers: { Vary: 'Accept' } });
+  }
+
+  const page = url.pathname.replace(/(\/index)?(\.html)?\/?$/, '');
+  const markdown = new URL(page ? `${page}.md` : '/llms.txt', url.origin);
+  // Not every page has a twin (archived versions, for one). Credentials are forwarded and redirects
+  // not followed, so a protected preview's login redirect does not pass for an existing file.
+  const headers = {};
+  for (const name of ['cookie', 'x-vercel-protection-bypass']) {
+    if (request.headers.has(name)) headers[name] = request.headers.get(name);
+  }
+  const twin = await fetch(markdown, {
+    method: 'HEAD',
+    headers,
+    redirect: 'manual',
+  }).catch(() => null);
+  if (!twin?.ok) {
+    return next({ headers: { Vary: 'Accept' } });
+  }
+  return rewrite(markdown, { headers: { Vary: 'Accept' } });
+}

website/package.json

@@ -32,7 +32,8 @@
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "^3.8.1",
-    "@docusaurus/types": "^3.8.1"
+    "@docusaurus/types": "^3.8.1",
+    "@vercel/functions": "^3.9.11"
   },
   "browserslist": {
     "production": [

website/plugins/llms-markdown.js

@@ -0,0 +1,55 @@
+// Wraps docusaurus-plugin-llms. Once it has written the markdown twin of each doc page (/page.md),
+// every built page that has a twin gets a <link rel="alternate" type="text/markdown"> to it, and the
+// home page one to llms.txt. Wrapping keeps the order: Docusaurus runs postBuild hooks in parallel.
+const fs = require('fs/promises');
+const path = require('path');
+const llms = require('docusaurus-plugin-llms').default;
+
+const exists = (file) =>
+  fs.access(file).then(
+    () => true,
+    () => false,
+  );
+
+async function htmlFiles(dir) {
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+  const files = await Promise.all(
+    entries.map((entry) => {
+      const file = path.join(dir, entry.name);
+      if (entry.isDirectory()) return htmlFiles(file);
+      return entry.name.endsWith('.html') ? [file] : [];
+    }),
+  );
+  return files.flat();
+}
+
+async function linkMarkdownTwins(outDir, siteUrl) {
+  for (const file of await htmlFiles(outDir)) {
+    const route = path
+      .relative(outDir, file)
+      .split(path.sep)
+      .join('/')
+      .replace(/(^|\/)index\.html$/, '')
+      .replace(/\.html$/, '');
+    if (route === '404') continue;
+    const twin = route ? `${route}.md` : 'llms.txt';
+    if (!(await exists(path.join(outDir, twin)))) continue;
+    const html = await fs.readFile(file, 'utf8');
+    const link = `<link rel="alternate" type="text/markdown" href="${siteUrl}/${twin}">`;
+    await fs.writeFile(file, html.replace('</head>', `${link}</head>`));
+  }
+}
+
+module.exports = function llmsMarkdown(context, options) {
+  const plugin = llms(context, options);
+  return {
+    ...plugin,
+    async postBuild(props) {
+      await plugin.postBuild(props);
+      await linkMarkdownTwins(
+        props.outDir,
+        context.siteConfig.url.replace(/\/$/, ''),
+      );
+    },
+  };
+};