Let the people whose writing is in the index sell it

apps/web/package.json

@@ -13,14 +13,15 @@
     "@nichedb/config": "workspace:*",
     "@nichedb/core": "workspace:*",
     "@nichedb/db": "workspace:*",
+    "@nichedb/enrichers": "workspace:*",
     "@nichedb/notify": "workspace:*",
     "@nichedb/payments": "workspace:*",
     "@nichedb/queue": "workspace:*",
+    "@profullstack/leaderboard": "^0.3.0",
+    "@profullstack/nichedb": "workspace:*",
+    "@profullstack/partners": "0.2.0",
     "@profullstack/x402-gateway": "^0.1.0",
     "@simplewebauthn/browser": "^13.2.0",
-    "hono": "^4.10.3",
-    "@profullstack/nichedb": "workspace:*",
-    "@nichedb/enrichers": "workspace:*",
-    "@profullstack/leaderboard": "^0.3.0"
+    "hono": "^4.10.3"
   }
 }

apps/web/src/app.js

@@ -3,6 +3,7 @@ import { Hono } from 'hono';
 import { isProUser, loadUser, render, wantsJson } from './lib/http.js';
 import { leaderboard } from './lib/leaderboard.js';
 import { modulesFor, withModules } from './lib/modules.js';
+import { partners } from './lib/partners.js';
 import { gateway, gatewayFor } from './lib/pricing.js';
 import { Denied } from './lib/service.js';
 import { registerApi } from './routes/api.js';
@@ -56,6 +57,17 @@ app.use('*', async (c, next) => {
   return answer ?? next();
 });
 
+/**
+ * The seller side: where the people whose writing is in this index sign up,
+ * prove they own a site, and get paid a share of what crawlers pay for access.
+ * Null when PARTNER_VERIFY_SECRET is unset, in which case /sell does not exist.
+ */
+if (partners)
+  app.use('*', async (c, next) => {
+    const answer = await partners.handle(c.req.raw);
+    return answer ?? next();
+  });
+
 /** Ads and tracking: on for free, off for Pro, the buyer's choice with a pass. */
 app.use('*', async (c, next) => {
   const modules = await modulesFor(c, isProUser);

apps/web/src/lib/partners.js

@@ -0,0 +1,127 @@
+import * as auth from '@nichedb/auth';
+import { config } from '@nichedb/config';
+import { sql } from '@nichedb/db';
+import { createPartners, sqlStore } from '@profullstack/partners';
+
+/**
+ * The seller side of NicheDB.
+ *
+ * Everything in this index is somebody's writing, and training crawlers are
+ * most of what reads it. This is how the people whose work is here get paid
+ * for that: prove you own a site, say what you publish about, and take a share
+ * of what a crawler pays for access.
+ *
+ * The module owns none of the authentication. It asks who is here and we
+ * answer from the same session cookie the rest of the site uses, so a partner
+ * signs in once, the ordinary way.
+ */
+
+const execute = async ({ sql: text, args = [] }) => ({ rows: await sql.unsafe(text, args) });
+const store = sqlStore({ execute, dialect: 'postgres' });
+
+// The three tables arrive through migration 0006, the same way as every other
+// schema change on this database, rather than through the package's own
+// migrate() at boot. The tables therefore exist whether or not the program is
+// switched on, so enabling it later needs no migration.
+
+/** The session cookie, read off a bare Request rather than a Hono context. */
+function sessionCookie(request) {
+  const header = request.headers.get('cookie') ?? '';
+  for (const part of header.split(';')) {
+    const eq = part.indexOf('=');
+    if (eq === -1) continue;
+    if (part.slice(0, eq).trim() === config.session.cookie) return part.slice(eq + 1).trim();
+  }
+  return null;
+}
+
+async function currentUser(request) {
+  const sid = sessionCookie(request);
+  if (!sid) return null;
+  const user = await auth.userFromRequest(sid);
+  if (!user) return null;
+  return { id: user.id, name: user.display_name ?? user.handle ?? null, email: user.email ?? null };
+}
+
+/**
+ * The niches a partner may claim: this site's own public collections, which is
+ * the vocabulary the directory is already organised by. A free-text field
+ * would collect forty spellings of "machine learning" and nothing to group on.
+ * Resolved per request, so a collection added today is claimable today.
+ */
+async function niches() {
+  const rows = await sql`select slug from collections where public = true order by slug limit 60`;
+  return rows.map((r) => String(r.slug));
+}
+
+/**
+ * The program, or null when PARTNER_VERIFY_SECRET is unset.
+ *
+ * The module refuses to start without a secret, and it is right to: a
+ * guessable verification token pays the wrong person for someone else's work.
+ * But that refusal must not take the whole site down on a deploy where the
+ * variable was forgotten, so it is caught here and /sell simply does not
+ * exist until the secret does.
+ */
+export const partners = (() => {
+  if (!config.partnerSecret) {
+    console.warn('[partners] PARTNER_VERIFY_SECRET is unset, so /sell is off');
+    return null;
+  }
+  return createPartners({
+    siteName: config.siteName,
+    siteUrl: config.siteUrl,
+    basePath: '/sell',
+    store,
+    secret: config.partnerSecret,
+    loginUrl: '/login?next=/sell',
+    currentUser,
+    niches,
+  });
+})();
+
+/**
+ * Split one crawl sale across the partners whose properties are in the index.
+ *
+ * Attribution is ours to decide, because only this side knows whose rows were
+ * in the crawl that got paid for. The rule: every partner with a verified
+ * property that has a source in the index shares the sale, each at their own
+ * rate, pro-rata by how many items they contributed. A sale nobody contributed
+ * to is simply not split.
+ *
+ * `ref` makes it idempotent, so a settlement delivered twice pays once.
+ */
+export async function splitSale(sale) {
+  if (!partners || !sale?.ref || !Number(sale.totalCents)) return 0;
+
+  const rows = await sql`
+    select p.id as partner_id, count(i.id)::int as items
+    from partner_properties pp
+    join partner_accounts p on p.id::text = pp.partner_id
+    join sources s on s.owner_id::text = p.user_id
+    join items i on i.source_id = s.id
+    where pp.verified_at is not null
+    group by p.id
+    having count(i.id) > 0`;
+  if (!rows.length) return 0;
+
+  const total = rows.reduce((n, r) => n + Number(r.items), 0);
+  let paid = 0;
+  for (const row of rows) {
+    const partner = await store.getPartnerById(String(row.partner_id));
+    if (!partner) continue;
+    const properties = await store.listProperties(partner.id);
+    const rate = partners.rateFor(partner, properties);
+    const share = Math.floor((Number(sale.totalCents) * (Number(row.items) / total) * rate) / 100);
+    if (share <= 0) continue;
+    if (
+      await partners.credit({
+        partnerId: partner.id,
+        cents: share,
+        ref: `${sale.ref}:${partner.id}`,
+      })
+    )
+      paid += share;
+  }
+  return paid;
+}

apps/web/src/lib/pricing.js

@@ -15,6 +15,8 @@ import { config } from '@nichedb/config';
 import * as q from '@nichedb/db/queries';
 import { createGateway, decodePayment } from '@profullstack/x402-gateway';
 
+import { splitSale } from './partners.js';
+
 /** "1000:20,5000:40" → [{ spentCents: 1000, off: 0.2 }, …], ascending by spend. */
 export function parseLoyalty(spec) {
   return String(spec ?? '')
@@ -112,6 +114,11 @@ export function gatewayOptions(priceCents) {
       '/manifest.webmanifest',
       '/leaderboard',
       '/leaderboard/',
+      // The pitch is how a publisher finds out they can be paid for what a
+      // crawler is already taking. Charging a crawler to read our own
+      // recruiting page would be an odd way to run a marketplace.
+      '/sell',
+      '/sell/',
     ],
     onSale: async (sale) => {
       console.log('[x402] sold a pass', {
@@ -129,6 +136,12 @@ export function gatewayOptions(priceCents) {
         userAgent: sale.userAgent,
         expiresAt: sale.expiresAt,
       });
+      // Pay the people whose writing was crawled. Never allowed to fail the
+      // sale: the money has already moved, and a split we can retry later is
+      // worth more than a 500 to a paying customer.
+      await splitSale(sale).catch((err) =>
+        console.error('[partners] could not split the sale', err),
+      );
     },
   };
 }

packages/config/src/index.js

@@ -213,6 +213,11 @@ export const config = {
     enabled: bool('CACHE_ENABLED', true),
   },
 
+  // Verification tokens for the partner program are HMACs under this. It has
+  // no default: a guessable token would let anyone claim anyone's domain and
+  // be paid for their work, and a weak fallback is how that ships by accident.
+  partnerSecret: opt('PARTNER_VERIFY_SECRET', ''),
+
   session: {
     cookie: 'ndb_session',
     ttlDays: num('SESSION_TTL_DAYS', 90),

packages/db/migrations/0006_partners.sql

@@ -0,0 +1,43 @@
+-- The seller side: who may be paid for the writing in this index.
+--
+-- Everything here is somebody's work, and training crawlers are most of what
+-- reads it. These three tables are how the people whose work it is get paid a
+-- share of what a crawler pays for access (@profullstack/partners).
+--
+-- The DDL matches the package's own `sqlStore({ dialect: 'postgres' }).schema`
+-- exactly, and lives here as a numbered migration rather than running at boot,
+-- so schema changes arrive the same way as every other one on this database.
+create table if not exists partner_accounts (
+  id             bigserial primary key,
+  user_id        text not null unique,
+  name           text,
+  -- A comma-joined list, which is the package's storage shape. Every read goes
+  -- through it, so nothing here parses this column by hand.
+  niches         text not null default '',
+  payout_address text,
+  created_at     bigint not null default (extract(epoch from now()) * 1000)
+);
+
+-- `domain` is unique across every partner, not per partner: two accounts
+-- claiming the same site is the shape of one person being paid for another's
+-- work, so the database refuses it rather than the application remembering to.
+create table if not exists partner_properties (
+  id          bigserial primary key,
+  partner_id  text not null,
+  domain      text not null unique,
+  verified_at bigint,
+  method      text,
+  created_at  bigint not null default (extract(epoch from now()) * 1000)
+);
+create index if not exists partner_properties_partner on partner_properties (partner_id);
+
+-- `ref` is unique so a settlement delivered twice pays once. Credits outlive
+-- the property that earned them: removing a site does not erase its earnings.
+create table if not exists partner_credits (
+  id         bigserial primary key,
+  partner_id text not null,
+  cents      bigint not null,
+  ref        text unique,
+  at         bigint not null
+);
+create index if not exists partner_credits_partner on partner_credits (partner_id);