Every paid crawl now reaches the people whose niche was crawled

apps/web/src/lib/attribution.js

@@ -0,0 +1,92 @@
+import { sql } from '@nichedb/db';
+import * as r from '@nichedb/db/revenue';
+import { splitSaleAcrossNiches } from '@nichedb/knowledge';
+
+/**
+ * Turning one crawl sale into revenue somebody is owed.
+ *
+ * A pass buys the whole index for a day, not one niche, so there is no single
+ * niche to hand it to. The only measurable answer to "whose data did it pay
+ * for" is how much of the index each niche holds, which is the same rule the
+ * partner programme splits on.
+ *
+ * So a sale is divided pro-rata by items across the WHOLE index, and only the
+ * niches somebody actually operates get an attributed event. The rest stays
+ * platform revenue. A niche with one percent of the rows does not collect the
+ * whole dollar because it happens to be the only one with an operator.
+ */
+
+/** Every niche with an active operator, and how many rows its collection holds. */
+async function operatedNiches() {
+  return sql`
+    select n.id, n.slug, count(i.id)::int as items
+    from niches n
+    join niche_members m on m.niche_id = n.id and m.status = 'active'
+      and m.role in ('operator', 'specialist')
+    left join items i on i.collection_id = n.collection_id
+    where n.collection_id is not null
+    group by n.id, n.slug
+    having count(i.id) > 0
+  `;
+}
+
+/**
+ * Book one paid crawl into the revenue ledger.
+ *
+ * Every event is keyed on the payment reference, so a settlement delivered
+ * twice books once even though it becomes several rows. The platform's own
+ * remainder is booked too, unattributed, so the ledger's total matches what
+ * was actually charged rather than only the part somebody is owed.
+ *
+ * Returns what it wrote. Never throws: the caller is inside a payment hook and
+ * the money has already moved.
+ */
+export async function attributeCrawlSale(sale) {
+  if (!sale?.ref) return { booked: 0, events: [] };
+  const totalCents = Math.max(0, Math.round(Number(sale.totalCents) || 0));
+  if (totalCents === 0) return { booked: 0, events: [] };
+
+  const [{ n: totalItems }] = await sql`select count(*)::int as n from items`;
+  const operated = await operatedNiches();
+  const split = splitSaleAcrossNiches({ totalCents, operated, totalItems });
+
+  const events = [];
+  for (const niche of split.niches) {
+    const out = await r.recordRevenueEvent({
+      // One reference per niche, so the whole sale stays idempotent even
+      // though it lands as several rows.
+      externalId: `x402:${sale.ref}:${niche.slug}`,
+      nicheId: niche.id,
+      sourceType: 'x402',
+      sourceId: sale.ref,
+      grossMinor: niche.cents,
+      currency: sale.currency ?? 'USD',
+      occurredAt: sale.occurredAt ?? null,
+      metadata: {
+        payer: sale.payer ?? null,
+        days: sale.days ?? 1,
+        userAgent: sale.userAgent ?? null,
+        // What the share was computed from, so the number can be argued with.
+        items: niche.items,
+        indexItems: totalItems,
+      },
+    });
+    if (out.event && !out.duplicate) events.push({ niche: niche.slug, cents: niche.cents });
+  }
+
+  if (split.remainderCents > 0) {
+    const out = await r.recordRevenueEvent({
+      externalId: `x402:${sale.ref}`,
+      nicheId: null,
+      sourceType: 'x402',
+      sourceId: sale.ref,
+      grossMinor: split.remainderCents,
+      currency: sale.currency ?? 'USD',
+      occurredAt: sale.occurredAt ?? null,
+      metadata: { payer: sale.payer ?? null, unattributed: true, indexItems: totalItems },
+    });
+    if (out.event && !out.duplicate) events.push({ niche: null, cents: split.remainderCents });
+  }
+
+  return { booked: events.reduce((n, e) => n + e.cents, 0), events };
+}

apps/web/src/lib/pricing.js

@@ -15,6 +15,7 @@ import { config } from '@nichedb/config';
 import * as q from '@nichedb/db/queries';
 import { createGateway, decodePayment } from '@profullstack/x402-gateway';
 
+import { attributeCrawlSale } from './attribution.js';
 import { splitSale } from './partners.js';
 
 /** "1000:20,5000:40" → [{ spentCents: 1000, off: 0.2 }, …], ascending by spend. */
@@ -147,6 +148,12 @@ export function gatewayOptions(priceCents) {
       await splitSale(sale).catch((err) =>
         console.error('[partners] could not split the sale', err),
       );
+      // And book it against the niches the crawl actually read, so the people
+      // operating them are owed their share. Same rule: the money has moved,
+      // so a failure here is a log line and a reconciliation job, never a 500.
+      await attributeCrawlSale(sale).catch((err) =>
+        console.error('[revenue] could not attribute the sale', err),
+      );
     },
   };
 }

docs/revenue-ledger.md

@@ -81,9 +81,10 @@ clicking at once take disjoint sets instead of blocking or double-claiming.
 
 ## Where earnings come from
 
-Today, only the signed internal endpoint. Wiring the x402 gateway's existing
-`onSale` hook into it is Phase 4 and is now a small job: `machineRevenueEvent`
-already normalises the sale object the gateway hands over, and it is tested.
+Two places. Every paid crawl pass, booked automatically by the gateway's
+`onSale` hook and divided across niches by how much of the index each holds
+(see [x402-attribution.md](./x402-attribution.md)); and the signed internal
+endpoint, for anything else.
 
 ```sh
 BODY='{"payload":{"nicheSlug":"games","externalId":"pay_1","sourceType":"x402",

docs/x402-attribution.md

@@ -52,13 +52,49 @@ twice is one row.
 A sale with no `ref` is not a ledger event and returns null, because there is
 nothing to be idempotent on.
 
-## Attribution
+## Attribution, as built
 
 Only this side knows whose rows were in the crawl that got paid for, which is
-why the gateway does not attempt it. Where a resource maps to a niche, the sale
-is attributed to that niche and divided by `allocate()` between its members and
-the platform. A sale nobody operates goes entirely to the platform. Not every
-request needs a Knowledge Influencer attached.
+why the gateway does not attempt it. `onSale` now does three things: records
+the sale in `crawl_sales` as before, splits it to partners as before, and books
+it into the revenue ledger.
+
+The hard part is that **a pass buys the whole index for a day, not one niche**.
+There is no single niche to hand it to. The only measurable answer to "whose
+data did this pay for" is how much of the index each niche holds, which is the
+rule the partner split already uses.
+
+So one sale becomes several revenue events:
+
+- one per niche that somebody actually operates, sized pro-rata by the items in
+  its collection **against the whole index**;
+- one unattributed event for the remainder, which is the part of the index
+  nobody operates.
+
+The share is taken against the whole index deliberately. A niche holding one
+percent of the rows does not collect the whole dollar because it happens to be
+the only one with an operator.
+
+```
+index: 100 items (games 90, research 10), only games operated
+$1 sale ->  x402:<ref>:games   90c   attributed, split by the ladder
+            x402:<ref>         10c   platform, unattributed
+                              ----
+                              100c
+```
+
+Every cent lands somewhere: `apportion` divides by largest remainder, so the
+parts sum to exactly the sale rather than to 99 cents. The unoperated part of
+the index is a claimant in that division rather than a leftover.
+
+Each event carries what its share was computed from (`items`, `indexItems`) in
+its metadata, so the number can be argued with rather than just believed.
+
+Idempotency is per event: `x402:<ref>:<niche>` and `x402:<ref>`. A settlement
+delivered twice books once even though it lands as several rows.
+
+Attribution is never allowed to fail the sale. The money has already moved, so
+a failure is a log line, the same rule the partner split follows.
 
 ## What is open, and why
 

packages/knowledge/src/events.js

@@ -10,7 +10,7 @@
  * ledger tables the arithmetic under them is already known to be right.
  */
 
-import { MAX_SHARE_BPS, splitShareBps } from './tiers.js';
+import { apportion, MAX_SHARE_BPS, splitShareBps } from './tiers.js';
 
 /**
  * Every event between NicheDB, Chovy and the gateway carries this envelope.
@@ -183,3 +183,39 @@ export const PAYOUT_STATES = [
   'failed',
   'reversed',
 ];
+
+/**
+ * How one crawl sale divides between niches.
+ *
+ * A pass buys the whole index for a day, not one niche, so there is no single
+ * niche to hand it to. The only measurable answer to "whose data did this pay
+ * for" is how much of the index each niche holds, which is the rule the
+ * partner programme already splits on.
+ *
+ * The share is taken against the WHOLE index, and the part nobody operates is
+ * a claimant too. A niche holding one percent of the rows does not collect the
+ * whole dollar because it happens to be the only one with an operator.
+ *
+ * Returns one entry per operated niche plus the platform remainder.
+ */
+export function splitSaleAcrossNiches({ totalCents, operated, totalItems }) {
+  const total = Math.max(0, Math.round(Number(totalCents) || 0));
+  const all = Math.max(0, Number(totalItems) || 0);
+  if (total === 0 || all === 0 || !operated?.length) return { niches: [], remainderCents: total };
+
+  // The unoperated remainder is a claimant too, so the apportionment covers
+  // the whole index and every cent lands somewhere.
+  const operatedItems = operated.reduce((n, o) => n + Math.max(0, Number(o.items) || 0), 0);
+  const weights = [
+    ...operated.map((o) => Math.max(0, Number(o.items) || 0)),
+    Math.max(0, all - operatedItems),
+  ];
+  const parts = apportion(total, weights);
+
+  // A niche apportioned zero cents contributes nothing, so dropping it cannot
+  // lose money: the kept shares plus the remainder still sum to the sale.
+  return {
+    niches: operated.map((o, i) => ({ ...o, cents: parts[i] })).filter((o) => o.cents > 0),
+    remainderCents: parts.at(-1),
+  };
+}

packages/knowledge/src/index.js

@@ -21,9 +21,11 @@ export {
   NICHEDB_EVENTS,
   PAYOUT_STATES,
   REVENUE_SOURCE_TYPES,
+  splitSaleAcrossNiches,
 } from './events.js';
 export { dedupeKeyFor, diminishFactor, scoreContribution } from './score.js';
 export {
+  apportion,
   BASE_SHARE_BPS,
   CONTRIBUTION_TIERS,
   formatBps,

packages/knowledge/src/tiers.js

@@ -85,19 +85,43 @@ export function splitShareBps(members, { maxBps = MAX_SHARE_BPS } = {}) {
   // scaled by the same factor, so relative standing survives the squeeze.
   if (wanted <= maxBps) return rows.map((r) => ({ ...r, shareBps: r.wanted }));
 
-  const exact = rows.map((r) => (r.wanted * maxBps) / wanted);
-  const floors = exact.map((n) => Math.floor(n));
-  let left = maxBps - floors.reduce((a, b) => a + b, 0);
+  const out = apportion(
+    maxBps,
+    rows.map((r) => r.wanted),
+  );
+  return rows.map((r, i) => ({ ...r, shareBps: out[i] }));
+}
+
+/**
+ * Divide a whole integer between claimants in proportion to their weights,
+ * losing nothing.
+ *
+ * Largest remainder: everyone takes their floor, then the leftover units go to
+ * whoever was cut hardest, one each. The parts sum to exactly `total`, which
+ * matters twice over here. Shares that sum to 7,999 basis points quietly
+ * underpay somebody, and cents that do not add up to the sale mean the ledger
+ * disagrees with the bank.
+ *
+ * Ties break on index so the same input always divides the same way.
+ */
+export function apportion(total, weights) {
+  const whole = Math.max(0, Math.floor(Number(total) || 0));
+  const w = (weights ?? []).map((n) => Math.max(0, Number(n) || 0));
+  const sum = w.reduce((a, b) => a + b, 0);
+  if (whole === 0 || sum === 0) return w.map(() => 0);
+
+  const exact = w.map((n) => (n * whole) / sum);
+  const out = exact.map((n) => Math.floor(n));
+  let left = whole - out.reduce((a, b) => a + b, 0);
   const order = exact
     .map((n, i) => ({ i, frac: n - Math.floor(n) }))
     .sort((a, b) => b.frac - a.frac || a.i - b.i);
-  const out = [...floors];
   for (const { i } of order) {
     if (left <= 0) break;
     out[i] += 1;
     left -= 1;
   }
-  return rows.map((r, i) => ({ ...r, shareBps: out[i] }));
+  return out;
 }
 
 /** "40%" from 4000, for a page. One decimal only when the number needs it. */