From 3c23656145255f67ea34d402c57755d59f3a0701 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 6 Sep 2026 11:28:08 +0000 Subject: [PATCH] Every paid crawl now reaches the people whose niche was crawled Phase 4. The gateway has been selling crawl passes since the site launched and the money went two places: crawl_sales, and the partner split. It never reached the revenue ledger, so a Knowledge Influencer operating a niche earned nothing from a crawler paying to read it. The hard part is that a pass buys the whole index for a day, not one niche, so there is no single niche to hand it to. The only measurable answer to whose data it paid for is how much of the index each niche holds, which is the rule the partner split already uses. One sale therefore becomes several revenue events: one per operated niche, sized pro-rata against the WHOLE index, plus an unattributed one for the part nobody operates. Against the whole index deliberately. A niche holding one percent of the rows must not collect the whole dollar because it happens to be the only one with an operator. Every cent lands somewhere. The largest-remainder split that already kept basis points summing to 8000 is now a general `apportion`, used by both, so the parts sum to exactly the sale rather than to 99 cents. The unoperated remainder is a claimant in that division rather than a leftover. Each event records the item counts its share was computed from, so the number can be argued with. Idempotent per event (`x402::` and `x402:`), so a settlement delivered twice books once even though it lands as several rows. Never allowed to fail the sale: the money has already moved. The splitting arithmetic lives in packages/knowledge with the rest of the pure domain rather than beside the database call, which is also what lets it be tested without a DATABASE_URL. Verified by calling the real onSale hook: a $1 sale over an index of 100 items with games (90) operated and research (10) not booked 90c to games and 10c unattributed, the operator was owed 18c at the contributor rate, redelivery changed nothing, and once research had an operator too the next sale split 90/10 with no remainder. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01W9NGGLDvNayi6uWheSXDGF --- apps/web/src/lib/attribution.js | 92 +++++++++++++++++++++ apps/web/src/lib/pricing.js | 7 ++ docs/revenue-ledger.md | 7 +- docs/x402-attribution.md | 46 +++++++++-- packages/knowledge/src/events.js | 38 ++++++++- packages/knowledge/src/index.js | 2 + packages/knowledge/src/tiers.js | 34 ++++++-- test/attribution.test.js | 135 +++++++++++++++++++++++++++++++ 8 files changed, 347 insertions(+), 14 deletions(-) create mode 100644 apps/web/src/lib/attribution.js create mode 100644 test/attribution.test.js diff --git a/apps/web/src/lib/attribution.js b/apps/web/src/lib/attribution.js new file mode 100644 index 0000000..0135822 --- /dev/null +++ b/apps/web/src/lib/attribution.js @@ -0,0 +1,92 @@ +import { sql } from '@nichedb/db'; +import * as r from '@nichedb/db/revenue'; +import { splitSaleAcrossNiches } from '@nichedb/knowledge'; + +/** + * Turning one crawl sale into revenue somebody is owed. + * + * A pass buys the whole index for a day, not one niche, so there is no single + * niche to hand it to. The only measurable answer to "whose data did it pay + * for" is how much of the index each niche holds, which is the same rule the + * partner programme splits on. + * + * So a sale is divided pro-rata by items across the WHOLE index, and only the + * niches somebody actually operates get an attributed event. The rest stays + * platform revenue. A niche with one percent of the rows does not collect the + * whole dollar because it happens to be the only one with an operator. + */ + +/** Every niche with an active operator, and how many rows its collection holds. */ +async function operatedNiches() { + return sql` + select n.id, n.slug, count(i.id)::int as items + from niches n + join niche_members m on m.niche_id = n.id and m.status = 'active' + and m.role in ('operator', 'specialist') + left join items i on i.collection_id = n.collection_id + where n.collection_id is not null + group by n.id, n.slug + having count(i.id) > 0 + `; +} + +/** + * Book one paid crawl into the revenue ledger. + * + * Every event is keyed on the payment reference, so a settlement delivered + * twice books once even though it becomes several rows. The platform's own + * remainder is booked too, unattributed, so the ledger's total matches what + * was actually charged rather than only the part somebody is owed. + * + * Returns what it wrote. Never throws: the caller is inside a payment hook and + * the money has already moved. + */ +export async function attributeCrawlSale(sale) { + if (!sale?.ref) return { booked: 0, events: [] }; + const totalCents = Math.max(0, Math.round(Number(sale.totalCents) || 0)); + if (totalCents === 0) return { booked: 0, events: [] }; + + const [{ n: totalItems }] = await sql`select count(*)::int as n from items`; + const operated = await operatedNiches(); + const split = splitSaleAcrossNiches({ totalCents, operated, totalItems }); + + const events = []; + for (const niche of split.niches) { + const out = await r.recordRevenueEvent({ + // One reference per niche, so the whole sale stays idempotent even + // though it lands as several rows. + externalId: `x402:${sale.ref}:${niche.slug}`, + nicheId: niche.id, + sourceType: 'x402', + sourceId: sale.ref, + grossMinor: niche.cents, + currency: sale.currency ?? 'USD', + occurredAt: sale.occurredAt ?? null, + metadata: { + payer: sale.payer ?? null, + days: sale.days ?? 1, + userAgent: sale.userAgent ?? null, + // What the share was computed from, so the number can be argued with. + items: niche.items, + indexItems: totalItems, + }, + }); + if (out.event && !out.duplicate) events.push({ niche: niche.slug, cents: niche.cents }); + } + + if (split.remainderCents > 0) { + const out = await r.recordRevenueEvent({ + externalId: `x402:${sale.ref}`, + nicheId: null, + sourceType: 'x402', + sourceId: sale.ref, + grossMinor: split.remainderCents, + currency: sale.currency ?? 'USD', + occurredAt: sale.occurredAt ?? null, + metadata: { payer: sale.payer ?? null, unattributed: true, indexItems: totalItems }, + }); + if (out.event && !out.duplicate) events.push({ niche: null, cents: split.remainderCents }); + } + + return { booked: events.reduce((n, e) => n + e.cents, 0), events }; +} diff --git a/apps/web/src/lib/pricing.js b/apps/web/src/lib/pricing.js index 7fe7a7e..240514b 100644 --- a/apps/web/src/lib/pricing.js +++ b/apps/web/src/lib/pricing.js @@ -15,6 +15,7 @@ import { config } from '@nichedb/config'; import * as q from '@nichedb/db/queries'; import { createGateway, decodePayment } from '@profullstack/x402-gateway'; +import { attributeCrawlSale } from './attribution.js'; import { splitSale } from './partners.js'; /** "1000:20,5000:40" → [{ spentCents: 1000, off: 0.2 }, …], ascending by spend. */ @@ -147,6 +148,12 @@ export function gatewayOptions(priceCents) { await splitSale(sale).catch((err) => console.error('[partners] could not split the sale', err), ); + // And book it against the niches the crawl actually read, so the people + // operating them are owed their share. Same rule: the money has moved, + // so a failure here is a log line and a reconciliation job, never a 500. + await attributeCrawlSale(sale).catch((err) => + console.error('[revenue] could not attribute the sale', err), + ); }, }; } diff --git a/docs/revenue-ledger.md b/docs/revenue-ledger.md index 147567b..cbac228 100644 --- a/docs/revenue-ledger.md +++ b/docs/revenue-ledger.md @@ -81,9 +81,10 @@ clicking at once take disjoint sets instead of blocking or double-claiming. ## Where earnings come from -Today, only the signed internal endpoint. Wiring the x402 gateway's existing -`onSale` hook into it is Phase 4 and is now a small job: `machineRevenueEvent` -already normalises the sale object the gateway hands over, and it is tested. +Two places. Every paid crawl pass, booked automatically by the gateway's +`onSale` hook and divided across niches by how much of the index each holds +(see [x402-attribution.md](./x402-attribution.md)); and the signed internal +endpoint, for anything else. ```sh BODY='{"payload":{"nicheSlug":"games","externalId":"pay_1","sourceType":"x402", diff --git a/docs/x402-attribution.md b/docs/x402-attribution.md index f25de6a..2583120 100644 --- a/docs/x402-attribution.md +++ b/docs/x402-attribution.md @@ -52,13 +52,49 @@ twice is one row. A sale with no `ref` is not a ledger event and returns null, because there is nothing to be idempotent on. -## Attribution +## Attribution, as built Only this side knows whose rows were in the crawl that got paid for, which is -why the gateway does not attempt it. Where a resource maps to a niche, the sale -is attributed to that niche and divided by `allocate()` between its members and -the platform. A sale nobody operates goes entirely to the platform. Not every -request needs a Knowledge Influencer attached. +why the gateway does not attempt it. `onSale` now does three things: records +the sale in `crawl_sales` as before, splits it to partners as before, and books +it into the revenue ledger. + +The hard part is that **a pass buys the whole index for a day, not one niche**. +There is no single niche to hand it to. The only measurable answer to "whose +data did this pay for" is how much of the index each niche holds, which is the +rule the partner split already uses. + +So one sale becomes several revenue events: + +- one per niche that somebody actually operates, sized pro-rata by the items in + its collection **against the whole index**; +- one unattributed event for the remainder, which is the part of the index + nobody operates. + +The share is taken against the whole index deliberately. A niche holding one +percent of the rows does not collect the whole dollar because it happens to be +the only one with an operator. + +``` +index: 100 items (games 90, research 10), only games operated +$1 sale -> x402::games 90c attributed, split by the ladder + x402: 10c platform, unattributed + ---- + 100c +``` + +Every cent lands somewhere: `apportion` divides by largest remainder, so the +parts sum to exactly the sale rather than to 99 cents. The unoperated part of +the index is a claimant in that division rather than a leftover. + +Each event carries what its share was computed from (`items`, `indexItems`) in +its metadata, so the number can be argued with rather than just believed. + +Idempotency is per event: `x402::` and `x402:`. A settlement +delivered twice books once even though it lands as several rows. + +Attribution is never allowed to fail the sale. The money has already moved, so +a failure is a log line, the same rule the partner split follows. ## What is open, and why diff --git a/packages/knowledge/src/events.js b/packages/knowledge/src/events.js index d838978..c823743 100644 --- a/packages/knowledge/src/events.js +++ b/packages/knowledge/src/events.js @@ -10,7 +10,7 @@ * ledger tables the arithmetic under them is already known to be right. */ -import { MAX_SHARE_BPS, splitShareBps } from './tiers.js'; +import { apportion, MAX_SHARE_BPS, splitShareBps } from './tiers.js'; /** * Every event between NicheDB, Chovy and the gateway carries this envelope. @@ -183,3 +183,39 @@ export const PAYOUT_STATES = [ 'failed', 'reversed', ]; + +/** + * How one crawl sale divides between niches. + * + * A pass buys the whole index for a day, not one niche, so there is no single + * niche to hand it to. The only measurable answer to "whose data did this pay + * for" is how much of the index each niche holds, which is the rule the + * partner programme already splits on. + * + * The share is taken against the WHOLE index, and the part nobody operates is + * a claimant too. A niche holding one percent of the rows does not collect the + * whole dollar because it happens to be the only one with an operator. + * + * Returns one entry per operated niche plus the platform remainder. + */ +export function splitSaleAcrossNiches({ totalCents, operated, totalItems }) { + const total = Math.max(0, Math.round(Number(totalCents) || 0)); + const all = Math.max(0, Number(totalItems) || 0); + if (total === 0 || all === 0 || !operated?.length) return { niches: [], remainderCents: total }; + + // The unoperated remainder is a claimant too, so the apportionment covers + // the whole index and every cent lands somewhere. + const operatedItems = operated.reduce((n, o) => n + Math.max(0, Number(o.items) || 0), 0); + const weights = [ + ...operated.map((o) => Math.max(0, Number(o.items) || 0)), + Math.max(0, all - operatedItems), + ]; + const parts = apportion(total, weights); + + // A niche apportioned zero cents contributes nothing, so dropping it cannot + // lose money: the kept shares plus the remainder still sum to the sale. + return { + niches: operated.map((o, i) => ({ ...o, cents: parts[i] })).filter((o) => o.cents > 0), + remainderCents: parts.at(-1), + }; +} diff --git a/packages/knowledge/src/index.js b/packages/knowledge/src/index.js index 5e26689..0788e4b 100644 --- a/packages/knowledge/src/index.js +++ b/packages/knowledge/src/index.js @@ -21,9 +21,11 @@ export { NICHEDB_EVENTS, PAYOUT_STATES, REVENUE_SOURCE_TYPES, + splitSaleAcrossNiches, } from './events.js'; export { dedupeKeyFor, diminishFactor, scoreContribution } from './score.js'; export { + apportion, BASE_SHARE_BPS, CONTRIBUTION_TIERS, formatBps, diff --git a/packages/knowledge/src/tiers.js b/packages/knowledge/src/tiers.js index 877452b..89c248e 100644 --- a/packages/knowledge/src/tiers.js +++ b/packages/knowledge/src/tiers.js @@ -85,19 +85,43 @@ export function splitShareBps(members, { maxBps = MAX_SHARE_BPS } = {}) { // scaled by the same factor, so relative standing survives the squeeze. if (wanted <= maxBps) return rows.map((r) => ({ ...r, shareBps: r.wanted })); - const exact = rows.map((r) => (r.wanted * maxBps) / wanted); - const floors = exact.map((n) => Math.floor(n)); - let left = maxBps - floors.reduce((a, b) => a + b, 0); + const out = apportion( + maxBps, + rows.map((r) => r.wanted), + ); + return rows.map((r, i) => ({ ...r, shareBps: out[i] })); +} + +/** + * Divide a whole integer between claimants in proportion to their weights, + * losing nothing. + * + * Largest remainder: everyone takes their floor, then the leftover units go to + * whoever was cut hardest, one each. The parts sum to exactly `total`, which + * matters twice over here. Shares that sum to 7,999 basis points quietly + * underpay somebody, and cents that do not add up to the sale mean the ledger + * disagrees with the bank. + * + * Ties break on index so the same input always divides the same way. + */ +export function apportion(total, weights) { + const whole = Math.max(0, Math.floor(Number(total) || 0)); + const w = (weights ?? []).map((n) => Math.max(0, Number(n) || 0)); + const sum = w.reduce((a, b) => a + b, 0); + if (whole === 0 || sum === 0) return w.map(() => 0); + + const exact = w.map((n) => (n * whole) / sum); + const out = exact.map((n) => Math.floor(n)); + let left = whole - out.reduce((a, b) => a + b, 0); const order = exact .map((n, i) => ({ i, frac: n - Math.floor(n) })) .sort((a, b) => b.frac - a.frac || a.i - b.i); - const out = [...floors]; for (const { i } of order) { if (left <= 0) break; out[i] += 1; left -= 1; } - return rows.map((r, i) => ({ ...r, shareBps: out[i] })); + return out; } /** "40%" from 4000, for a page. One decimal only when the number needs it. */ diff --git a/test/attribution.test.js b/test/attribution.test.js new file mode 100644 index 0000000..f1f2814 --- /dev/null +++ b/test/attribution.test.js @@ -0,0 +1,135 @@ +import { describe, expect, test } from 'bun:test'; +import { apportion, splitSaleAcrossNiches } from '../packages/knowledge/src/index.js'; + +/** + * Dividing a crawl sale. A pass buys the whole index for a day, so the + * question these answer is "whose data did it pay for", and the invariant + * under all of them is that every cent lands somewhere. + */ + +describe('apportion', () => { + test('the parts always sum to exactly the whole', () => { + for (const [total, weights] of [ + [100, [1, 1, 1]], + [1, [1, 1, 1]], + [7, [5, 3, 1]], + [9999, [103822, 9421, 1654]], + [3, [1]], + [1000, [1, 0, 0]], + ]) { + const parts = apportion(total, weights); + expect(parts.reduce((a, b) => a + b, 0)).toBe(total); + for (const p of parts) expect(Number.isInteger(p)).toBe(true); + for (const p of parts) expect(p).toBeGreaterThanOrEqual(0); + } + }); + + test('a cent that cannot divide goes to whoever was cut hardest, deterministically', () => { + // One cent, three equal claimants: the first by index takes it, and takes + // it again on a re-run rather than moving around. + expect(apportion(1, [1, 1, 1])).toEqual([1, 0, 0]); + expect(apportion(1, [1, 1, 1])).toEqual([1, 0, 0]); + expect(apportion(2, [1, 1, 1])).toEqual([1, 1, 0]); + }); + + test('weight is respected, not just count', () => { + expect(apportion(100, [90, 10])).toEqual([90, 10]); + expect(apportion(100, [1, 99])).toEqual([1, 99]); + }); + + test('nothing to divide, or nobody to divide between, divides to nothing', () => { + expect(apportion(0, [1, 2])).toEqual([0, 0]); + expect(apportion(100, [0, 0])).toEqual([0, 0]); + expect(apportion(100, [])).toEqual([]); + }); + + test('rubbish in does not produce NaN out', () => { + expect(apportion(null, [1])).toEqual([0]); + expect(apportion(100, [null, 'x', 1])).toEqual([0, 0, 100]); + }); +}); + +describe('splitting a crawl sale across niches', () => { + // The real shape of the index at the time of writing. + const index = 119_824; + + test('an operated niche gets its share of the index, not the whole sale', () => { + const out = splitSaleAcrossNiches({ + totalCents: 100, + totalItems: index, + operated: [{ slug: 'packages', items: 103_822 }], + }); + // 103822/119824 is about 87%, so the operator's niche books 87c and the + // rest of the index keeps 13c. Being the only niche with an operator does + // not entitle it to the whole dollar. + expect(out.niches[0].cents).toBe(87); + expect(out.remainderCents).toBe(13); + expect(out.niches[0].cents + out.remainderCents).toBe(100); + }); + + test('several operated niches divide against the whole index', () => { + const out = splitSaleAcrossNiches({ + totalCents: 1000, + totalItems: index, + operated: [ + { slug: 'packages', items: 103_822 }, + { slug: 'research', items: 1654 }, + ], + }); + const total = out.niches.reduce((n, x) => n + x.cents, 0) + out.remainderCents; + expect(total).toBe(1000); + expect(out.niches[0].cents).toBeGreaterThan(out.niches[1].cents); + }); + + test('with nobody operating anything, the whole sale is the platform remainder', () => { + const out = splitSaleAcrossNiches({ totalCents: 100, totalItems: index, operated: [] }); + expect(out.niches).toEqual([]); + expect(out.remainderCents).toBe(100); + }); + + test('an empty index attributes nothing rather than dividing by zero', () => { + const out = splitSaleAcrossNiches({ + totalCents: 100, + totalItems: 0, + operated: [{ slug: 'x', items: 0 }], + }); + expect(out.niches).toEqual([]); + expect(out.remainderCents).toBe(100); + }); + + test('a niche too small to earn a cent is dropped, and its cent is not lost', () => { + const out = splitSaleAcrossNiches({ + totalCents: 1, + totalItems: 1_000_000, + operated: [{ slug: 'tiny', items: 1 }], + }); + expect(out.niches).toEqual([]); + expect(out.remainderCents).toBe(1); + }); + + test('every cent of the sale is always accounted for', () => { + for (const total of [1, 2, 3, 99, 100, 333, 1000, 9999]) { + const out = splitSaleAcrossNiches({ + totalCents: total, + totalItems: index, + operated: [ + { slug: 'a', items: 103_822 }, + { slug: 'b', items: 9421 }, + { slug: 'c', items: 1654 }, + ], + }); + const sum = out.niches.reduce((n, x) => n + x.cents, 0) + out.remainderCents; + expect(sum).toBe(total); + } + }); + + test('a sale of nothing books nothing', () => { + const out = splitSaleAcrossNiches({ + totalCents: 0, + totalItems: index, + operated: [{ slug: 'a', items: 100 }], + }); + expect(out.niches).toEqual([]); + expect(out.remainderCents).toBe(0); + }); +});