diff --git a/apps/web/src/lib/attribution.js b/apps/web/src/lib/attribution.js new file mode 100644 index 0000000..0135822 --- /dev/null +++ b/apps/web/src/lib/attribution.js @@ -0,0 +1,92 @@ +import { sql } from '@nichedb/db'; +import * as r from '@nichedb/db/revenue'; +import { splitSaleAcrossNiches } from '@nichedb/knowledge'; + +/** + * Turning one crawl sale into revenue somebody is owed. + * + * A pass buys the whole index for a day, not one niche, so there is no single + * niche to hand it to. The only measurable answer to "whose data did it pay + * for" is how much of the index each niche holds, which is the same rule the + * partner programme splits on. + * + * So a sale is divided pro-rata by items across the WHOLE index, and only the + * niches somebody actually operates get an attributed event. The rest stays + * platform revenue. A niche with one percent of the rows does not collect the + * whole dollar because it happens to be the only one with an operator. + */ + +/** Every niche with an active operator, and how many rows its collection holds. */ +async function operatedNiches() { + return sql` + select n.id, n.slug, count(i.id)::int as items + from niches n + join niche_members m on m.niche_id = n.id and m.status = 'active' + and m.role in ('operator', 'specialist') + left join items i on i.collection_id = n.collection_id + where n.collection_id is not null + group by n.id, n.slug + having count(i.id) > 0 + `; +} + +/** + * Book one paid crawl into the revenue ledger. + * + * Every event is keyed on the payment reference, so a settlement delivered + * twice books once even though it becomes several rows. The platform's own + * remainder is booked too, unattributed, so the ledger's total matches what + * was actually charged rather than only the part somebody is owed. + * + * Returns what it wrote. Never throws: the caller is inside a payment hook and + * the money has already moved. + */ +export async function attributeCrawlSale(sale) { + if (!sale?.ref) return { booked: 0, events: [] }; + const totalCents = Math.max(0, Math.round(Number(sale.totalCents) || 0)); + if (totalCents === 0) return { booked: 0, events: [] }; + + const [{ n: totalItems }] = await sql`select count(*)::int as n from items`; + const operated = await operatedNiches(); + const split = splitSaleAcrossNiches({ totalCents, operated, totalItems }); + + const events = []; + for (const niche of split.niches) { + const out = await r.recordRevenueEvent({ + // One reference per niche, so the whole sale stays idempotent even + // though it lands as several rows. + externalId: `x402:${sale.ref}:${niche.slug}`, + nicheId: niche.id, + sourceType: 'x402', + sourceId: sale.ref, + grossMinor: niche.cents, + currency: sale.currency ?? 'USD', + occurredAt: sale.occurredAt ?? null, + metadata: { + payer: sale.payer ?? null, + days: sale.days ?? 1, + userAgent: sale.userAgent ?? null, + // What the share was computed from, so the number can be argued with. + items: niche.items, + indexItems: totalItems, + }, + }); + if (out.event && !out.duplicate) events.push({ niche: niche.slug, cents: niche.cents }); + } + + if (split.remainderCents > 0) { + const out = await r.recordRevenueEvent({ + externalId: `x402:${sale.ref}`, + nicheId: null, + sourceType: 'x402', + sourceId: sale.ref, + grossMinor: split.remainderCents, + currency: sale.currency ?? 'USD', + occurredAt: sale.occurredAt ?? null, + metadata: { payer: sale.payer ?? null, unattributed: true, indexItems: totalItems }, + }); + if (out.event && !out.duplicate) events.push({ niche: null, cents: split.remainderCents }); + } + + return { booked: events.reduce((n, e) => n + e.cents, 0), events }; +} diff --git a/apps/web/src/lib/pricing.js b/apps/web/src/lib/pricing.js index 7fe7a7e..240514b 100644 --- a/apps/web/src/lib/pricing.js +++ b/apps/web/src/lib/pricing.js @@ -15,6 +15,7 @@ import { config } from '@nichedb/config'; import * as q from '@nichedb/db/queries'; import { createGateway, decodePayment } from '@profullstack/x402-gateway'; +import { attributeCrawlSale } from './attribution.js'; import { splitSale } from './partners.js'; /** "1000:20,5000:40" → [{ spentCents: 1000, off: 0.2 }, …], ascending by spend. */ @@ -147,6 +148,12 @@ export function gatewayOptions(priceCents) { await splitSale(sale).catch((err) => console.error('[partners] could not split the sale', err), ); + // And book it against the niches the crawl actually read, so the people + // operating them are owed their share. Same rule: the money has moved, + // so a failure here is a log line and a reconciliation job, never a 500. + await attributeCrawlSale(sale).catch((err) => + console.error('[revenue] could not attribute the sale', err), + ); }, }; } diff --git a/docs/revenue-ledger.md b/docs/revenue-ledger.md index 147567b..cbac228 100644 --- a/docs/revenue-ledger.md +++ b/docs/revenue-ledger.md @@ -81,9 +81,10 @@ clicking at once take disjoint sets instead of blocking or double-claiming. ## Where earnings come from -Today, only the signed internal endpoint. Wiring the x402 gateway's existing -`onSale` hook into it is Phase 4 and is now a small job: `machineRevenueEvent` -already normalises the sale object the gateway hands over, and it is tested. +Two places. Every paid crawl pass, booked automatically by the gateway's +`onSale` hook and divided across niches by how much of the index each holds +(see [x402-attribution.md](./x402-attribution.md)); and the signed internal +endpoint, for anything else. ```sh BODY='{"payload":{"nicheSlug":"games","externalId":"pay_1","sourceType":"x402", diff --git a/docs/x402-attribution.md b/docs/x402-attribution.md index f25de6a..2583120 100644 --- a/docs/x402-attribution.md +++ b/docs/x402-attribution.md @@ -52,13 +52,49 @@ twice is one row. A sale with no `ref` is not a ledger event and returns null, because there is nothing to be idempotent on. -## Attribution +## Attribution, as built Only this side knows whose rows were in the crawl that got paid for, which is -why the gateway does not attempt it. Where a resource maps to a niche, the sale -is attributed to that niche and divided by `allocate()` between its members and -the platform. A sale nobody operates goes entirely to the platform. Not every -request needs a Knowledge Influencer attached. +why the gateway does not attempt it. `onSale` now does three things: records +the sale in `crawl_sales` as before, splits it to partners as before, and books +it into the revenue ledger. + +The hard part is that **a pass buys the whole index for a day, not one niche**. +There is no single niche to hand it to. The only measurable answer to "whose +data did this pay for" is how much of the index each niche holds, which is the +rule the partner split already uses. + +So one sale becomes several revenue events: + +- one per niche that somebody actually operates, sized pro-rata by the items in + its collection **against the whole index**; +- one unattributed event for the remainder, which is the part of the index + nobody operates. + +The share is taken against the whole index deliberately. A niche holding one +percent of the rows does not collect the whole dollar because it happens to be +the only one with an operator. + +``` +index: 100 items (games 90, research 10), only games operated +$1 sale -> x402::games 90c attributed, split by the ladder + x402: 10c platform, unattributed + ---- + 100c +``` + +Every cent lands somewhere: `apportion` divides by largest remainder, so the +parts sum to exactly the sale rather than to 99 cents. The unoperated part of +the index is a claimant in that division rather than a leftover. + +Each event carries what its share was computed from (`items`, `indexItems`) in +its metadata, so the number can be argued with rather than just believed. + +Idempotency is per event: `x402::` and `x402:`. A settlement +delivered twice books once even though it lands as several rows. + +Attribution is never allowed to fail the sale. The money has already moved, so +a failure is a log line, the same rule the partner split follows. ## What is open, and why diff --git a/packages/knowledge/src/events.js b/packages/knowledge/src/events.js index d838978..c823743 100644 --- a/packages/knowledge/src/events.js +++ b/packages/knowledge/src/events.js @@ -10,7 +10,7 @@ * ledger tables the arithmetic under them is already known to be right. */ -import { MAX_SHARE_BPS, splitShareBps } from './tiers.js'; +import { apportion, MAX_SHARE_BPS, splitShareBps } from './tiers.js'; /** * Every event between NicheDB, Chovy and the gateway carries this envelope. @@ -183,3 +183,39 @@ export const PAYOUT_STATES = [ 'failed', 'reversed', ]; + +/** + * How one crawl sale divides between niches. + * + * A pass buys the whole index for a day, not one niche, so there is no single + * niche to hand it to. The only measurable answer to "whose data did this pay + * for" is how much of the index each niche holds, which is the rule the + * partner programme already splits on. + * + * The share is taken against the WHOLE index, and the part nobody operates is + * a claimant too. A niche holding one percent of the rows does not collect the + * whole dollar because it happens to be the only one with an operator. + * + * Returns one entry per operated niche plus the platform remainder. + */ +export function splitSaleAcrossNiches({ totalCents, operated, totalItems }) { + const total = Math.max(0, Math.round(Number(totalCents) || 0)); + const all = Math.max(0, Number(totalItems) || 0); + if (total === 0 || all === 0 || !operated?.length) return { niches: [], remainderCents: total }; + + // The unoperated remainder is a claimant too, so the apportionment covers + // the whole index and every cent lands somewhere. + const operatedItems = operated.reduce((n, o) => n + Math.max(0, Number(o.items) || 0), 0); + const weights = [ + ...operated.map((o) => Math.max(0, Number(o.items) || 0)), + Math.max(0, all - operatedItems), + ]; + const parts = apportion(total, weights); + + // A niche apportioned zero cents contributes nothing, so dropping it cannot + // lose money: the kept shares plus the remainder still sum to the sale. + return { + niches: operated.map((o, i) => ({ ...o, cents: parts[i] })).filter((o) => o.cents > 0), + remainderCents: parts.at(-1), + }; +} diff --git a/packages/knowledge/src/index.js b/packages/knowledge/src/index.js index 5e26689..0788e4b 100644 --- a/packages/knowledge/src/index.js +++ b/packages/knowledge/src/index.js @@ -21,9 +21,11 @@ export { NICHEDB_EVENTS, PAYOUT_STATES, REVENUE_SOURCE_TYPES, + splitSaleAcrossNiches, } from './events.js'; export { dedupeKeyFor, diminishFactor, scoreContribution } from './score.js'; export { + apportion, BASE_SHARE_BPS, CONTRIBUTION_TIERS, formatBps, diff --git a/packages/knowledge/src/tiers.js b/packages/knowledge/src/tiers.js index 877452b..89c248e 100644 --- a/packages/knowledge/src/tiers.js +++ b/packages/knowledge/src/tiers.js @@ -85,19 +85,43 @@ export function splitShareBps(members, { maxBps = MAX_SHARE_BPS } = {}) { // scaled by the same factor, so relative standing survives the squeeze. if (wanted <= maxBps) return rows.map((r) => ({ ...r, shareBps: r.wanted })); - const exact = rows.map((r) => (r.wanted * maxBps) / wanted); - const floors = exact.map((n) => Math.floor(n)); - let left = maxBps - floors.reduce((a, b) => a + b, 0); + const out = apportion( + maxBps, + rows.map((r) => r.wanted), + ); + return rows.map((r, i) => ({ ...r, shareBps: out[i] })); +} + +/** + * Divide a whole integer between claimants in proportion to their weights, + * losing nothing. + * + * Largest remainder: everyone takes their floor, then the leftover units go to + * whoever was cut hardest, one each. The parts sum to exactly `total`, which + * matters twice over here. Shares that sum to 7,999 basis points quietly + * underpay somebody, and cents that do not add up to the sale mean the ledger + * disagrees with the bank. + * + * Ties break on index so the same input always divides the same way. + */ +export function apportion(total, weights) { + const whole = Math.max(0, Math.floor(Number(total) || 0)); + const w = (weights ?? []).map((n) => Math.max(0, Number(n) || 0)); + const sum = w.reduce((a, b) => a + b, 0); + if (whole === 0 || sum === 0) return w.map(() => 0); + + const exact = w.map((n) => (n * whole) / sum); + const out = exact.map((n) => Math.floor(n)); + let left = whole - out.reduce((a, b) => a + b, 0); const order = exact .map((n, i) => ({ i, frac: n - Math.floor(n) })) .sort((a, b) => b.frac - a.frac || a.i - b.i); - const out = [...floors]; for (const { i } of order) { if (left <= 0) break; out[i] += 1; left -= 1; } - return rows.map((r, i) => ({ ...r, shareBps: out[i] })); + return out; } /** "40%" from 4000, for a page. One decimal only when the number needs it. */ diff --git a/test/attribution.test.js b/test/attribution.test.js new file mode 100644 index 0000000..f1f2814 --- /dev/null +++ b/test/attribution.test.js @@ -0,0 +1,135 @@ +import { describe, expect, test } from 'bun:test'; +import { apportion, splitSaleAcrossNiches } from '../packages/knowledge/src/index.js'; + +/** + * Dividing a crawl sale. A pass buys the whole index for a day, so the + * question these answer is "whose data did it pay for", and the invariant + * under all of them is that every cent lands somewhere. + */ + +describe('apportion', () => { + test('the parts always sum to exactly the whole', () => { + for (const [total, weights] of [ + [100, [1, 1, 1]], + [1, [1, 1, 1]], + [7, [5, 3, 1]], + [9999, [103822, 9421, 1654]], + [3, [1]], + [1000, [1, 0, 0]], + ]) { + const parts = apportion(total, weights); + expect(parts.reduce((a, b) => a + b, 0)).toBe(total); + for (const p of parts) expect(Number.isInteger(p)).toBe(true); + for (const p of parts) expect(p).toBeGreaterThanOrEqual(0); + } + }); + + test('a cent that cannot divide goes to whoever was cut hardest, deterministically', () => { + // One cent, three equal claimants: the first by index takes it, and takes + // it again on a re-run rather than moving around. + expect(apportion(1, [1, 1, 1])).toEqual([1, 0, 0]); + expect(apportion(1, [1, 1, 1])).toEqual([1, 0, 0]); + expect(apportion(2, [1, 1, 1])).toEqual([1, 1, 0]); + }); + + test('weight is respected, not just count', () => { + expect(apportion(100, [90, 10])).toEqual([90, 10]); + expect(apportion(100, [1, 99])).toEqual([1, 99]); + }); + + test('nothing to divide, or nobody to divide between, divides to nothing', () => { + expect(apportion(0, [1, 2])).toEqual([0, 0]); + expect(apportion(100, [0, 0])).toEqual([0, 0]); + expect(apportion(100, [])).toEqual([]); + }); + + test('rubbish in does not produce NaN out', () => { + expect(apportion(null, [1])).toEqual([0]); + expect(apportion(100, [null, 'x', 1])).toEqual([0, 0, 100]); + }); +}); + +describe('splitting a crawl sale across niches', () => { + // The real shape of the index at the time of writing. + const index = 119_824; + + test('an operated niche gets its share of the index, not the whole sale', () => { + const out = splitSaleAcrossNiches({ + totalCents: 100, + totalItems: index, + operated: [{ slug: 'packages', items: 103_822 }], + }); + // 103822/119824 is about 87%, so the operator's niche books 87c and the + // rest of the index keeps 13c. Being the only niche with an operator does + // not entitle it to the whole dollar. + expect(out.niches[0].cents).toBe(87); + expect(out.remainderCents).toBe(13); + expect(out.niches[0].cents + out.remainderCents).toBe(100); + }); + + test('several operated niches divide against the whole index', () => { + const out = splitSaleAcrossNiches({ + totalCents: 1000, + totalItems: index, + operated: [ + { slug: 'packages', items: 103_822 }, + { slug: 'research', items: 1654 }, + ], + }); + const total = out.niches.reduce((n, x) => n + x.cents, 0) + out.remainderCents; + expect(total).toBe(1000); + expect(out.niches[0].cents).toBeGreaterThan(out.niches[1].cents); + }); + + test('with nobody operating anything, the whole sale is the platform remainder', () => { + const out = splitSaleAcrossNiches({ totalCents: 100, totalItems: index, operated: [] }); + expect(out.niches).toEqual([]); + expect(out.remainderCents).toBe(100); + }); + + test('an empty index attributes nothing rather than dividing by zero', () => { + const out = splitSaleAcrossNiches({ + totalCents: 100, + totalItems: 0, + operated: [{ slug: 'x', items: 0 }], + }); + expect(out.niches).toEqual([]); + expect(out.remainderCents).toBe(100); + }); + + test('a niche too small to earn a cent is dropped, and its cent is not lost', () => { + const out = splitSaleAcrossNiches({ + totalCents: 1, + totalItems: 1_000_000, + operated: [{ slug: 'tiny', items: 1 }], + }); + expect(out.niches).toEqual([]); + expect(out.remainderCents).toBe(1); + }); + + test('every cent of the sale is always accounted for', () => { + for (const total of [1, 2, 3, 99, 100, 333, 1000, 9999]) { + const out = splitSaleAcrossNiches({ + totalCents: total, + totalItems: index, + operated: [ + { slug: 'a', items: 103_822 }, + { slug: 'b', items: 9421 }, + { slug: 'c', items: 1654 }, + ], + }); + const sum = out.niches.reduce((n, x) => n + x.cents, 0) + out.remainderCents; + expect(sum).toBe(total); + } + }); + + test('a sale of nothing books nothing', () => { + const out = splitSaleAcrossNiches({ + totalCents: 0, + totalItems: index, + operated: [{ slug: 'a', items: 100 }], + }); + expect(out.niches).toEqual([]); + expect(out.remainderCents).toBe(0); + }); +});