From 46ae3f913a254bba368bad40aa9daa7d6a19f599 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 6 Sep 2026 04:41:19 +0000 Subject: [PATCH] Let the people whose writing is in the index sell it The board says who pays for this data and who is refused. Nothing said how the people whose work is being sold get any of it, and the blog post pitching exactly that had nowhere to send anyone: no signup, no onboarding, no payout. /sell is that. Prove you own a site (DNS TXT or one file under /.well-known, both automated, no 'email us' path), say which of our collections you publish into, add a USDC address, and take a share of what a crawler pays for access. The rate starts at 20% and climbs 5 points per verified property and per niche to a cap of 80%, using the same ladder the public board displays, so a partner is never shown two different numbers for the same thing. Attribution is ours because only this side knows whose rows were in the crawl that got paid for: every partner with a verified property and sources in the index shares each sale, pro-rata by items contributed, each at their own rate. The credit ref is the sale ref plus the partner id, so a settlement delivered twice pays once. It runs after the sale is booked and can never fail it: the money has already moved, and a split we can retry beats a 500 to a customer. @profullstack/partners owns none of the auth. It asks who is here and we answer from the session cookie the rest of the site already uses. Without PARTNER_VERIFY_SECRET the module refuses to start, which is correct (a guessable token pays the wrong person) but must not take the site down, so it is caught and /sell simply does not exist until the secret does. The pitch is in openPaths: charging a crawler to read our own recruiting page would be an odd way to run a marketplace. Verified against a throwaway Postgres, driving the real app: signed out gets the pitch and a redirect, apply lands on the dashboard, a property shows its real token, a verified property plus one niche reads 30%, a $10 sale splits to 300 cents, the same sale replayed adds nothing, and GPTBot reads /sell at 200 while / still answers 402. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_0144uEVbZK3jdaQkwcLYTXPE --- apps/web/package.json | 9 +- apps/web/src/app.js | 12 +++ apps/web/src/lib/partners.js | 127 +++++++++++++++++++++++ apps/web/src/lib/pricing.js | 13 +++ bun.lock | 5 + packages/config/src/index.js | 5 + packages/db/migrations/0006_partners.sql | 43 ++++++++ 7 files changed, 210 insertions(+), 4 deletions(-) create mode 100644 apps/web/src/lib/partners.js create mode 100644 packages/db/migrations/0006_partners.sql diff --git a/apps/web/package.json b/apps/web/package.json index 12052bb..ebb5f7d 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -13,14 +13,15 @@ "@nichedb/config": "workspace:*", "@nichedb/core": "workspace:*", "@nichedb/db": "workspace:*", + "@nichedb/enrichers": "workspace:*", "@nichedb/notify": "workspace:*", "@nichedb/payments": "workspace:*", "@nichedb/queue": "workspace:*", + "@profullstack/leaderboard": "^0.3.0", + "@profullstack/nichedb": "workspace:*", + "@profullstack/partners": "0.2.0", "@profullstack/x402-gateway": "^0.1.0", "@simplewebauthn/browser": "^13.2.0", - "hono": "^4.10.3", - "@profullstack/nichedb": "workspace:*", - "@nichedb/enrichers": "workspace:*", - "@profullstack/leaderboard": "^0.3.0" + "hono": "^4.10.3" } } diff --git a/apps/web/src/app.js b/apps/web/src/app.js index e7929f5..27bc764 100644 --- a/apps/web/src/app.js +++ b/apps/web/src/app.js @@ -3,6 +3,7 @@ import { Hono } from 'hono'; import { isProUser, loadUser, render, wantsJson } from './lib/http.js'; import { leaderboard } from './lib/leaderboard.js'; import { modulesFor, withModules } from './lib/modules.js'; +import { partners } from './lib/partners.js'; import { gateway, gatewayFor } from './lib/pricing.js'; import { Denied } from './lib/service.js'; import { registerApi } from './routes/api.js'; @@ -56,6 +57,17 @@ app.use('*', async (c, next) => { return answer ?? next(); }); +/** + * The seller side: where the people whose writing is in this index sign up, + * prove they own a site, and get paid a share of what crawlers pay for access. + * Null when PARTNER_VERIFY_SECRET is unset, in which case /sell does not exist. + */ +if (partners) + app.use('*', async (c, next) => { + const answer = await partners.handle(c.req.raw); + return answer ?? next(); + }); + /** Ads and tracking: on for free, off for Pro, the buyer's choice with a pass. */ app.use('*', async (c, next) => { const modules = await modulesFor(c, isProUser); diff --git a/apps/web/src/lib/partners.js b/apps/web/src/lib/partners.js new file mode 100644 index 0000000..873c14b --- /dev/null +++ b/apps/web/src/lib/partners.js @@ -0,0 +1,127 @@ +import * as auth from '@nichedb/auth'; +import { config } from '@nichedb/config'; +import { sql } from '@nichedb/db'; +import { createPartners, sqlStore } from '@profullstack/partners'; + +/** + * The seller side of NicheDB. + * + * Everything in this index is somebody's writing, and training crawlers are + * most of what reads it. This is how the people whose work is here get paid + * for that: prove you own a site, say what you publish about, and take a share + * of what a crawler pays for access. + * + * The module owns none of the authentication. It asks who is here and we + * answer from the same session cookie the rest of the site uses, so a partner + * signs in once, the ordinary way. + */ + +const execute = async ({ sql: text, args = [] }) => ({ rows: await sql.unsafe(text, args) }); +const store = sqlStore({ execute, dialect: 'postgres' }); + +// The three tables arrive through migration 0006, the same way as every other +// schema change on this database, rather than through the package's own +// migrate() at boot. The tables therefore exist whether or not the program is +// switched on, so enabling it later needs no migration. + +/** The session cookie, read off a bare Request rather than a Hono context. */ +function sessionCookie(request) { + const header = request.headers.get('cookie') ?? ''; + for (const part of header.split(';')) { + const eq = part.indexOf('='); + if (eq === -1) continue; + if (part.slice(0, eq).trim() === config.session.cookie) return part.slice(eq + 1).trim(); + } + return null; +} + +async function currentUser(request) { + const sid = sessionCookie(request); + if (!sid) return null; + const user = await auth.userFromRequest(sid); + if (!user) return null; + return { id: user.id, name: user.display_name ?? user.handle ?? null, email: user.email ?? null }; +} + +/** + * The niches a partner may claim: this site's own public collections, which is + * the vocabulary the directory is already organised by. A free-text field + * would collect forty spellings of "machine learning" and nothing to group on. + * Resolved per request, so a collection added today is claimable today. + */ +async function niches() { + const rows = await sql`select slug from collections where public = true order by slug limit 60`; + return rows.map((r) => String(r.slug)); +} + +/** + * The program, or null when PARTNER_VERIFY_SECRET is unset. + * + * The module refuses to start without a secret, and it is right to: a + * guessable verification token pays the wrong person for someone else's work. + * But that refusal must not take the whole site down on a deploy where the + * variable was forgotten, so it is caught here and /sell simply does not + * exist until the secret does. + */ +export const partners = (() => { + if (!config.partnerSecret) { + console.warn('[partners] PARTNER_VERIFY_SECRET is unset, so /sell is off'); + return null; + } + return createPartners({ + siteName: config.siteName, + siteUrl: config.siteUrl, + basePath: '/sell', + store, + secret: config.partnerSecret, + loginUrl: '/login?next=/sell', + currentUser, + niches, + }); +})(); + +/** + * Split one crawl sale across the partners whose properties are in the index. + * + * Attribution is ours to decide, because only this side knows whose rows were + * in the crawl that got paid for. The rule: every partner with a verified + * property that has a source in the index shares the sale, each at their own + * rate, pro-rata by how many items they contributed. A sale nobody contributed + * to is simply not split. + * + * `ref` makes it idempotent, so a settlement delivered twice pays once. + */ +export async function splitSale(sale) { + if (!partners || !sale?.ref || !Number(sale.totalCents)) return 0; + + const rows = await sql` + select p.id as partner_id, count(i.id)::int as items + from partner_properties pp + join partner_accounts p on p.id::text = pp.partner_id + join sources s on s.owner_id::text = p.user_id + join items i on i.source_id = s.id + where pp.verified_at is not null + group by p.id + having count(i.id) > 0`; + if (!rows.length) return 0; + + const total = rows.reduce((n, r) => n + Number(r.items), 0); + let paid = 0; + for (const row of rows) { + const partner = await store.getPartnerById(String(row.partner_id)); + if (!partner) continue; + const properties = await store.listProperties(partner.id); + const rate = partners.rateFor(partner, properties); + const share = Math.floor((Number(sale.totalCents) * (Number(row.items) / total) * rate) / 100); + if (share <= 0) continue; + if ( + await partners.credit({ + partnerId: partner.id, + cents: share, + ref: `${sale.ref}:${partner.id}`, + }) + ) + paid += share; + } + return paid; +} diff --git a/apps/web/src/lib/pricing.js b/apps/web/src/lib/pricing.js index 79f8b96..83497ec 100644 --- a/apps/web/src/lib/pricing.js +++ b/apps/web/src/lib/pricing.js @@ -15,6 +15,8 @@ import { config } from '@nichedb/config'; import * as q from '@nichedb/db/queries'; import { createGateway, decodePayment } from '@profullstack/x402-gateway'; +import { splitSale } from './partners.js'; + /** "1000:20,5000:40" → [{ spentCents: 1000, off: 0.2 }, …], ascending by spend. */ export function parseLoyalty(spec) { return String(spec ?? '') @@ -112,6 +114,11 @@ export function gatewayOptions(priceCents) { '/manifest.webmanifest', '/leaderboard', '/leaderboard/', + // The pitch is how a publisher finds out they can be paid for what a + // crawler is already taking. Charging a crawler to read our own + // recruiting page would be an odd way to run a marketplace. + '/sell', + '/sell/', ], onSale: async (sale) => { console.log('[x402] sold a pass', { @@ -129,6 +136,12 @@ export function gatewayOptions(priceCents) { userAgent: sale.userAgent, expiresAt: sale.expiresAt, }); + // Pay the people whose writing was crawled. Never allowed to fail the + // sale: the money has already moved, and a split we can retry later is + // worth more than a 500 to a paying customer. + await splitSale(sale).catch((err) => + console.error('[partners] could not split the sale', err), + ); }, }; } diff --git a/bun.lock b/bun.lock index 79a9f8d..19262c4 100644 --- a/bun.lock +++ b/bun.lock @@ -31,6 +31,7 @@ "@nichedb/queue": "workspace:*", "@profullstack/leaderboard": "^0.3.0", "@profullstack/nichedb": "workspace:*", + "@profullstack/partners": "0.2.0", "@profullstack/x402-gateway": "^0.1.0", "@simplewebauthn/browser": "^13.2.0", "hono": "^4.10.3", @@ -301,6 +302,8 @@ "@profullstack/nichedb": ["@profullstack/nichedb@workspace:apps/cli"], + "@profullstack/partners": ["@profullstack/partners@0.2.0", "", { "dependencies": { "@profullstack/leaderboard": "^0.3.1" } }, "sha512-h8Fal+ZMqpR2xixo8OC/Xb/OXfLXmt2BdBIgohC3tK6U3AkAsDrkUyrlf+DEF8Ut8mptDEjCWDz5EV/dIXBRHw=="], + "@profullstack/referrals": ["@profullstack/referrals@0.1.0", "", { "peerDependencies": { "react": ">=18" }, "optionalPeers": ["react"] }, "sha512-u66SdBVpsv3kc0N+NWISPoYD5vjCERyv5wfD07iSkZwQeC2IA+ihX5jNA4e7Xr+Y4AUvxLycG+3b4VaROqzgRg=="], "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.1.0", "", {}, "sha512-B7tWvWk/bIEoqyec6UoyRF1pO7X/+b+wFRv2ZFIClqskmEpyxoA559ZgdTvnxqAIvuDeE9v56nVpYRQ+lmOZQQ=="], @@ -533,6 +536,8 @@ "@inquirer/select/@inquirer/type": ["@inquirer/type@4.1.1", "", { "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-yJoHYrMnxIsJZCY+0Vb66Dy3he3kL3e2wOBKhoSwWWAzZAY82emlxwgprCtp6yRixvNRNq9ztfRWQYPNr3Go7A=="], + "@profullstack/partners/@profullstack/leaderboard": ["@profullstack/leaderboard@0.3.1", "", {}, "sha512-LbxLwy+RvT/Qez0mfWCk4o9hvcq0kzwBZ0vF+CYp+oGvyqJWUaSTLhCHWGtKKAI3c0ZUXj6ll0iuRjbuZUE2Sg=="], + "@redis/client/cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="], "@types/mute-stream/@types/node": ["@types/node@26.4.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-k97ENvZWtvA6yqz5/FS6a7duDgOPEeOQOc2iKS/nY6mX6qJUKtLnWzQS+Xj6tXweyj6ZcTAK2Qecetnvi9nCLA=="], diff --git a/packages/config/src/index.js b/packages/config/src/index.js index 007fb5b..2cab4f7 100644 --- a/packages/config/src/index.js +++ b/packages/config/src/index.js @@ -213,6 +213,11 @@ export const config = { enabled: bool('CACHE_ENABLED', true), }, + // Verification tokens for the partner program are HMACs under this. It has + // no default: a guessable token would let anyone claim anyone's domain and + // be paid for their work, and a weak fallback is how that ships by accident. + partnerSecret: opt('PARTNER_VERIFY_SECRET', ''), + session: { cookie: 'ndb_session', ttlDays: num('SESSION_TTL_DAYS', 90), diff --git a/packages/db/migrations/0006_partners.sql b/packages/db/migrations/0006_partners.sql new file mode 100644 index 0000000..a7a04fc --- /dev/null +++ b/packages/db/migrations/0006_partners.sql @@ -0,0 +1,43 @@ +-- The seller side: who may be paid for the writing in this index. +-- +-- Everything here is somebody's work, and training crawlers are most of what +-- reads it. These three tables are how the people whose work it is get paid a +-- share of what a crawler pays for access (@profullstack/partners). +-- +-- The DDL matches the package's own `sqlStore({ dialect: 'postgres' }).schema` +-- exactly, and lives here as a numbered migration rather than running at boot, +-- so schema changes arrive the same way as every other one on this database. +create table if not exists partner_accounts ( + id bigserial primary key, + user_id text not null unique, + name text, + -- A comma-joined list, which is the package's storage shape. Every read goes + -- through it, so nothing here parses this column by hand. + niches text not null default '', + payout_address text, + created_at bigint not null default (extract(epoch from now()) * 1000) +); + +-- `domain` is unique across every partner, not per partner: two accounts +-- claiming the same site is the shape of one person being paid for another's +-- work, so the database refuses it rather than the application remembering to. +create table if not exists partner_properties ( + id bigserial primary key, + partner_id text not null, + domain text not null unique, + verified_at bigint, + method text, + created_at bigint not null default (extract(epoch from now()) * 1000) +); +create index if not exists partner_properties_partner on partner_properties (partner_id); + +-- `ref` is unique so a settlement delivered twice pays once. Credits outlive +-- the property that earned them: removing a site does not erase its earnings. +create table if not exists partner_credits ( + id bigserial primary key, + partner_id text not null, + cents bigint not null, + ref text unique, + at bigint not null +); +create index if not exists partner_credits_partner on partner_credits (partner_id);