diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7664576..ed42440 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: pull_request: jobs: - test: + node: strategy: fail-fast: false matrix: @@ -25,3 +25,31 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm test + + bun: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + - run: bun install --frozen-lockfile || bun install + - run: bun test + - run: bun bin/x402.js --help + + deno: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + # Deno resolves the noble packages from node_modules next to package.json. + - run: pnpm install --frozen-lockfile + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + - run: deno test -A --no-check test/ + - run: deno run -A bin/x402.js --help diff --git a/README.md b/README.md index 0dd22b9..ddf5438 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,35 @@ # @profullstack/x402-client -Pay an [x402](https://x402.org) `402 Payment Required` from Node with a local key. Sign the EIP-3009 authorization, retry with the proof, keep the pass. - -A crawler, an agent, or a script that meets a paid site needs three things: to read the offer, to sign it with a key it holds, and to remember what it bought so the next thousand requests are free. That is the whole package: a `fetch` that does all three, a CLI for shells, and the header that lets Lightpanda, Puppeteer or curl through a site sold by [@profullstack/x402-gateway](https://github.com/profullstack/x402-gateway) or any other x402 v2 server. +[![npm](https://img.shields.io/npm/v/%40profullstack%2Fx402-client?logo=npm&label=npm)](https://www.npmjs.com/package/@profullstack/x402-client) +[![ci](https://github.com/profullstack/x402-client/actions/workflows/ci.yml/badge.svg)](https://github.com/profullstack/x402-client/actions/workflows/ci.yml) +[![node](https://img.shields.io/badge/node-%E2%89%A520.19-339933?logo=node.js&logoColor=white)](#runtimes) +[![bun](https://img.shields.io/badge/bun-tested-fbf0df?logo=bun&logoColor=black)](#runtimes) +[![deno](https://img.shields.io/badge/deno-tested-000000?logo=deno&logoColor=white)](#runtimes) +[![types](https://img.shields.io/badge/types-included-3178c6?logo=typescript&logoColor=white)](index.d.ts) +[![license](https://img.shields.io/npm/l/%40profullstack%2Fx402-client)](LICENSE) + +Pay an [x402](https://x402.org) `402 Payment Required` from JavaScript with a local key. Sign the EIP-3009 authorization, retry with the proof, keep the pass. + +A crawler, an agent, or a script that meets a paid site needs three things: to read the offer, to sign it with a key it holds, and to remember what it bought so the next thousand requests are free. That is the whole package. It ships as a `fetch`, a CLI, and adapters that make the tools you already crawl with pay as they go. + +| You crawl with | Add | Docs | +| --- | --- | --- | +| `fetch` — Node, Bun, Deno, browser, `undici`, `node-fetch`, `ky`, `ofetch` | `wrapFetch(fetch)` | [fetch](#fetch) | +| **axios** | `attachX402(axios, client)` | [axios](#axios) | +| **got**, `got-scraping`, **Crawlee** | `got.extend(x402Hooks(client))` | [got](#got-and-crawlee) | +| **Puppeteer**, `puppeteer-core`, Lightpanda over CDP | `attachX402(page, client)` | [Puppeteer](#puppeteer) | +| **Playwright** | `await attachX402(context, client)` | [Playwright](#playwright) | +| **curl**, wget, HTTPie, Lightpanda's CLI, anything with a header flag | `-H "x-crawl-pass: $(x402 pass URL)"` | [CLI](#the-cli) | ``` -npm install @profullstack/x402-client +npm install @profullstack/x402-client # node +bun add @profullstack/x402-client # bun +deno add npm:@profullstack/x402-client # deno ``` Two dependencies, `@noble/curves` and `@noble/hashes`, for one hash function and one signature. No ethers, no viem, no RPC: the payer never sends a transaction and never needs ETH. The facilitator broadcasts the authorization and pays the gas; the wallet holds USDC and signs. -## Fetch +## The client ```js import { createClient } from '@profullstack/x402-client'; @@ -23,12 +42,10 @@ const again = await client.fetch('https://rssamplifier.com/another/page'); // presents the pass, pays nothing. ``` -`fetch` behaves like the global one until it sees a 402 carrying an x402 offer. Then it picks the first option it can sign, signs an EIP-3009 `TransferWithAuthorization` for exactly the offered amount to exactly the offered address, and asks again with the proof in `X-PAYMENT` and `PAYMENT-SIGNATURE`. A gateway answers with a receipt naming a pass, which is filed and presented on every later request to that origin; a plain x402 resource answers with the resource, which is returned as it came. +`client.fetch` behaves like the global one until it sees a 402 carrying an x402 offer. Then it picks the first option it can sign, signs an EIP-3009 `TransferWithAuthorization` for exactly the offered amount to exactly the offered address, and asks again with the proof in `X-PAYMENT` and `PAYMENT-SIGNATURE`. A gateway answers with a receipt naming a pass, which is filed and presented on every later request to that origin; a plain x402 resource answers with the resource, which is returned as it came. It pays once per request and never guesses: a second 402 after a proof means the payment was refused, and that is thrown as an `X402Error` with `code: 'rejected'` and the server's reason, not retried with a fresh signature. -## Pay, explicitly - ```js const result = await client.pay('https://rssamplifier.com/crawl'); result.payer; // 0x… the address that signed @@ -37,36 +54,126 @@ result.network; // 'eip155:8453' result.receipt.pass; // 'cp_…' the day pass result.receipt.header; // 'x-crawl-pass' where to put it result.receipt.expires; // ISO timestamp -``` -`pay` asks the URL for its offer (as JSON), signs, and files whatever pass comes back. Give it `{ offer }` to skip the first request when you already hold one. +client.passFor(url); // the live pass filed for that site, or null +``` -## Options +### Options ```js createClient({ - key, // 32-byte secp256k1 private key, hex. Or `wallet`. - maxUsd: 5, // refuse any single payment above this. Default 5. + key, // 32-byte secp256k1 private key, hex. Or `wallet`. + maxUsd: 5, // refuse any single payment above this. Default 5. networks: ['eip155:8453'], // only pay on these chains, e.g. the one you funded - store: 'file', // file passes in ~/.config/x402-client/passes.json; default memory + store: 'file', // file passes in ~/.config/x402-client/passes.json; default memory userAgent: 'MyBot/1.0 (+https://…)', - validForSeconds, // authorization lifetime; defaults to the offer's maxTimeoutSeconds - fetch, // your own fetch, for proxies and tests + validForSeconds, // authorization lifetime; defaults to the offer's maxTimeoutSeconds + fetch, // your own fetch, for proxies and tests }); ``` -The ceiling is the one setting to think about. A client that holds a key and pays whatever it is asked is a client a hostile server can drain by asking. A day of crawling on a gateway costs a dollar; five is room for a dearer site and not for a mistake. +The ceiling is the one setting to think about. A client that holds a key and pays whatever it is asked is a client a hostile server can drain by asking. A day of crawling on a gateway costs a dollar; five is room for a dearer site and not for a mistake. Buying a week at once is `?days=7` on the sales page and `maxUsd: 7`. `networks` matters because the server's `accepts` is in its order of preference, and the first entry may be a chain your wallet has no USDC on. A signed authorization on an empty wallet verifies and then fails to settle, and the server answers 402 as if you had never paid. +## Adapters + +Every adapter takes a client, presents the filed pass on the way out, pays a 402 once, and replays the request with the pass. None of them import the library they adapt: you pass in the instance you already have, and the package stays two dependencies. The axios and got adapters are tested against the real libraries; the browser adapters are tested against library-shaped fakes in CI and were verified by hand with real Puppeteer on Chrome and on Lightpanda, and real Playwright on Chrome, each loading a gated page with exactly one payment. + +A gateway answers a browser's 402 with an HTML sales page rather than the JSON offer, since the browser asked for HTML. Every adapter handles that: when the 402 carries no offer, the client asks the URL again as JSON and pays what it says. + +### fetch + +```js +import { wrapFetch, x402Fetch } from '@profullstack/x402-client/fetch'; + +const fetch = x402Fetch({ key }); // the global fetch, paying +const paying = wrapFetch(undici.fetch, { key }); // any fetch-shaped function + +// Hand it to anything with a fetch option: +const api = ky.create({ fetch: paying }); +const ai = new OpenAI({ fetch: paying }); +``` + +Give several wrappers one `client` to share a wallet and a pass file: + +```js +const client = createClient({ key, store: 'file' }); +const a = wrapFetch(fetch, { client }); +const b = wrapFetch(nodeFetch, { client }); +``` + +### axios + +```js +import axios from 'axios'; +import { createClient } from '@profullstack/x402-client'; +import { attachX402 } from '@profullstack/x402-client/axios'; + +const detach = attachX402(axios, createClient({ key })); // or an axios.create() +const { data } = await axios.get('https://site/paid/page'); // paid once, then free +``` + +A request interceptor puts the pass on requests to sites that have one; a response interceptor catches the 402, pays, and replays with the pass. Works whether the 402 arrives as a rejection (axios' default) or as a response (a widened `validateStatus`). `baseURL` and relative URLs resolve as axios resolves them. The one request that carries the proof goes through the client's own fetch, not through axios, so an axios proxy does not apply to it. + +### got and Crawlee + +```js +import got from 'got'; +import { x402Hooks } from '@profullstack/x402-client/got'; + +const paying = got.extend(x402Hooks(client)); +const { body } = await paying('https://site/paid/page'); +``` + +`beforeRequest` presents the pass, `afterResponse` pays a 402 and asks got to retry with the pass merged in. Crawlee's `got-scraping` is got underneath and takes the same hooks; for its browser crawlers, use the Playwright or Puppeteer adapter on the page in `preNavigationHooks`. + +### Puppeteer + +```js +import puppeteer from 'puppeteer'; +import { attachX402 } from '@profullstack/x402-client/puppeteer'; + +const page = await browser.newPage(); +attachX402(page, client); +await page.goto('https://site/paid/page'); // 402 → paid from Node → loaded +``` + +Puppeteer's interception cannot retry a response, so the handshake lives at the navigation: `page.goto` presents the filed pass, and on a 402 pays, presents, and navigates again. `gotoPaid(page, url, client)` is the same without patching, `applyPass(page, url, client)` just sets the header. Against Lightpanda, `puppeteer.connect({ browserWSEndpoint: 'ws://127.0.0.1:9222' })` and nothing else changes; see [`examples/lightpanda.mjs`](examples/lightpanda.mjs). The pass is sent with `setExtraHTTPHeaders`, which is page-wide; it is a bearer token for one site and nothing else, and it expires, but clear it with `page.setExtraHTTPHeaders({})` if the crawl leaves the site. + +### Playwright + +```js +import { chromium } from 'playwright'; +import { attachX402 } from '@profullstack/x402-client/playwright'; + +const context = await browser.newContext(); +await attachX402(context, client); // or a single page +const page = await context.newPage(); +await page.goto('https://site/paid/page'); // the page sees the paid answer +``` + +Playwright's routing makes the request itself, so this is the cleanest of the adapters: every document, XHR and fetch request goes out with the pass, a 402 is paid and the request made again, and the page receives the paid answer as if the site had never asked. Other resources continue untouched but for the header. `resourceTypes` and `pattern` are options; `gotoPaid` exists for symmetry. The browser never holds the key. See [`examples/playwright.mjs`](examples/playwright.mjs). + +### Lightpanda + +No plugin system, and none needed: pay once from Node, hand the browser the header. + +``` +lightpanda fetch --http-header "x-crawl-pass: $(x402 pass https://site/crawl)" https://site/page +``` + +Or connect Puppeteer or Playwright to `lightpanda serve` and use the adapter above. + ## The CLI ``` -npm install -g @profullstack/x402-client +npm install -g @profullstack/x402-client # or: npx / bunx / deno run -A npm:@profullstack/x402-client export X402_PRIVATE_KEY=0x… x402 address # fund this x402 pay https://site/crawl # receipt as JSON +x402 pay "https://site/crawl?days=7" --max-usd 7 x402 pass https://site/crawl # the pass, buying one if none is on file x402 fetch https://site/page # the page, paying if it asks x402 passes # what is on file @@ -75,13 +182,22 @@ x402 passes # what is on file The key is read from the environment or `--key-file`, never from an argument. Passes are filed in `$XDG_CONFIG_HOME/x402-client/passes.json` with mode 0600, so one invocation buys and the rest present: ``` -curl -H "x-crawl-pass: $(x402 pass https://site/crawl)" https://site/page -lightpanda fetch --http-header "x-crawl-pass: $(x402 pass https://site/crawl)" https://site/page +curl -H "x-crawl-pass: $(x402 pass https://site/crawl)" https://site/page +wget --header="x-crawl-pass: $(x402 pass https://site/crawl)" https://site/page +http https://site/page "x-crawl-pass:$(x402 pass https://site/crawl)" ``` -## Lightpanda and Puppeteer +## Runtimes + +Tested in CI on every push: -Lightpanda has no plugin system, and does not need one: a gateway sells time, not pages, so the browser only has to carry the header a payment bought. The shell line above is the whole integration. For a script, [`examples/lightpanda.mjs`](examples/lightpanda.mjs) connects Puppeteer to `lightpanda serve`, presents the filed pass, and on a 402 pays from Node and reloads. The browser never holds the key. Nothing in it is Lightpanda-specific; it works against Chrome. +| runtime | how | status | +| --- | --- | --- | +| Node 22, 24 (works on 20.19+) | `node --test` | [![ci](https://github.com/profullstack/x402-client/actions/workflows/ci.yml/badge.svg)](https://github.com/profullstack/x402-client/actions/workflows/ci.yml) | +| Bun | `bun test`, `bun bin/x402.js` | same workflow | +| Deno 2 | `deno test -A`, `deno run -A bin/x402.js` | same workflow | + +Nothing in the package is runtime-specific: base64 goes through `atob`/`btoa` and the text encoders rather than `Buffer`, randomness through `globalThis.crypto`, and the pass file through `node:fs`, which all three provide. The CLI is a plain script: `bunx @profullstack/x402-client pass URL` and `deno run -A npm:@profullstack/x402-client pass URL` both work. ## What is signed @@ -106,6 +222,10 @@ The signer is a port of the one in CoinPay Wallet, which was checked byte for by | `too-expensive` | over `maxUsd`, or a token this client cannot price | | `rejected` | the server refused the proof; `body` has its reason | +## Selling, not buying? + +[@profullstack/x402-gateway](https://github.com/profullstack/x402-gateway) is the other half: one middleware that answers training crawlers with a 402, sells passes by the day, and keeps search crawlers welcome. + ## License MIT diff --git a/examples/axios.mjs b/examples/axios.mjs new file mode 100644 index 0000000..4adf84f --- /dev/null +++ b/examples/axios.mjs @@ -0,0 +1,19 @@ +/** + * axios through a paid site. + * + * npm i axios @profullstack/x402-client + * X402_PRIVATE_KEY=0x… node examples/axios.mjs https://site/page + */ + +import axios from 'axios'; + +import { createClient } from '@profullstack/x402-client'; +import { attachX402 } from '@profullstack/x402-client/axios'; + +const [url = 'https://rssamplifier.com/'] = process.argv.slice(2); + +const http = axios.create({ headers: { 'user-agent': 'MyAgent/1.0 (+https://example.test/bot)' } }); +attachX402(http, createClient({ key: process.env.X402_PRIVATE_KEY, store: 'file', maxUsd: 2 })); + +const res = await http.get(url); +console.log(res.status, String(res.data).slice(0, 500)); diff --git a/examples/lightpanda.mjs b/examples/lightpanda.mjs index 684764d..ef0e364 100644 --- a/examples/lightpanda.mjs +++ b/examples/lightpanda.mjs @@ -3,19 +3,13 @@ * * Lightpanda has no plugin system, and does not need one here: a gateway that * sells crawl time hands back a pass after one payment, and the browser only - * has to present it. So the shape is - * - * 1. pay once, from Node, with this client - * 2. give the pass to the browser as a header - * - * Two ways to do step 2. From the shell, with Lightpanda's own header flag: + * has to present it. From the shell that is one flag: * * lightpanda fetch --http-header "x-crawl-pass: $(x402 pass https://site/crawl)" https://site/page * - * Or from Puppeteer, connected to `lightpanda serve` (or to Chrome — nothing - * below is Lightpanda-specific), which is what this file does. The pass is - * set once per page, and a 402 that still comes through is paid and the page - * reloaded, once. + * From a script, connect Puppeteer to `lightpanda serve` and let the adapter + * make `page.goto` pay. Nothing below is Lightpanda-specific; it is the same + * against Chrome. * * npm i puppeteer-core @profullstack/x402-client * lightpanda serve --host 127.0.0.1 --port 9222 & @@ -25,6 +19,7 @@ import puppeteer from 'puppeteer-core'; import { createClient } from '@profullstack/x402-client'; +import { attachX402 } from '@profullstack/x402-client/puppeteer'; const [url = 'https://rssamplifier.com/'] = process.argv.slice(2); @@ -38,25 +33,11 @@ const client = createClient({ const browser = await puppeteer.connect({ browserWSEndpoint: 'ws://127.0.0.1:9222' }); const page = await browser.newPage(); -/** Present whatever pass is on file for the page's site. */ -async function presentPass(target) { - const pass = client.passFor(target); - if (pass) await page.setExtraHTTPHeaders({ [pass.header]: pass.token }); - return Boolean(pass); -} - -await presentPass(url); -let response = await page.goto(url, { waitUntil: 'networkidle0' }); - -if (response?.status() === 402) { - // The browser was refused. Pay from Node — the browser never sees the key — - // file the pass, hand it to the page, and go once more. - const paid = await client.pay(url); - console.error(`paid ${paid.amountUsd} USD on ${paid.network} from ${paid.payer}; pass good until ${paid.receipt.expires}`); - await presentPass(url); - response = await page.goto(url, { waitUntil: 'networkidle0' }); -} +// page.goto now presents the filed pass, and on a 402 pays from Node, files +// the pass, and navigates again. The browser never holds the key. +attachX402(page, client); +const response = await page.goto(url, { waitUntil: 'networkidle0' }); console.log(`${response?.status()} ${url}`); console.log((await page.content()).slice(0, 500)); diff --git a/examples/playwright.mjs b/examples/playwright.mjs new file mode 100644 index 0000000..6765c93 --- /dev/null +++ b/examples/playwright.mjs @@ -0,0 +1,36 @@ +/** + * Playwright through a paid site, paying as it goes. + * + * npm i playwright @profullstack/x402-client + * X402_PRIVATE_KEY=0x… node examples/playwright.mjs https://site/page + * + * Works the same against Lightpanda: `chromium.connectOverCDP('ws://127.0.0.1:9222')`. + */ + +import { chromium } from 'playwright'; + +import { createClient } from '@profullstack/x402-client'; +import { attachX402 } from '@profullstack/x402-client/playwright'; + +const [url = 'https://rssamplifier.com/'] = process.argv.slice(2); + +const client = createClient({ + key: process.env.X402_PRIVATE_KEY, + store: 'file', // shared with the CLI and every other script on this machine + maxUsd: 2, + userAgent: 'MyAgent/1.0 (+https://example.test/bot)', +}); + +const browser = await chromium.launch(); +const context = await browser.newContext(); + +// Route the whole context: every page it opens presents the pass, and a 402 +// is paid from here — the browser never sees the key. +await attachX402(context, client); + +const page = await context.newPage(); +const response = await page.goto(url, { waitUntil: 'networkidle' }); +console.log(`${response?.status()} ${url}`); +console.log((await page.content()).slice(0, 500)); + +await browser.close(); diff --git a/index.d.ts b/index.d.ts index 8ffae36..982dbbc 100644 --- a/index.d.ts +++ b/index.d.ts @@ -113,6 +113,8 @@ export interface ClientOptions { export interface X402Client { /** The paying address, or null for a client without a key. */ readonly address: string | null; + /** The wallet, or null. Signs and names itself; holds no readable key. */ + readonly wallet: Wallet | null; readonly store: PassStore; /** fetch that presents a filed pass and pays a 402 once. */ fetch(input: string | URL | Request, init?: RequestInit, options?: { pay?: boolean }): Promise; diff --git a/package.json b/package.json index 1cf3d4d..7fdb7b6 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "@profullstack/x402-client", - "version": "0.1.0", + "version": "0.2.0", "type": "module", - "description": "Pay an x402 402 from Node with a local key: sign the EIP-3009 authorization, retry with the proof, keep the pass. A fetch, a CLI, and the header that lets Lightpanda, Puppeteer or curl through a paid site.", + "description": "Pay x402 402s from JavaScript with a local key: a fetch, a CLI, and adapters for axios, got, Puppeteer and Playwright. Signs the EIP-3009 authorization, retries with the proof, keeps the pass. Node, Bun and Deno.", "keywords": [ "x402", "402", @@ -16,26 +16,81 @@ "puppeteer", "headless", "agent", - "micropayment" + "micropayment", + "axios", + "got", + "playwright", + "fetch", + "undici", + "crawlee", + "scraper", + "bun", + "deno", + "paywall", + "bot" ], - "repository": { "type": "git", "url": "git+https://github.com/profullstack/x402-client.git" }, + "repository": { + "type": "git", + "url": "git+https://github.com/profullstack/x402-client.git" + }, "homepage": "https://github.com/profullstack/x402-client#readme", - "bugs": { "url": "https://github.com/profullstack/x402-client/issues" }, + "bugs": { + "url": "https://github.com/profullstack/x402-client/issues" + }, "license": "MIT", "author": "Profullstack, LLC", - "bin": { "x402": "./bin/x402.js" }, + "bin": { + "x402": "./bin/x402.js" + }, "exports": { - ".": { "types": "./index.d.ts", "import": "./src/index.js" } + ".": { + "types": "./index.d.ts", + "import": "./src/index.js" + }, + "./fetch": { + "types": "./src/adapters/fetch.d.ts", + "import": "./src/adapters/fetch.js" + }, + "./axios": { + "types": "./src/adapters/axios.d.ts", + "import": "./src/adapters/axios.js" + }, + "./got": { + "types": "./src/adapters/got.d.ts", + "import": "./src/adapters/got.js" + }, + "./puppeteer": { + "types": "./src/adapters/puppeteer.d.ts", + "import": "./src/adapters/puppeteer.js" + }, + "./playwright": { + "types": "./src/adapters/playwright.d.ts", + "import": "./src/adapters/playwright.js" + } }, "types": "./index.d.ts", - "files": ["src", "bin", "index.d.ts", "README.md", "LICENSE"], - "engines": { "node": ">=20.19" }, + "files": [ + "src", + "bin", + "index.d.ts", + "README.md", + "LICENSE" + ], + "engines": { + "node": ">=20.19" + }, "sideEffects": false, "scripts": { - "test": "node --test test/*.test.js" + "test": "node --test test/*.test.js", + "test:bun": "bun test", + "test:deno": "deno test -A --no-check test/" }, "dependencies": { "@noble/curves": "^2.4.0", "@noble/hashes": "^2.4.0" + }, + "devDependencies": { + "axios": "1.20.0", + "got": "16.0.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b63e177..2fadd7a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,9 +14,19 @@ importers: '@noble/hashes': specifier: ^2.4.0 version: 2.4.0 + devDependencies: + axios: + specifier: 1.20.0 + version: 1.20.0(debug@4.4.3) + got: + specifier: 16.0.0 + version: 16.0.0 packages: + '@keyv/serialize@1.1.1': + resolution: {integrity: sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==} + '@noble/curves@2.4.0': resolution: {integrity: sha512-P4/62zrgfH33CneE3Dn4WhJVA22YUU0eR51wKIan4NVRvwsA0YnPTwWGpNbpuacSujmSFLvyzpyuR30+fbq2Ew==} engines: {node: '>= 20.19.0'} @@ -25,10 +35,389 @@ packages: resolution: {integrity: sha512-X5XaVWZIBCT7HHZGm5I7ZQXDwLG+bGXuSrMQAW+7Zvl87h1kmc1ZB1VSRJcpUfoUrGQp4Fkoxm5kZ+Ms+aW+eA==} engines: {node: '>= 20.19.0'} + '@sec-ant/readable-stream@0.4.1': + resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} + + '@sindresorhus/is@8.1.0': + resolution: {integrity: sha512-2SX/1jW6CIMAiebvVv5ZInoCEuWQmMyBoJXXGC6Vjakjp/fpxP5eHs7/V6WKuPEIbuK06+VpjH+vjLQhr98rDQ==} + engines: {node: '>=22'} + + '@types/http-cache-semantics@4.2.0': + resolution: {integrity: sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==} + + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + axios@1.20.0: + resolution: {integrity: sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==} + + byte-counter@0.1.0: + resolution: {integrity: sha512-jheRLVMeUKrDBjVw2O5+k4EvR4t9wtxHL+bo/LxfkxsVeuGMy3a5SEGgXdAFA4FSzTrU8rQXQIrsZ3oBq5a0pQ==} + engines: {node: '>=20'} + + cacheable-request@13.0.19: + resolution: {integrity: sha512-SVXGH037+Mo1aIMO5B2UcleR43FGjFdN+M8JObSyEoQ2Mn4CODRWx28gN5jiTF0n5ItsgtIZfyargMNs8GX4kg==} + engines: {node: '>=18'} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + chunk-data@0.1.0: + resolution: {integrity: sha512-zFyPtyC0SZ6Zu79b9sOYtXZcgrsXe0RpePrzRyj52hYVFG1+Rk6rBqjjOEk+GNQwc3PIX+86teQMok970pod1g==} + engines: {node: '>=20'} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + decompress-response@10.0.0: + resolution: {integrity: sha512-oj7KWToJuuxlPr7VV0vabvxEIiqNMo+q0NueIiL3XhtwC6FVOX7Hr1c0C4eD0bmf7Zr+S/dSf2xvkH3Ad6sU3Q==} + engines: {node: '>=20'} + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.2: + resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + + follow-redirects@1.16.0: + resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} + engines: {node: '>= 6'} + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + get-stream@9.0.1: + resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} + engines: {node: '>=18'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + got@16.0.0: + resolution: {integrity: sha512-UtzSUebtRHHAUaNB/X37X1LQgHvdSrgXUIWGUBDhLu05CYtUr9svjeN6wmG333nzdJASFUjXMOuRBjZLf8oF1w==} + engines: {node: '>=22'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + + http-cache-semantics@4.2.0: + resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} + + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + + is-stream@4.0.1: + resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==} + engines: {node: '>=18'} + + keyv@5.6.0: + resolution: {integrity: sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==} + + lowercase-keys@3.0.0: + resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + lowercase-keys@4.0.1: + resolution: {integrity: sha512-wI9Nui/L8VfADa/cr/7NQruaASk1k23/Uh1khQ02BCVYiiy8F4AhOGnQzJy3Fl/c44GnYSbZHv8g7EcG3kJ1Qg==} + engines: {node: '>=20'} + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + mimic-response@4.0.0: + resolution: {integrity: sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + normalize-url@8.1.1: + resolution: {integrity: sha512-JYc0DPlpGWB40kH5g07gGTrYuMqV653k3uBKY6uITPWds3M0ov3GaWGp9lbE3Bzngx8+XkfzgvASb9vk9JDFXQ==} + engines: {node: '>=14.16'} + + proxy-from-env@2.1.0: + resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} + engines: {node: '>=10'} + + responselike@4.0.2: + resolution: {integrity: sha512-cGk8IbWEAnaCpdAt1BHzJ3Ahz5ewDJa0KseTsE3qIRMJ3C698W8psM7byCeWVpd/Ha7FUYzuRVzXoKoM6nRUbA==} + engines: {node: '>=20'} + + tagged-tag@1.0.0: + resolution: {integrity: sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==} + engines: {node: '>=20'} + + type-fest@5.9.0: + resolution: {integrity: sha512-yANm3Jr3GiJ1qgJlxGAVxTOIcEOk1rhQHamlXtnrCK7EHP4HeM9OGxtMg/W7HFdrVzw/ZWJKGVIJusVH85sLtw==} + engines: {node: '>=20'} + + uint8array-extras@1.5.0: + resolution: {integrity: sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==} + engines: {node: '>=18'} + snapshots: + '@keyv/serialize@1.1.1': {} + '@noble/curves@2.4.0': dependencies: '@noble/hashes': 2.4.0 '@noble/hashes@2.4.0': {} + + '@sec-ant/readable-stream@0.4.1': {} + + '@sindresorhus/is@8.1.0': {} + + '@types/http-cache-semantics@4.2.0': {} + + agent-base@6.0.2: + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + asynckit@0.4.0: {} + + axios@1.20.0(debug@4.4.3): + dependencies: + follow-redirects: 1.16.0(debug@4.4.3) + form-data: 4.0.6 + https-proxy-agent: 5.0.1 + proxy-from-env: 2.1.0 + transitivePeerDependencies: + - debug + - supports-color + + byte-counter@0.1.0: {} + + cacheable-request@13.0.19: + dependencies: + '@types/http-cache-semantics': 4.2.0 + get-stream: 9.0.1 + http-cache-semantics: 4.2.0 + keyv: 5.6.0 + mimic-response: 4.0.0 + normalize-url: 8.1.1 + responselike: 4.0.2 + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + chunk-data@0.1.0: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + decompress-response@10.0.0: + dependencies: + mimic-response: 4.0.0 + + delayed-stream@1.0.0: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.2: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.4 + + follow-redirects@1.16.0(debug@4.4.3): + optionalDependencies: + debug: 4.4.3 + + form-data@4.0.6: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.4 + mime-types: 2.1.35 + + function-bind@1.1.2: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.2 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.4 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.2 + + get-stream@9.0.1: + dependencies: + '@sec-ant/readable-stream': 0.4.1 + is-stream: 4.0.1 + + gopd@1.2.0: {} + + got@16.0.0: + dependencies: + '@sindresorhus/is': 8.1.0 + byte-counter: 0.1.0 + cacheable-request: 13.0.19 + chunk-data: 0.1.0 + decompress-response: 10.0.0 + keyv: 5.6.0 + lowercase-keys: 4.0.1 + responselike: 4.0.2 + type-fest: 5.9.0 + uint8array-extras: 1.5.0 + + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + + http-cache-semantics@4.2.0: {} + + https-proxy-agent@5.0.1: + dependencies: + agent-base: 6.0.2 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + is-stream@4.0.1: {} + + keyv@5.6.0: + dependencies: + '@keyv/serialize': 1.1.1 + + lowercase-keys@3.0.0: {} + + lowercase-keys@4.0.1: {} + + math-intrinsics@1.1.0: {} + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + mimic-response@4.0.0: {} + + ms@2.1.3: {} + + normalize-url@8.1.1: {} + + proxy-from-env@2.1.0: {} + + responselike@4.0.2: + dependencies: + lowercase-keys: 3.0.0 + + tagged-tag@1.0.0: {} + + type-fest@5.9.0: + dependencies: + tagged-tag: 1.0.0 + + uint8array-extras@1.5.0: {} diff --git a/src/adapters/axios.d.ts b/src/adapters/axios.d.ts new file mode 100644 index 0000000..82c35a4 --- /dev/null +++ b/src/adapters/axios.d.ts @@ -0,0 +1,13 @@ +import type { X402Client } from '../../index.js'; + +/** Minimal shape of an axios instance: what the adapter touches. */ +export interface AxiosLike { + request: (config: any) => Promise; + interceptors: { + request: { use: (onFulfilled: (config: any) => any) => number; eject: (id: number) => void }; + response: { use: (onFulfilled?: (response: any) => any, onRejected?: (error: any) => any) => number; eject: (id: number) => void }; + }; +} + +/** Present filed passes and pay 402s on an axios instance. Returns a detach function. */ +export function attachX402(axios: AxiosLike, client: X402Client): () => void; diff --git a/src/adapters/axios.js b/src/adapters/axios.js new file mode 100644 index 0000000..c61cbfa --- /dev/null +++ b/src/adapters/axios.js @@ -0,0 +1,116 @@ +/** + * axios, paying. + * + * Two interceptors: one puts the filed pass on every request to a site that + * has one, the other catches a 402, pays it, and replays the request with + * the pass. Nothing about the caller's code changes — `axios.get(url)` on a + * paid site returns the page. + * + * import axios from 'axios'; + * import { attachX402 } from '@profullstack/x402-client/axios'; + * attachX402(axios, createClient({ key })); + * + * The payment itself goes through the client's own fetch rather than through + * axios, so an axios proxy or agent does not apply to the one request that + * carries the proof. Everything else — the offer, the replay — is axios. + */ + +import { isOffer, parseOfferHeader } from '../x402.js'; +import { acceptingJson } from './util.js'; + +const PAID = Symbol.for('@profullstack/x402-client/paid'); + +/** + * @param {import('axios').AxiosInstance} axios the default export or an `axios.create()` + * @param {import('../../index.d.ts').X402Client} client + * @returns {() => void} detach + */ +export function attachX402(axios, client) { + const onRequest = (config) => { + const url = urlOf(config); + if (!url) return config; + const pass = client.passFor(url); + if (pass) config.headers = withHeader(config.headers, pass.header, pass.token); + return config; + }; + + const settle = async (response) => { + const config = response?.config; + if (!config || config[PAID]) return null; + const url = urlOf(config); + if (!url) return null; + // The offer is in the 402 when the request asked for JSON. A scraper + // wearing a browser's Accept header gets the HTML sales page instead, so + // when there is no offer here the client asks the URL again as JSON. A + // 402 that is not x402 fails that with `no-offer`, and the caller gets + // the error axios raised. + const offer = isOffer(response.data) ? response.data : parseOfferHeader(headerOf(response.headers, 'payment-required')); + + let paid; + try { + paid = await client.pay(url, { + offer: offer ?? undefined, + init: { + method: String(config.method ?? 'get').toUpperCase(), + headers: acceptingJson(plain(config.headers)), + }, + }); + } catch (err) { + if (err?.code === 'no-offer') return null; + throw err; + } + + if (!paid.receipt.pass) { + // A plain x402 resource: the paid answer is the resource, in axios' shape. + return { ...response, status: paid.status, statusText: 'OK', data: paid.body }; + } + const headers = withHeader(plain(config.headers), paid.receipt.header, paid.receipt.pass); + return axios.request({ ...config, headers, [PAID]: true }); + }; + + // A 402 arrives as a rejection under axios' default validateStatus, and + // as a fulfilment when the caller widened it. Both are handled. + const onFulfilled = async (response) => (response?.status === 402 ? (await settle(response)) ?? response : response); + const onRejected = async (error) => { + if (error?.response?.status !== 402) throw error; + const settled = await settle(error.response); + if (!settled) throw error; + return settled; + }; + + const req = axios.interceptors.request.use(onRequest); + const res = axios.interceptors.response.use(onFulfilled, onRejected); + return () => { + axios.interceptors.request.eject(req); + axios.interceptors.response.eject(res); + }; +} + +function urlOf(config) { + try { + return config.baseURL ? new URL(String(config.url ?? ''), config.baseURL).href : new URL(String(config.url)).href; + } catch { + return null; + } +} + +/** Headers as a plain object, whether axios gave an AxiosHeaders or a literal. */ +function plain(headers) { + if (!headers) return {}; + if (typeof headers.toJSON === 'function') return { ...headers.toJSON() }; + return { ...headers }; +} + +function withHeader(headers, name, value) { + if (headers && typeof headers.set === 'function') { + headers.set(name, value); + return headers; + } + return { ...(headers ?? {}), [name]: value }; +} + +function headerOf(headers, name) { + if (!headers) return null; + if (typeof headers.get === 'function') return headers.get(name) ?? null; + return headers[name] ?? headers[name.toLowerCase()] ?? null; +} diff --git a/src/adapters/fetch.d.ts b/src/adapters/fetch.d.ts new file mode 100644 index 0000000..317ee9a --- /dev/null +++ b/src/adapters/fetch.d.ts @@ -0,0 +1,6 @@ +import type { ClientOptions, X402Client } from '../../index.js'; + +/** A fetch that presents a filed pass and pays a 402 once, over the fetch you give it. */ +export function wrapFetch(fetchImpl: typeof fetch, options?: ClientOptions & { client?: X402Client }): typeof fetch; +/** The global fetch, paying. */ +export function x402Fetch(options?: ClientOptions): typeof fetch; diff --git a/src/adapters/fetch.js b/src/adapters/fetch.js new file mode 100644 index 0000000..50b86f4 --- /dev/null +++ b/src/adapters/fetch.js @@ -0,0 +1,43 @@ +/** + * A paying `fetch`, for anything that takes a fetch. + * + * Most of the JavaScript crawling world is a `fetch` under a different name: + * the globals in Node, Bun, Deno and the browser; `undici.fetch`; + * `node-fetch`; and every library with a `fetch` option — ky, ofetch, + * openai's and Anthropic's SDKs, LangChain loaders, Crawlee's `sendRequest`. + * Wrap the one you have and hand the result to them. + */ + +import { createClient } from '../client.js'; + +/** + * A `fetch` that presents a filed pass and pays a 402 once. + * + * Same signature as the function it wraps. `options` are + * {@link createClient}'s; pass `client` to share one wallet and pass file + * across several wrapped fetches. + * + * @param {typeof fetch} fetchImpl the fetch to make requests with + * @param {import('../../index.d.ts').ClientOptions & { client?: import('../../index.d.ts').X402Client }} [options] + * @returns {typeof fetch} + */ +export function wrapFetch(fetchImpl, options = {}) { + const { client: given, ...rest } = options; + // A shared client keeps its own fetch; the wrapped one is used only when + // this wrapper is the client's whole reason to exist. + const client = given ?? createClient({ ...rest, fetch: fetchImpl }); + const paying = given + ? createClient({ ...rest, wallet: given.wallet, store: given.store, fetch: fetchImpl }) + : client; + return (input, init) => paying.fetch(input, init); +} + +/** + * The global fetch, paying. `const fetch = x402Fetch({ key })`. + * + * @param {import('../../index.d.ts').ClientOptions} [options] + * @returns {typeof fetch} + */ +export function x402Fetch(options = {}) { + return wrapFetch(globalThis.fetch, options); +} diff --git a/src/adapters/got.d.ts b/src/adapters/got.d.ts new file mode 100644 index 0000000..4e64a34 --- /dev/null +++ b/src/adapters/got.d.ts @@ -0,0 +1,9 @@ +import type { X402Client } from '../../index.js'; + +/** got hooks that present filed passes and pay 402s; spread into `got.extend()`. */ +export function x402Hooks(client: X402Client): { + hooks: { + beforeRequest: Array<(options: any) => void>; + afterResponse: Array<(response: any, retryWithMergedOptions: (options: any) => any) => any>; + }; +}; diff --git a/src/adapters/got.js b/src/adapters/got.js new file mode 100644 index 0000000..2a1e0de --- /dev/null +++ b/src/adapters/got.js @@ -0,0 +1,88 @@ +/** + * got, paying. + * + * got's hooks are the right shape for this: `beforeRequest` presents the + * filed pass, `afterResponse` sees the 402, pays it, and asks got to retry + * the same request with the pass merged in. Spread the result into + * `got.extend()` and the crawler pays as it goes. + * + * import got from 'got'; + * import { x402Hooks } from '@profullstack/x402-client/got'; + * const paying = got.extend(x402Hooks(client)); + * + * Crawlee's `got-scraping` is got underneath and takes the same hooks. + */ + +import { isOffer, parseOfferHeader } from '../x402.js'; +import { acceptingJson } from './util.js'; + +/** + * @param {import('../../index.d.ts').X402Client} client + * @returns {{ hooks: { beforeRequest: Function[], afterResponse: Function[] } }} + */ +export function x402Hooks(client) { + return { + hooks: { + beforeRequest: [ + (options) => { + const url = hrefOf(options.url); + if (!url) return; + const pass = client.passFor(url); + if (pass) options.headers[pass.header] = pass.token; + }, + ], + afterResponse: [ + async (response, retryWithMergedOptions) => { + if (response.statusCode !== 402) return response; + const options = response.request?.options; + if (options?.context?.x402Paid) return response; + + const url = response.url ?? hrefOf(options?.url); + if (!url) return response; + + let json = null; + try { + json = JSON.parse(String(response.body ?? '')); + } catch { + json = null; + } + // The offer is in the 402 when the request asked for JSON. A scraper + // wearing a browser's Accept header gets the HTML sales page, so + // when there is no offer here the client asks the URL again as + // JSON. A 402 that is not x402 fails that with `no-offer`, and the + // 402 is returned as it came. + const offer = isOffer(json) ? json : parseOfferHeader(response.headers?.['payment-required']); + + let paid; + try { + paid = await client.pay(url, { + offer: offer ?? undefined, + init: { method: options?.method ?? 'GET', headers: acceptingJson(options?.headers) }, + }); + } catch (err) { + if (err?.code === 'no-offer') return response; + throw err; + } + + if (!paid.receipt.pass) { + // A plain x402 resource: the paid answer is the resource. + response.statusCode = paid.status; + response.body = typeof paid.body === 'string' ? paid.body : JSON.stringify(paid.body); + return response; + } + + return retryWithMergedOptions({ + headers: { [paid.receipt.header]: paid.receipt.pass }, + context: { ...(options?.context ?? {}), x402Paid: true }, + }); + }, + ], + }, + }; +} + +function hrefOf(url) { + if (!url) return null; + if (typeof url === 'string') return url; + return url.href ?? String(url); +} diff --git a/src/adapters/playwright.d.ts b/src/adapters/playwright.d.ts new file mode 100644 index 0000000..9238cd1 --- /dev/null +++ b/src/adapters/playwright.d.ts @@ -0,0 +1,22 @@ +import type { X402Client } from '../../index.js'; + +/** What the adapter needs of a Playwright Page or BrowserContext. */ +export interface Routable { + route: (pattern: string | RegExp, handler: (route: any, request: any) => Promise | void) => Promise; + unroute: (pattern: string | RegExp, handler?: (route: any, request: any) => Promise | void) => Promise; +} + +export interface Navigable { + goto: (url: string, options?: any) => Promise; + setExtraHTTPHeaders: (headers: Record) => Promise; +} + +/** Route document, XHR and fetch requests through the client: present the pass, pay a 402 once, replay. Returns a detach function. */ +export function attachX402( + target: Routable, + client: X402Client, + options?: { pattern?: string | RegExp; resourceTypes?: string[] }, +): Promise<() => Promise>; + +/** Navigate, paying a 402 once via `setExtraHTTPHeaders`. */ +export function gotoPaid(page: Navigable, url: string, client: X402Client, options?: any): Promise; diff --git a/src/adapters/playwright.js b/src/adapters/playwright.js new file mode 100644 index 0000000..4cab4ee --- /dev/null +++ b/src/adapters/playwright.js @@ -0,0 +1,122 @@ +/** + * Playwright, paying. + * + * Playwright's routing can make the request itself, look at the answer, and + * decide what the page sees — which is exactly the shape of a 402 handshake. + * `attachX402` routes the page (or the whole context): every document, XHR + * and fetch request goes out with the filed pass, a 402 is paid once and the + * request made again with the pass, and the page receives the paid answer + * as if the site had never asked. Other resources go through untouched but + * for the pass header. + * + * import { chromium } from 'playwright'; + * import { attachX402 } from '@profullstack/x402-client/playwright'; + * const page = await browser.newPage(); + * await attachX402(page, client); + * await page.goto('https://site/paid/page'); + * + * The browser never holds the key: the proof is signed and sent from Node + * by the client, and only the pass reaches the browser. Works the same + * against Lightpanda's CDP endpoint, since routing is Playwright's, not the + * browser's. + */ + +import { isOffer, parseOfferHeader } from '../x402.js'; +import { acceptingJson } from './util.js'; + +const DEFAULT_TYPES = ['document', 'xhr', 'fetch']; + +/** + * @param {{ route: Function, unroute: Function }} target a Page or a BrowserContext + * @param {import('../../index.d.ts').X402Client} client + * @param {{ pattern?: string|RegExp, resourceTypes?: string[] }} [options] + * @returns {Promise<() => Promise>} detach + */ +export async function attachX402(target, client, { pattern = '**/*', resourceTypes = DEFAULT_TYPES } = {}) { + const types = new Set(resourceTypes); + + const handler = async (route, request) => { + const url = request.url(); + const headers = { ...request.headers() }; + const filed = client.passFor(url); + if (filed) headers[filed.header] = filed.token; + + if (!types.has(request.resourceType())) return route.continue({ headers }); + + const first = await route.fetch({ headers }); + if (first.status() !== 402) return route.fulfill({ response: first }); + + const text = await first.text(); + let json = null; + try { + json = JSON.parse(text); + } catch { + json = null; + } + // A browser asks for text/html, and a gateway answers a browser's 402 + // with its sales page rather than the JSON offer. So the offer may not + // be in what came back; if it is not, the client asks the URL again as + // JSON and pays what that says. A 402 that is not x402 at all fails that + // with `no-offer`, and the page sees the 402 the site sent. + const offer = isOffer(json) ? json : parseOfferHeader(first.headers()['payment-required']); + + let paid; + try { + paid = await client.pay(url, { + offer: offer ?? undefined, + init: { method: request.method(), headers: acceptingJson(headers) }, + }); + } catch { + // Not x402, refused, too expensive, no key: the page sees the 402. + return route.fulfill({ response: first, body: text }); + } + + if (!paid.receipt.pass) { + const body = typeof paid.body === 'string' ? paid.body : JSON.stringify(paid.body); + return route.fulfill({ + status: paid.status, + headers: { 'content-type': typeof paid.body === 'string' ? 'text/plain; charset=utf-8' : 'application/json' }, + body, + }); + } + + headers[paid.receipt.header] = paid.receipt.pass; + const again = await route.fetch({ headers }); + return route.fulfill({ response: again }); + }; + + await target.route(pattern, handler); + return async () => { + await target.unroute(pattern, handler); + }; +} + +/** + * Navigate, paying a 402 once. For a script that would rather not route. + * + * Presents the filed pass through `setExtraHTTPHeaders` — which Playwright + * sends on every request from the page, cross-origin included, so prefer + * {@link attachX402} when the crawl leaves the site. + * + * @param {{ goto: Function, setExtraHTTPHeaders: Function }} page + * @param {string} url + * @param {import('../../index.d.ts').X402Client} client + * @param {object} [options] forwarded to `page.goto` + */ +export async function gotoPaid(page, url, client, options) { + const present = async () => { + const pass = client.passFor(url); + if (pass) await page.setExtraHTTPHeaders({ [pass.header]: pass.token }); + return Boolean(pass); + }; + await present(); + let response = await page.goto(url, options); + if (response && response.status() === 402) { + const paid = await client.pay(url); + if (paid.receipt.pass) { + await present(); + response = await page.goto(url, options); + } + } + return response; +} diff --git a/src/adapters/puppeteer.d.ts b/src/adapters/puppeteer.d.ts new file mode 100644 index 0000000..e856a10 --- /dev/null +++ b/src/adapters/puppeteer.d.ts @@ -0,0 +1,13 @@ +import type { X402Client } from '../../index.js'; + +export interface Navigable { + goto: (url: string, options?: any) => Promise; + setExtraHTTPHeaders: (headers: Record) => Promise; +} + +/** Present the filed pass for a URL's site on every request the page makes. */ +export function applyPass(page: Navigable, url: string, client: X402Client): Promise; +/** Navigate, paying a 402 once. */ +export function gotoPaid(page: Navigable, url: string, client: X402Client, options?: any): Promise; +/** Make `page.goto` pay. Returns a function that restores the original. */ +export function attachX402(page: Navigable, client: X402Client): () => void; diff --git a/src/adapters/puppeteer.js b/src/adapters/puppeteer.js new file mode 100644 index 0000000..b660f55 --- /dev/null +++ b/src/adapters/puppeteer.js @@ -0,0 +1,76 @@ +/** + * Puppeteer, paying. + * + * Puppeteer's request interception can stop a request and change it, but it + * cannot look at a response and then retry — so the 402 handshake lives at + * the navigation instead. `gotoPaid` presents the filed pass, navigates, + * and on a 402 pays from Node, presents the pass, and navigates once more. + * `attachX402` makes `page.goto` do that, so an existing script pays with + * one added line. + * + * import puppeteer from 'puppeteer'; + * import { attachX402 } from '@profullstack/x402-client/puppeteer'; + * const page = await browser.newPage(); + * attachX402(page, client); + * await page.goto('https://site/paid/page'); + * + * Same against Lightpanda: `puppeteer.connect({ browserWSEndpoint })` and + * everything below is unchanged. The key never enters the browser. + * + * `setExtraHTTPHeaders` is page-wide, so the pass goes out on every request + * the page makes, cross-origin included. A pass is a bearer token for one + * site and nothing else, and it expires; if the crawl leaves the site, + * clear it with `page.setExtraHTTPHeaders({})` or use Playwright's routing. + */ + +/** + * Present the filed pass for a URL's site on every request the page makes. + * + * @param {{ setExtraHTTPHeaders: Function }} page + * @param {string} url + * @param {import('../../index.d.ts').X402Client} client + * @returns {Promise} whether there was one + */ +export async function applyPass(page, url, client) { + const pass = client.passFor(url); + if (!pass) return false; + await page.setExtraHTTPHeaders({ [pass.header]: pass.token }); + return true; +} + +/** + * Navigate, paying a 402 once. + * + * @param {{ goto: Function, setExtraHTTPHeaders: Function }} page + * @param {string} url + * @param {import('../../index.d.ts').X402Client} client + * @param {object} [options] forwarded to `page.goto` + */ +export async function gotoPaid(page, url, client, options) { + await applyPass(page, url, client); + let response = await page.goto(url, options); + if (response && response.status() === 402) { + const paid = await client.pay(url); + if (paid.receipt.pass) { + await applyPass(page, url, client); + response = await page.goto(url, options); + } + } + return response; +} + +/** + * Make `page.goto` pay. Returns a function that puts the original back. + * + * @param {{ goto: Function, setExtraHTTPHeaders: Function }} page + * @param {import('../../index.d.ts').X402Client} client + * @returns {() => void} + */ +export function attachX402(page, client) { + const original = page.goto; + const plain = { goto: original.bind(page), setExtraHTTPHeaders: page.setExtraHTTPHeaders.bind(page) }; + page.goto = (url, options) => gotoPaid(plain, String(url), client, options); + return () => { + page.goto = original; + }; +} diff --git a/src/adapters/util.js b/src/adapters/util.js new file mode 100644 index 0000000..2dc5024 --- /dev/null +++ b/src/adapters/util.js @@ -0,0 +1,21 @@ +/** + * Ask for JSON, whatever the caller was asking for. + * + * A gateway answers a 402 with its HTML sales page when the request accepts + * text/html, and with the JSON offer otherwise. When an adapter has to go + * back for the offer it must replace the caller's Accept, not add to it: + * header names are case-insensitive, `{ Accept: 'text/html', accept: + * 'application/json' }` becomes one header with both values, and the + * gateway sees the text/html and serves the page again. + * + * @param {Record|null|undefined} headers + * @returns {Record} + */ +export function acceptingJson(headers) { + const out = {}; + for (const [name, value] of Object.entries(headers ?? {})) { + if (name.toLowerCase() !== 'accept' && value !== undefined && value !== null) out[name] = String(value); + } + out.accept = 'application/json'; + return out; +} diff --git a/src/client.js b/src/client.js index db2a8a0..9b88c5f 100644 --- a/src/client.js +++ b/src/client.js @@ -222,6 +222,9 @@ export function createClient(options = {}) { get address() { return wallet?.address ?? null; }, + // The wallet signs and names itself and does nothing else; sharing it + // between clients (one per fetch implementation, say) leaks no key. + wallet, store, fetch: paidFetch, pay, diff --git a/src/passes.js b/src/passes.js index 46e19f5..00bcca7 100644 --- a/src/passes.js +++ b/src/passes.js @@ -66,7 +66,7 @@ export function memoryStore() { * under `~/.config`. */ export function defaultPassPath() { - const base = process.env.XDG_CONFIG_HOME || join(homedir(), '.config'); + const base = globalThis.process?.env?.XDG_CONFIG_HOME || join(homedir(), '.config'); return join(base, 'x402-client', 'passes.json'); } diff --git a/src/x402.js b/src/x402.js index 302728c..5c5a131 100644 --- a/src/x402.js +++ b/src/x402.js @@ -41,10 +41,45 @@ export function evmChainId(network) { return match ? Number(match[1]) : null; } -/** base64 (standard or url-safe) -> utf8. */ +/** + * base64 (standard or url-safe) -> utf8, and back. + * + * On `atob`/`btoa` and the text encoders rather than `Buffer`, because the + * four places this runs — Node, Bun, Deno and a browser — agree on those and + * not on `Buffer`. + */ function fromBase64(s) { const clean = String(s).trim().replace(/-/g, '+').replace(/_/g, '/'); - return Buffer.from(clean, 'base64').toString('utf8'); + const bin = atob(clean); + const bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + return new TextDecoder().decode(bytes); +} + +function toBase64(text) { + const bytes = new TextEncoder().encode(text); + let bin = ''; + for (const b of bytes) bin += String.fromCharCode(b); + return btoa(bin); +} + +/** + * The offer carried in a `PAYMENT-REQUIRED` header, or null. + * + * Shared by the adapters, which each see a response in their own library's + * shape and only have the raw header value in common. + * + * @param {string|null|undefined} header + * @returns {object|null} + */ +export function parseOfferHeader(header) { + if (!header) return null; + try { + const parsed = JSON.parse(fromBase64(header)); + return isOffer(parsed) ? parsed : null; + } catch { + return null; + } } /** @@ -66,15 +101,8 @@ export async function readOffer(response) { body = text; } - const header = response.headers.get('payment-required'); - if (header) { - try { - const parsed = JSON.parse(fromBase64(header)); - if (isOffer(parsed)) return { offer: parsed, body }; - } catch { - /* fall through to the body */ - } - } + const fromHeader = parseOfferHeader(response.headers.get('payment-required')); + if (fromHeader) return { offer: fromHeader, body }; return { offer: isOffer(body) ? body : null, body }; } @@ -232,7 +260,7 @@ export function signPayment(entry, wallet, options = {}) { /** base64 of a payment, as the proof headers carry it. */ export function encodePaymentHeader(payment) { - return Buffer.from(JSON.stringify(payment), 'utf8').toString('base64'); + return toBase64(JSON.stringify(payment)); } /** Inverse of {@link encodePaymentHeader}. Null when the header is not base64 JSON. */ diff --git a/test/adapters.test.js b/test/adapters.test.js new file mode 100644 index 0000000..888927f --- /dev/null +++ b/test/adapters.test.js @@ -0,0 +1,401 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import axios from 'axios'; +import got from 'got'; + +import { createClient, memoryStore } from '../src/index.js'; +import { attachX402 as attachAxios } from '../src/adapters/axios.js'; +import { wrapFetch, x402Fetch } from '../src/adapters/fetch.js'; +import { x402Hooks } from '../src/adapters/got.js'; +import { attachX402 as attachPlaywright, gotoPaid as playwrightGoto } from '../src/adapters/playwright.js'; +import { applyPass, attachX402 as attachPuppeteer, gotoPaid as puppeteerGoto } from '../src/adapters/puppeteer.js'; +import { KEY_ONE, startGateway } from './helpers.js'; + +// ---------------------------------------------------------------- fetch + +test('wrapFetch pays through the fetch it is given, and x402Fetch through the global', async () => { + const gateway = await startGateway(); + try { + let calls = 0; + const counting = (input, init) => { + calls++; + return globalThis.fetch(input, init); + }; + const paying = wrapFetch(counting, { key: KEY_ONE }); + const res = await paying(`${gateway.url}/page`); + assert.equal(res.status, 200); + assert.equal(await res.text(), '

the site

'); + assert.equal(gateway.state.payments.length, 1); + assert.equal(calls, 3, 'the 402, the proof, and the page with the pass'); + + const again = await paying(`${gateway.url}/other`); + assert.equal(again.status, 200); + assert.equal(gateway.state.payments.length, 1); + + const global = x402Fetch({ key: KEY_ONE, store: memoryStore() }); + assert.equal((await global(`${gateway.url}/page`)).status, 200); + assert.equal(gateway.state.payments.length, 2); + } finally { + await gateway.close(); + } +}); + +test('wrapFetch can share a client, so two fetches share one pass file', async () => { + const gateway = await startGateway(); + try { + const client = createClient({ key: KEY_ONE }); + const a = wrapFetch(globalThis.fetch, { client }); + const b = wrapFetch(globalThis.fetch, { client }); + assert.equal((await a(`${gateway.url}/one`)).status, 200); + assert.equal((await b(`${gateway.url}/two`)).status, 200); + assert.equal(gateway.state.payments.length, 1); + assert.ok(client.passFor(gateway.url)); + } finally { + await gateway.close(); + } +}); + +// ---------------------------------------------------------------- axios + +test('axios: a 402 is paid and replayed, and the pass rides on later requests', async () => { + const gateway = await startGateway(); + try { + const instance = axios.create(); + const detach = attachAxios(instance, createClient({ key: KEY_ONE })); + + const res = await instance.get(`${gateway.url}/page`); + assert.equal(res.status, 200); + assert.equal(res.data, '

the site

'); + assert.equal(gateway.state.payments.length, 1); + + const again = await instance.get(`${gateway.url}/other`); + assert.equal(again.status, 200); + assert.equal(gateway.state.payments.length, 1); + assert.match(gateway.state.hits.at(-1).headers['x-crawl-pass'], /^cp_test1/); + + detach(); + await assert.rejects(instance.get(`${gateway.url}/fresh`), (err) => err.response?.status === 402 || err.status === 402); + assert.equal(gateway.state.payments.length, 1, 'detached: nothing pays'); + } finally { + await gateway.close(); + } +}); + +test('axios: relative URLs resolve against baseURL, and a widened validateStatus is handled too', async () => { + const gateway = await startGateway(); + try { + const instance = axios.create({ baseURL: gateway.url, validateStatus: () => true }); + attachAxios(instance, createClient({ key: KEY_ONE })); + const res = await instance.get('/page'); + assert.equal(res.status, 200); + assert.equal(res.data, '

the site

'); + assert.equal(gateway.state.payments.length, 1); + } finally { + await gateway.close(); + } +}); + +test('axios: a refused payment surfaces as the original 402 error, paid exactly once', async () => { + const gateway = await startGateway({ mode: 'reject' }); + try { + const instance = axios.create(); + attachAxios(instance, createClient({ key: KEY_ONE })); + await assert.rejects(instance.get(`${gateway.url}/page`), (err) => { + assert.equal(err.name, 'X402Error'); + assert.equal(err.code, 'rejected'); + return true; + }); + assert.equal(gateway.state.payments.length, 1); + } finally { + await gateway.close(); + } +}); + +test('axios: a plain x402 resource comes back as data', async () => { + const gateway = await startGateway({ mode: 'resource' }); + try { + const instance = axios.create(); + attachAxios(instance, createClient({ key: KEY_ONE })); + const res = await instance.get(`${gateway.url}/premium`); + assert.equal(res.status, 200); + assert.equal(res.data, 'the premium thing'); + } finally { + await gateway.close(); + } +}); + +// ---------------------------------------------------------------- got + +test('got: hooks present the pass and retry a 402 with it', async () => { + const gateway = await startGateway(); + try { + const paying = got.extend(x402Hooks(createClient({ key: KEY_ONE }))); + const res = await paying(`${gateway.url}/page`); + assert.equal(res.statusCode, 200); + assert.equal(res.body, '

the site

'); + assert.equal(gateway.state.payments.length, 1); + + const again = await paying(`${gateway.url}/other`); + assert.equal(again.statusCode, 200); + assert.equal(gateway.state.payments.length, 1); + assert.match(gateway.state.hits.at(-1).headers['x-crawl-pass'], /^cp_test1/); + } finally { + await gateway.close(); + } +}); + +test('got: a plain x402 resource is returned as the body', async () => { + const gateway = await startGateway({ mode: 'resource' }); + try { + const paying = got.extend(x402Hooks(createClient({ key: KEY_ONE }))); + const res = await paying(`${gateway.url}/premium`); + assert.equal(res.statusCode, 200); + assert.equal(res.body, 'the premium thing'); + } finally { + await gateway.close(); + } +}); + +// ---------------------------------------------------------------- playwright + +/** + * A Playwright-shaped route and request over a real HTTP request, so the + * adapter's handling of `route.fetch` / `route.fulfill` / `route.continue` + * is exercised without a browser. + */ +function fakePlaywright() { + const routes = []; + const fulfilled = []; + const continued = []; + const target = { + route: async (pattern, handler) => { + routes.push({ pattern, handler }); + }, + unroute: async (pattern, handler) => { + const i = routes.findIndex((r) => r.pattern === pattern && r.handler === handler); + if (i >= 0) routes.splice(i, 1); + }, + }; + const request = (url, { method = 'GET', resourceType = 'document', headers = {} } = {}) => ({ + url: () => url, + method: () => method, + resourceType: () => resourceType, + headers: () => ({ ...headers }), + }); + const route = (req) => ({ + fetch: async ({ headers } = {}) => { + const res = await globalThis.fetch(req.url(), { method: req.method(), headers }); + const text = await res.text(); + return { + status: () => res.status, + headers: () => Object.fromEntries(res.headers), + text: async () => text, + }; + }, + fulfill: async (args) => { + fulfilled.push(args); + }, + continue: async (args) => { + continued.push(args); + }, + }); + const drive = async (req) => { + for (const r of routes) await r.handler(route(req), req); + }; + return { target, request, drive, fulfilled, continued, routes }; +} + +test('playwright: a routed document request pays once and is fulfilled with the page', async () => { + const gateway = await startGateway(); + try { + const client = createClient({ key: KEY_ONE }); + const fake = fakePlaywright(); + const detach = await attachPlaywright(fake.target, client); + assert.equal(fake.routes.length, 1); + + await fake.drive(fake.request(`${gateway.url}/page`)); + assert.equal(fake.fulfilled.length, 1); + assert.equal(fake.fulfilled[0].response.status(), 200); + assert.equal(await fake.fulfilled[0].response.text(), '

the site

'); + assert.equal(gateway.state.payments.length, 1); + + // Second document: pass presented, nothing paid. + await fake.drive(fake.request(`${gateway.url}/other`)); + assert.equal(fake.fulfilled[1].response.status(), 200); + assert.equal(gateway.state.payments.length, 1); + + // An image is not fetched by us; it continues with the pass on it. + await fake.drive(fake.request(`${gateway.url}/logo.png`, { resourceType: 'image' })); + assert.equal(fake.continued.length, 1); + assert.match(fake.continued[0].headers['x-crawl-pass'], /^cp_test1/); + + await detach(); + assert.equal(fake.routes.length, 0); + } finally { + await gateway.close(); + } +}); + +test('playwright: a 402 the client will not pay reaches the page as the 402', async () => { + const gateway = await startGateway(); + try { + const fake = fakePlaywright(); + await attachPlaywright(fake.target, createClient({ key: KEY_ONE, maxUsd: 0.1 })); + await fake.drive(fake.request(`${gateway.url}/page`)); + assert.equal(fake.fulfilled[0].response.status(), 402); + assert.match(fake.fulfilled[0].body, /accepts/); + assert.equal(gateway.state.payments.length, 0); + } finally { + await gateway.close(); + } +}); + +test('playwright: gotoPaid navigates, pays, presents and navigates again', async () => { + const gateway = await startGateway(); + try { + const client = createClient({ key: KEY_ONE }); + let extra = {}; + const page = { + setExtraHTTPHeaders: async (h) => { + extra = h; + }, + goto: async (url) => { + const res = await globalThis.fetch(url, { headers: extra }); + return { status: () => res.status }; + }, + }; + const res = await playwrightGoto(page, `${gateway.url}/page`, client); + assert.equal(res.status(), 200); + assert.equal(gateway.state.payments.length, 1); + assert.match(extra['x-crawl-pass'], /^cp_test1/); + } finally { + await gateway.close(); + } +}); + +test('playwright: a browser navigation gets the HTML sales page, and is still paid', async () => { + // A real browser sends Accept: text/html, and a gateway answers that with + // its sales page rather than the JSON offer. Found with real Playwright on + // Chrome: the first version of this adapter handed the page the 402. + const gateway = await startGateway(); + try { + const fake = fakePlaywright(); + await attachPlaywright(fake.target, createClient({ key: KEY_ONE })); + await fake.drive(fake.request(`${gateway.url}/page`, { headers: { accept: 'text/html,application/xhtml+xml,*/*;q=0.8' } })); + assert.equal(fake.fulfilled[0].response.status(), 200); + assert.equal(await fake.fulfilled[0].response.text(), '

the site

'); + assert.equal(gateway.state.payments.length, 1); + } finally { + await gateway.close(); + } +}); + +test('playwright: a 402 that is not x402 reaches the page as itself', async () => { + const gateway = await startGateway({ mode: 'bare-402' }); + try { + const fake = fakePlaywright(); + await attachPlaywright(fake.target, createClient({ key: KEY_ONE })); + await fake.drive(fake.request(`${gateway.url}/page`)); + assert.equal(fake.fulfilled[0].response.status(), 402); + assert.equal(fake.fulfilled[0].body, 'Payment required, but not like that.'); + assert.equal(gateway.state.payments.length, 0); + } finally { + await gateway.close(); + } +}); + +test('axios and got: a scraper wearing a browser Accept header is still paid', async () => { + const gateway = await startGateway(); + try { + const instance = axios.create({ headers: { accept: 'text/html,*/*;q=0.8' } }); + attachAxios(instance, createClient({ key: KEY_ONE })); + const res = await instance.get(`${gateway.url}/page`); + assert.equal(res.status, 200); + assert.equal(res.data, '

the site

'); + assert.equal(gateway.state.payments.length, 1); + + const paying = got.extend({ headers: { accept: 'text/html,*/*;q=0.8' }, ...x402Hooks(createClient({ key: KEY_ONE })) }); + const out = await paying(`${gateway.url}/other`); + assert.equal(out.statusCode, 200); + assert.equal(gateway.state.payments.length, 2); + } finally { + await gateway.close(); + } +}); + +test('axios and got: a 402 that is not x402 is left alone', async () => { + const gateway = await startGateway({ mode: 'bare-402' }); + try { + const instance = axios.create(); + attachAxios(instance, createClient({ key: KEY_ONE })); + await assert.rejects(instance.get(`${gateway.url}/page`), (err) => err.response?.status === 402); + + const paying = got.extend({ throwHttpErrors: false, ...x402Hooks(createClient({ key: KEY_ONE })) }); + const out = await paying(`${gateway.url}/page`); + assert.equal(out.statusCode, 402); + assert.equal(gateway.state.payments.length, 0); + } finally { + await gateway.close(); + } +}); + +// ---------------------------------------------------------------- puppeteer + +function fakePage() { + let extra = {}; + const gotos = []; + const page = { + setExtraHTTPHeaders: async (h) => { + extra = h; + }, + goto: async (url, options) => { + gotos.push({ url, options, headers: { ...extra } }); + const res = await globalThis.fetch(url, { headers: extra }); + return { status: () => res.status, text: () => res.text() }; + }, + }; + return { page, gotos, extra: () => extra }; +} + +test('puppeteer: gotoPaid pays on a 402 and navigates again with the pass', async () => { + const gateway = await startGateway(); + try { + const client = createClient({ key: KEY_ONE }); + const { page, gotos, extra } = fakePage(); + const res = await puppeteerGoto(page, `${gateway.url}/page`, client, { waitUntil: 'load' }); + assert.equal(res.status(), 200); + assert.equal(gotos.length, 2, 'the refused navigation and the paid one'); + assert.deepEqual(gotos[1].options, { waitUntil: 'load' }); + assert.match(gotos[1].headers['x-crawl-pass'], /^cp_test1/); + assert.equal(gateway.state.payments.length, 1); + assert.equal(await applyPass(page, `${gateway.url}/x`, client), true); + assert.match(extra()['x-crawl-pass'], /^cp_test1/); + } finally { + await gateway.close(); + } +}); + +test('puppeteer: attachX402 makes page.goto pay, and detaches cleanly', async () => { + const gateway = await startGateway(); + try { + const client = createClient({ key: KEY_ONE }); + const { page, gotos } = fakePage(); + const original = page.goto; + const detach = attachPuppeteer(page, client); + const res = await page.goto(`${gateway.url}/page`); + assert.equal(res.status(), 200); + assert.equal(gateway.state.payments.length, 1); + assert.equal(gotos.length, 2); + + const second = await page.goto(`${gateway.url}/other`); + assert.equal(second.status(), 200); + assert.equal(gateway.state.payments.length, 1); + assert.equal(gotos.length, 3); + + detach(); + assert.equal(page.goto, original); + } finally { + await gateway.close(); + } +});