From 21cf0b33fd7e9d52bc8af081a40a699dd5988cb0 Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Mon, 20 Apr 2026 16:00:59 +0300 Subject: [PATCH 1/4] feat(api): retry on transient chain-RPC failures + fail loud in provider fan-out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two layered fixes for the "8 providers silently missing from historical rewards responses" bug: fetchRest now retries transient failures with exponential backoff, and fetchProvidersWithSpecs / fetchAllProvidersImpl no longer swallow errors via .catch(() => []). Before: • 25-wide concurrent chain-RPC fan-out against the public archive node occasionally triggered rate-limits / slow paths / connection drops • Errors were caught with `.catch(() => [])`, dropping an entire spec's providers without any signal to the caller or in logs • A historical /provider-estimated-rewards?block=N response could be missing 8-20% of providers vs jsinfo; the wrong response would then be cached for a year under CACHE_TTL.IMMUTABLE After: • fetchRest wraps its fetch() in fetchWithRetry: - Retries on 408, 429 (honors Retry-After), 5xx, network errors - Does NOT retry on 4xx (wrong path, retrying won't help) - 3 attempts, exponential backoff capped at 8s - Request timeout bumped 15s → 20s (archive-node slow paths) • fetchProvidersWithSpecs and fetchAllProvidersImpl drop `.catch(...)`: if a spec fetch fails after retries, the whole request throws. Fastify returns 500, cache plugin skips 4xx/5xx so no bad response gets cached. Next caller retries from scratch. Verified against block 4697952: all 40 common providers match jsinfo byte-exact. Remaining 8 will come in after this ships (retries will recover the transient drops that were previously silently lost). 114 tests passing. --- apps/api/src/rpc/providers.ts | 13 +++++--- apps/api/src/rpc/rest.ts | 61 ++++++++++++++++++++++++++++++++--- apps/api/src/rpc/rewards.ts | 38 +++++++++++----------- 3 files changed, 86 insertions(+), 26 deletions(-) diff --git a/apps/api/src/rpc/providers.ts b/apps/api/src/rpc/providers.ts index 6b0bc89..31adfcf 100644 --- a/apps/api/src/rpc/providers.ts +++ b/apps/api/src/rpc/providers.ts @@ -119,10 +119,13 @@ export async function fetchProvidersWithSpecs( for (let i = 0; i < allSpecs.length; i += RPC_BATCH_SIZE) { const batch = allSpecs.slice(i, i + RPC_BATCH_SIZE); + // No .catch() — fetchRest now retries transient failures (5xx, 429, + // network) up to 3 times. If it still throws, we'd rather 500 loudly + // than silently drop providers and cache an incomplete response. The + // cache plugin skips 4xx/5xx so a failed request never poisons Redis. const results = await Promise.all( batch.map((s) => fetchProvidersForSpec(s.index, blockHeight) - .then((ps) => ps.map((p) => ({ ...p, specId: s.index, specName: s.name }))) - .catch(() => [] as Array)), + .then((ps) => ps.map((p) => ({ ...p, specId: s.index, specName: s.name })))), ); for (const providers of results) { @@ -222,6 +225,9 @@ async function fetchAllProvidersImpl(): Promise { for (let i = 0; i < specs.length; i += RPC_BATCH_SIZE) { const batch = specs.slice(i, i + RPC_BATCH_SIZE); const results = await Promise.all( + // Same rationale as fetchProvidersWithSpecs: no .catch(). fetchRest + // retries transient failures; if it still throws, fail loud rather + // than silently drop providers from /providers or /all_providers_apr. batch.map((s) => fetchProvidersForSpec(s.index) .then((providers) => providers.map((p): SpecEntry => ({ @@ -232,8 +238,7 @@ async function fetchAllProvidersImpl(): Promise { delegate_total: p.delegate_total, delegate_commission: p.delegate_commission, specId: s.index, - }))) - .catch(() => [] as SpecEntry[]), + }))), ), ); specProviders.push(...results); diff --git a/apps/api/src/rpc/rest.ts b/apps/api/src/rpc/rest.ts index 47e275f..5ec5d39 100644 --- a/apps/api/src/rpc/rest.ts +++ b/apps/api/src/rpc/rest.ts @@ -13,6 +13,62 @@ export const RPC_BATCH_SIZE = config.lava.rpcBatchSize; // Request coalescing: concurrent fetches for the same path share one in-flight request const inflightRpc = new Map>(); +// ── Retry policy ───────────────────────────────────────────────────────────── +// Transient failures on the public chain RPC (network blips, rate limits on +// concurrent historical queries, archive-node slow paths) previously returned +// errors that upstream callers silently swallowed via `.catch(() => [])`, +// dropping providers from historical rewards responses. Retry with exponential +// backoff so transient blips don't leak into user-visible output. +// +// Retry on "likely transient" failures: +// • Network error (fetch rejected — timeout, connection reset, DNS, etc.) +// • 408 Request Timeout +// • 429 Too Many Requests (honor Retry-After if present) +// • 5xx Server Error +// 4xx client errors (including 404) are NOT retried — wrong path/params, +// retrying won't help. +const RPC_MAX_ATTEMPTS = 3; +const RPC_MAX_BACKOFF_MS = 8_000; +const RPC_REQUEST_TIMEOUT_MS = 20_000; + +function shouldRetry(statusOrNull: number | null): boolean { + if (statusOrNull === null) return true; // network-level error + if (statusOrNull === 408 || statusOrNull === 429) return true; + if (statusOrNull >= 500 && statusOrNull < 600) return true; + return false; +} + +async function fetchWithRetry(url: string, headers: Record): Promise { + let lastErr: unknown; + for (let attempt = 1; attempt <= RPC_MAX_ATTEMPTS; attempt++) { + try { + const res = await fetch(url, { + headers, + signal: AbortSignal.timeout(RPC_REQUEST_TIMEOUT_MS), + }); + if (res.ok) return res; + if (!shouldRetry(res.status)) return res; // 4xx — let caller throw + + if (attempt === RPC_MAX_ATTEMPTS) return res; // exhausted — return last + + const retryAfter = parseInt(res.headers.get("retry-after") ?? "0", 10); + const backoffMs = Math.min( + RPC_MAX_BACKOFF_MS, + retryAfter > 0 ? retryAfter * 1000 : 2 ** attempt * 500, + ); + logger.warn({ url, status: res.status, attempt, backoffMs }, "chain RPC retryable error, backing off"); + await new Promise((r) => setTimeout(r, backoffMs)); + } catch (e) { + lastErr = e; + if (attempt === RPC_MAX_ATTEMPTS) throw e; + const backoffMs = Math.min(RPC_MAX_BACKOFF_MS, 2 ** attempt * 500); + logger.warn({ url, attempt, err: String(e) }, "chain RPC network error, retrying"); + await new Promise((r) => setTimeout(r, backoffMs)); + } + } + throw lastErr ?? new Error("fetchWithRetry: unreachable"); +} + export async function fetchRest(path: string, blockHeight?: number): Promise { const cacheKey = blockHeight ? `${path}@${blockHeight}` : path; const existing = inflightRpc.get(cacheKey); @@ -22,10 +78,7 @@ export async function fetchRest(path: string, blockHeight?: number): Promise< const headers: Record = {}; if (blockHeight) headers["x-cosmos-block-height"] = String(blockHeight); - const res = await fetch(`${config.lava.restUrl}${path}`, { - headers, - signal: AbortSignal.timeout(15_000), - }); + const res = await fetchWithRetry(`${config.lava.restUrl}${path}`, headers); if (!res.ok) throw new Error(`RPC ${res.status}: ${res.statusText}`); return (await res.json()) as T; })(); diff --git a/apps/api/src/rpc/rewards.ts b/apps/api/src/rpc/rewards.ts index ca039e6..cd67375 100644 --- a/apps/api/src/rpc/rewards.ts +++ b/apps/api/src/rpc/rewards.ts @@ -149,19 +149,19 @@ export interface RewardsBySpecEntry { } /** Fetch the raw estimated_provider_rewards chain response for one provider. - * Returns an empty structure on error so callers can iterate without throwing. */ + * Throws on failure (after fetchRest's internal retries are exhausted) so the + * caller can fail loud — historical rewards responses are cached for a year, + * so silently dropping a provider on transient RPC flakes would bake a wrong + * response into the cache. The cache plugin skips 4xx/5xx, so the next caller + * retries from scratch. */ export async function fetchRawProviderRewards( address: string, blockHeight?: number, ): Promise { - try { - return await fetchRest( - `/lavanet/lava/subscription/estimated_provider_rewards/${address}/`, - blockHeight, - ); - } catch { - return { info: [], total: [] }; - } + return await fetchRest( + `/lavanet/lava/subscription/estimated_provider_rewards/${address}/`, + blockHeight, + ); } /** Extract the set of base denoms (post-IBC-resolution) that appear in one or @@ -276,13 +276,11 @@ export async function processRawProviderRewards( } /** - * Fetch a provider's actual earned rewards (no benchmark amount) and group by spec. - * Composes fetchRawProviderRewards + processRawProviderRewards for callers that - * don't need the raw/extract/process split (e.g. APR's computeAllProvidersApr). - * - * Pass blockHeight to query historical chain state (archive node required). - * Pass priceOverrides (keyed by base denom, e.g. "lava") to price tokens at a - * specific point in time. + * Best-effort variant for live-dashboard consumers (APR). Catches per-provider + * RPC failures and returns [] so one flaky provider doesn't 500 the whole APR + * computation. The new historical-rewards route calls fetchRawProviderRewards + * directly and lets failures propagate, because incomplete data would be + * cached for a year under CACHE_TTL.IMMUTABLE. */ export async function fetchRewardsBySpec( address: string, @@ -290,8 +288,12 @@ export async function fetchRewardsBySpec( blockHeight?: number, priceOverrides?: Record, ): Promise { - const raw = await fetchRawProviderRewards(address, blockHeight); - return processRawProviderRewards(raw, specNameMap, priceOverrides); + try { + const raw = await fetchRawProviderRewards(address, blockHeight); + return await processRawProviderRewards(raw, specNameMap, priceOverrides); + } catch { + return []; + } } export interface ClaimableRewardEntry { From d84bea64a77a8aff2450a7b48ff17591df0d39d6 Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Tue, 21 Apr 2026 09:57:22 +0300 Subject: [PATCH 2/4] feat(api): archive-header routing + indexer-backed address list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold-fetch cost for /provider-estimated-rewards?block=N used to be 70-95s and often 500'd on blocks older than a week. Two-part fix: 1. rest.ts: chain LB routes historical queries to non-archive replicas which return a 200 OK body with `{success:false, "version does not exist"}`. Previously that silently parsed as "empty rewards" and dropped providers from the response. Now: - classify body-level errors: "retry" (pruned state — version missing) vs "passthrough" (legitimate empty, e.g. provider not found) so we don't spin through retries on real empties. - set `lava-extension: archive` header on queries for blocks older than ~7 days (auto-detected via cached latest-height). - retry on a fresh no-keepalive undici dispatcher so each attempt breaks out of CF's socket-level replica pinning and gets a fresh routing decision. - 10 attempts with jittered backoff; Connection: close after first. 2. provider-estimated-rewards.ts: swap the 25-30s of per-spec stake-list chain RPCs for a single indexer GraphQL query. - address list comes from indexer `app.providers` (every provider ever paid a relay — superset of any address with rewards at a historical block). - specNames from fetchAllSpecs (block-agnostic, cached). - monikers from fetchAllProviderMonikers (currently-staked; unstaked ghosts render as "-" in the FE). - LAVA historical price fetch kicked off in parallel with the chain fan-out instead of serially after it. Non-LAVA denoms (rare) are fetched lazily if they appear in the raw rewards. Verified against jsinfo: 50/50 providers, 0 USD mismatches at block 4895283. Cold fetch 70-95s → ~54s. The remaining cost is almost entirely the per-provider estimated_provider_rewards fan-out, which can only be eliminated by a lava-indexer snapshot job. --- .../src/routes/provider-estimated-rewards.ts | 94 ++++++----- apps/api/src/rpc/rest.ts | 156 +++++++++++++++--- 2 files changed, 191 insertions(+), 59 deletions(-) diff --git a/apps/api/src/routes/provider-estimated-rewards.ts b/apps/api/src/routes/provider-estimated-rewards.ts index 5cfa884..122b76e 100644 --- a/apps/api/src/routes/provider-estimated-rewards.ts +++ b/apps/api/src/routes/provider-estimated-rewards.ts @@ -4,16 +4,18 @@ import { RPC_BATCH_SIZE, buildHistoricalPriceMap, extractBaseDenoms, + fetchAllProviderMonikers, + fetchAllSpecs, fetchBlockAtTimestamp, fetchBlockTime, fetchLavaUsdPrice, - fetchProvidersWithSpecs, fetchRawProviderRewards, prewarmPriceCache, processRawProviderRewards, type EstimatedRewardsResponse, type RewardsBySpecEntry, } from "../rpc/lava.js"; +import { gqlSafe } from "../graphql/client.js"; import { sendApiError } from "../plugins/error-handler.js"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -138,9 +140,6 @@ export async function providerEstimatedRewardsRoutes(app: FastifyInstance) { await prewarmPriceCache(); - // Compute block metadata upfront (needed for both path branches) and - // snapshot the provider set AT the block (when set) so providers who have - // since unstaked / deregistered still appear in historical responses. let priceTimestamp: string; let priceLavaUsd: number | undefined; const blockTimeIso = block ? await fetchBlockTime(block) : null; @@ -151,13 +150,40 @@ export async function providerEstimatedRewardsRoutes(app: FastifyInstance) { priceLavaUsd = await fetchLavaUsdPrice(); } - const { providers: providerMap, specNames } = await fetchProvidersWithSpecs(block); - const addresses = Array.from(providerMap.keys()); + // ── Resolve addresses ──────────────────────────────────────────────── + // Source of truth is the indexer's `app.providers` table — every provider + // that's ever been paid a relay, which is a superset of anyone who can + // have estimated rewards at a historical block. Monikers come from the + // current chain stake list (fast, cached); providers who've since unstaked + // won't have a moniker and render as "-" in the FE. + const [specs, indexerProviders, monikerMap] = await Promise.all([ + fetchAllSpecs(), + gqlSafe<{ allProviders: { nodes: Array<{ addr: string }> } } | null>( + `query { allProviders { nodes { addr } } }`, + undefined, + null, + ), + fetchAllProviderMonikers(), + ]); + const specNames = new Map(specs.map((s) => [s.index, s.name])); + const providerMap = new Map(); + const addresses: string[] = []; + for (const node of indexerProviders?.allProviders.nodes ?? []) { + addresses.push(node.addr); + providerMap.set(node.addr, { moniker: monikerMap.get(node.addr) ?? "" }); + } + + // ── LAVA price: kicked off here so it overlaps with the chain fan-out. + // ~99% of reward denoms are ulava; non-LAVA denoms that actually appear + // are fetched after fan-out inside the rare-path below. On live (non-block) + // queries we use the live cache, no override needed. + let priceOverrides: Record | undefined; + const lavaPricePromise: Promise> | null = + block && blockTimeIso + ? buildHistoricalPriceMap(new Date(blockTimeIso), ["lava"]) + : null; - // ── Pass 1: fetch raw rewards for every provider ───────────────────── - // Keeps the chain response so we can (a) inspect which denoms actually - // appear and (b) format the final output with the right prices, without - // paying for a second chain-RPC round. + // ── Chain fan-out: per-provider estimated_provider_rewards ────────── const rawByAddr = new Map(); for (let i = 0; i < addresses.length; i += RPC_BATCH_SIZE) { const batch = addresses.slice(i, i + RPC_BATCH_SIZE); @@ -165,38 +191,30 @@ export async function providerEstimatedRewardsRoutes(app: FastifyInstance) { batch.forEach((addr, j) => rawByAddr.set(addr, raws[j]!)); } - // ── Price overrides ────────────────────────────────────────────────── - // For historical blocks, fetch prices ONLY for denoms that appear in the - // rewards (typically just LAVA + 0-2 IBC denoms). Fetching all 22 known - // denoms sequentially with retry backoff would exceed the gateway timeout - // under any CoinGecko throttling. - let priceOverrides: Record | undefined; - if (block && blockTimeIso) { + if (lavaPricePromise) { + try { + priceOverrides = await lavaPricePromise; + } catch (err) { + // Cache plugin skips 4xx/5xx — refusing to ship avoids poisoning the + // block-keyed cache with a wrong price for a year. + return sendApiError( + reply, 503, + `historical LAVA price unavailable for block ${block} (${blockTimeIso}); please retry: ${(err as Error).message}`, + ); + } + + // Rare non-LAVA denoms that actually appeared in this block. IBC + // resolution hits chain denom_trace (cached), so this is cheap. const relevantDenoms = await extractBaseDenoms([...rawByAddr.values()]); - if (relevantDenoms.size === 0) { - // No priceable rewards in this block — nothing to override. - priceOverrides = {}; - priceLavaUsd = 0; - } else { - try { - priceOverrides = await buildHistoricalPriceMap( - new Date(blockTimeIso), - [...relevantDenoms], - ); - } catch (err) { - // LAVA historical price unavailable. Refuse to ship a response — - // the cache plugin skips 4xx/5xx so we don't poison the block-keyed - // cache with a wrong price for a year. - return sendApiError( - reply, 503, - `historical LAVA price unavailable for block ${block} (${blockTimeIso}); please retry: ${(err as Error).message}`, - ); - } - priceLavaUsd = priceOverrides.lava ?? await fetchLavaUsdPrice(); + const missing = [...relevantDenoms].filter((d) => priceOverrides![d] === undefined); + if (missing.length > 0) { + const extra = await buildHistoricalPriceMap(new Date(blockTimeIso!), missing); + priceOverrides = { ...priceOverrides, ...extra }; } + priceLavaUsd = priceOverrides.lava ?? priceLavaUsd ?? await fetchLavaUsdPrice(); } - // ── Pass 2: format each provider's raw response with the right prices ─ + // ── Format each provider's raw response ───────────────────────────── const results: Array<{ provider: string; moniker: string; diff --git a/apps/api/src/rpc/rest.ts b/apps/api/src/rpc/rest.ts index 5ec5d39..118f5c3 100644 --- a/apps/api/src/rpc/rest.ts +++ b/apps/api/src/rpc/rest.ts @@ -1,8 +1,23 @@ import pino from "pino"; +import { Agent } from "undici"; import { config } from "../config.js"; export const logger = pino({ name: "rpc" }); +// One-shot dispatcher used only for retry attempts after a pruned-replica +// error. The default global dispatcher pools 32 sockets per origin for 30s, +// which means every retry inside that window reuses the SAME TCP connection +// and lands on the same backend (the one that doesn't have the block). A +// dispatcher with `keepAliveTimeout: 0` and `pipelining: 0` opens — and +// closes — one connection per request, giving Cloudflare a fresh routing +// decision and a chance at a healthy archive replica. +const noKeepAliveDispatcher = new Agent({ + connections: 1, + pipelining: 0, + keepAliveTimeout: 1, + keepAliveMaxTimeout: 1, +}); + /** * Concurrent in-flight RPC calls per batch loop. Operators running dedicated * RPC endpoints can raise this (env: RPC_BATCH_SIZE). Public endpoints should @@ -13,6 +28,29 @@ export const RPC_BATCH_SIZE = config.lava.rpcBatchSize; // Request coalescing: concurrent fetches for the same path share one in-flight request const inflightRpc = new Map>(); +// Archive-routing threshold. Recent blocks live on every replica; older blocks +// risk hitting a non-archive replica that has pruned the state and responds +// with `{success:false, "version does not exist"}`. Empirically the public LB +// starts losing blocks well within a week, so keep a tight 7-day window — +// archive nodes are heavier so we still want to avoid them for fresh queries. +// Lava produces ~1 block per 30s → 7 days ≈ 20_160 blocks. +const ARCHIVE_THRESHOLD_BLOCKS = 20_160; +const LATEST_HEIGHT_CACHE_MS = 60_000; +let latestHeightCache: { height: number; at: number } | null = null; + +async function getLatestHeight(): Promise { + if (latestHeightCache && Date.now() - latestHeightCache.at < LATEST_HEIGHT_CACHE_MS) { + return latestHeightCache.height; + } + // No blockHeight → won't trigger the archive-routing logic, no recursion. + const data = await fetchRest<{ block: { header: { height: string } } }>( + "/cosmos/base/tendermint/v1beta1/blocks/latest", + ); + const height = parseInt(data.block.header.height, 10); + latestHeightCache = { height, at: Date.now() }; + return height; +} + // ── Retry policy ───────────────────────────────────────────────────────────── // Transient failures on the public chain RPC (network blips, rate limits on // concurrent historical queries, archive-node slow paths) previously returned @@ -25,10 +63,15 @@ const inflightRpc = new Map>(); // • 408 Request Timeout // • 429 Too Many Requests (honor Retry-After if present) // • 5xx Server Error +// • Chain LB body-level error (200 OK with `{success:false, message:"upstream_error: ..."}`) +// The lava REST gateway fronts multiple archive nodes; some have older +// blocks pruned and respond with this wrapper instead of the expected +// payload. Retries on a fresh connection (Connection: close + jittered +// backoff) usually land on a healthy replica. // 4xx client errors (including 404) are NOT retried — wrong path/params, // retrying won't help. -const RPC_MAX_ATTEMPTS = 3; -const RPC_MAX_BACKOFF_MS = 8_000; +const RPC_MAX_ATTEMPTS = 10; +const RPC_MAX_BACKOFF_MS = 5_000; const RPC_REQUEST_TIMEOUT_MS = 20_000; function shouldRetry(statusOrNull: number | null): boolean { @@ -38,31 +81,94 @@ function shouldRetry(statusOrNull: number | null): boolean { return false; } -async function fetchWithRetry(url: string, headers: Record): Promise { +/** + * Lava REST returns a 200 OK with `{success:false, message:"upstream_error: ..."}` + * for two very different conditions: + * 1. The replica doesn't have the requested block in archive (pruned). Retry + * on a fresh connection — another replica may have it. + * 2. The chain genuinely has no data for the request (provider not found, + * empty rewards collection, etc). Don't retry — return the body so + * downstream processors handle it as "no data". + * + * Returns "retry" for archive misses, "passthrough" for legitimate empty + * answers, or null if the body isn't an error wrapper at all. + */ +function classifyChainUpstreamError(body: unknown): "retry" | "passthrough" | null { + if ( + typeof body !== "object" || + body === null || + (body as { success?: unknown }).success !== false + ) { + return null; + } + const msg = ((body as { message?: unknown }).message as string | undefined) ?? ""; + // Archive pruning markers — replica doesn't have this block. + if (msg.includes("version does not exist") || msg.includes("version mismatch")) { + return "retry"; + } + // Anything else is treated as a legitimate "no data" — passthrough so the + // caller sees an empty payload rather than a 5xx after wasted retries. + return "passthrough"; +} + +function backoffWithJitter(attempt: number): number { + const base = Math.min(RPC_MAX_BACKOFF_MS, 2 ** attempt * 300); + return base + Math.floor(Math.random() * 500); +} + +async function fetchWithRetry(url: string, baseHeaders: Record): Promise { let lastErr: unknown; for (let attempt = 1; attempt <= RPC_MAX_ATTEMPTS; attempt++) { + // After the first attempt force a brand-new TCP connection via a no-pool + // dispatcher. The default global pool keeps sockets alive for ~30s; reusing + // any of them means landing on the same backend that just rejected us. + const fetchOpts: RequestInit = { + headers: attempt === 1 ? baseHeaders : { ...baseHeaders, Connection: "close" }, + signal: AbortSignal.timeout(RPC_REQUEST_TIMEOUT_MS), + }; + if (attempt > 1) (fetchOpts as { dispatcher?: unknown }).dispatcher = noKeepAliveDispatcher; try { - const res = await fetch(url, { - headers, - signal: AbortSignal.timeout(RPC_REQUEST_TIMEOUT_MS), - }); - if (res.ok) return res; - if (!shouldRetry(res.status)) return res; // 4xx — let caller throw + const res = await fetch(url, fetchOpts); + if (res.ok) { + const body = (await res.json()) as unknown; + const cls = classifyChainUpstreamError(body); + if (cls === "retry") { + const message = (body as { message?: string }).message; + if (attempt === RPC_MAX_ATTEMPTS) { + throw new Error( + `chain upstream error after ${RPC_MAX_ATTEMPTS} attempts: ${message ?? "(no message)"}`, + ); + } + const backoffMs = backoffWithJitter(attempt); + logger.warn( + { url, attempt, backoffMs, message }, + "chain upstream returned success=false (pruned replica), retrying on fresh connection", + ); + await new Promise((r) => setTimeout(r, backoffMs)); + continue; + } + // cls === "passthrough" → legitimate empty answer; cls === null → normal payload. + return body as T; + } + if (!shouldRetry(res.status)) { + throw new Error(`RPC ${res.status}: ${res.statusText}`); + } - if (attempt === RPC_MAX_ATTEMPTS) return res; // exhausted — return last + if (attempt === RPC_MAX_ATTEMPTS) { + throw new Error(`RPC ${res.status}: ${res.statusText}`); + } const retryAfter = parseInt(res.headers.get("retry-after") ?? "0", 10); - const backoffMs = Math.min( - RPC_MAX_BACKOFF_MS, - retryAfter > 0 ? retryAfter * 1000 : 2 ** attempt * 500, - ); + const backoffMs = retryAfter > 0 + ? Math.min(RPC_MAX_BACKOFF_MS, retryAfter * 1000) + : backoffWithJitter(attempt); logger.warn({ url, status: res.status, attempt, backoffMs }, "chain RPC retryable error, backing off"); await new Promise((r) => setTimeout(r, backoffMs)); } catch (e) { lastErr = e; if (attempt === RPC_MAX_ATTEMPTS) throw e; - const backoffMs = Math.min(RPC_MAX_BACKOFF_MS, 2 ** attempt * 500); - logger.warn({ url, attempt, err: String(e) }, "chain RPC network error, retrying"); + const backoffMs = backoffWithJitter(attempt); + logger.warn({ url, attempt, err: String(e) }, "chain RPC error, retrying"); await new Promise((r) => setTimeout(r, backoffMs)); } } @@ -76,11 +182,19 @@ export async function fetchRest(path: string, blockHeight?: number): Promise< const promise = (async () => { const headers: Record = {}; - if (blockHeight) headers["x-cosmos-block-height"] = String(blockHeight); - - const res = await fetchWithRetry(`${config.lava.restUrl}${path}`, headers); - if (!res.ok) throw new Error(`RPC ${res.status}: ${res.statusText}`); - return (await res.json()) as T; + if (blockHeight) { + headers["x-cosmos-block-height"] = String(blockHeight); + // Pin OLD historical queries to archive replicas. Without the header the + // public LB freely routes to non-archive replicas that have pruned the + // requested block and return `{success:false, "version does not exist"}`. + // Recent blocks are still on every replica, and archive nodes are heavier + // / under more load — so only opt in when the block is genuinely old. + const latest = await getLatestHeight().catch(() => null); + if (latest !== null && latest - blockHeight > ARCHIVE_THRESHOLD_BLOCKS) { + headers["lava-extension"] = "archive"; + } + } + return await fetchWithRetry(`${config.lava.restUrl}${path}`, headers); })(); inflightRpc.set(cacheKey, promise); From 68d2aab64ebc4b3d6094169ba4251728988e438b Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Tue, 21 Apr 2026 10:02:12 +0300 Subject: [PATCH 3/4] test(api): update mocks for indexer-backed address list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Route no longer calls fetchProvidersWithSpecs / fetchAllProviders — address list comes from the indexer's app.providers table via gqlSafe. Tests now mock fetchAllSpecs + fetchAllProviderMonikers + gqlSafe. Also split the historical-prices assertion into two calls: LAVA upfront (kicked off in parallel with the chain fan-out) + any non-LAVA denoms that actually appear after fan-out. Replaces the "snapshots the provider set at the historical block" test with one asserting that indexer ghost providers (unstaked but historically active) still get queried. --- .../routes-provider-estimated-rewards.test.ts | 81 ++++++++++++------- 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/apps/api/src/__tests__/routes-provider-estimated-rewards.test.ts b/apps/api/src/__tests__/routes-provider-estimated-rewards.test.ts index 5b5e9a2..4af7a0b 100644 --- a/apps/api/src/__tests__/routes-provider-estimated-rewards.test.ts +++ b/apps/api/src/__tests__/routes-provider-estimated-rewards.test.ts @@ -5,7 +5,8 @@ import { errorHandlerPlugin } from "../plugins/error-handler.js"; vi.mock("../rpc/lava.js", () => ({ RPC_BATCH_SIZE: 5, prewarmPriceCache: vi.fn(), - fetchProvidersWithSpecs: vi.fn(), + fetchAllSpecs: vi.fn(), + fetchAllProviderMonikers: vi.fn(), fetchRawProviderRewards: vi.fn(), extractBaseDenoms: vi.fn(), processRawProviderRewards: vi.fn(), @@ -15,9 +16,14 @@ vi.mock("../rpc/lava.js", () => ({ buildHistoricalPriceMap: vi.fn(), })); +vi.mock("../graphql/client.js", () => ({ + gqlSafe: vi.fn(), +})); + const { prewarmPriceCache, - fetchProvidersWithSpecs, + fetchAllSpecs, + fetchAllProviderMonikers, fetchRawProviderRewards, extractBaseDenoms, processRawProviderRewards, @@ -26,6 +32,7 @@ const { fetchLavaUsdPrice, buildHistoricalPriceMap, } = await import("../rpc/lava.js"); +const { gqlSafe } = await import("../graphql/client.js"); const { providerEstimatedRewardsRoutes } = await import("../routes/provider-estimated-rewards.js"); async function buildApp() { @@ -35,12 +42,16 @@ async function buildApp() { return app; } -const MOCK_PROVIDERS_WITH_SPECS = { - providers: new Map([ - ["lava@1abc", { moniker: "AlphaProvider", identity: "", commission: "50", specs: [] }], - ["lava@2def", { moniker: "BetaProvider", identity: "", commission: "30", specs: [] }], - ]), - specNames: new Map([["ETH1", "Ethereum Mainnet"], ["NEAR", "Near"]]), +const MOCK_SPECS = [ + { index: "ETH1", name: "Ethereum Mainnet" }, + { index: "NEAR", name: "Near" }, +]; +const MOCK_MONIKERS = new Map([ + ["lava@1abc", "AlphaProvider"], + ["lava@2def", "BetaProvider"], +]); +const MOCK_INDEXER_PROVIDERS = { + allProviders: { nodes: [{ addr: "lava@1abc" }, { addr: "lava@2def" }] }, }; const TOK = (amount: string, usd: string) => ({ @@ -90,7 +101,9 @@ beforeEach(() => { vi.resetAllMocks(); (prewarmPriceCache as ReturnType).mockResolvedValue(undefined); (fetchLavaUsdPrice as ReturnType).mockResolvedValue(0.12); - (fetchProvidersWithSpecs as ReturnType).mockResolvedValue(MOCK_PROVIDERS_WITH_SPECS); + (fetchAllSpecs as ReturnType).mockResolvedValue(MOCK_SPECS); + (fetchAllProviderMonikers as ReturnType).mockResolvedValue(MOCK_MONIKERS); + (gqlSafe as ReturnType).mockResolvedValue(MOCK_INDEXER_PROVIDERS); (fetchRawProviderRewards as ReturnType).mockResolvedValue(STUB_RAW); (extractBaseDenoms as ReturnType).mockResolvedValue(new Set(["lava"])); (processRawProviderRewards as ReturnType).mockImplementation( @@ -179,13 +192,9 @@ describe("GET /provider-estimated-rewards", () => { }); it("batches provider RPC calls 5 at a time", async () => { - const providers = new Map(); - for (let i = 0; i < 12; i++) { - providers.set(`lava@p${i}`, { moniker: `P${i}`, identity: "", commission: "50", specs: [] }); - } - (fetchProvidersWithSpecs as ReturnType).mockResolvedValue({ - providers, specNames: new Map(), - }); + const nodes = Array.from({ length: 12 }, (_, i) => ({ addr: `lava@p${i}` })); + (gqlSafe as ReturnType).mockResolvedValue({ allProviders: { nodes } }); + (fetchAllProviderMonikers as ReturnType).mockResolvedValue(new Map()); (processRawProviderRewards as ReturnType).mockResolvedValue([]); const app = await buildApp(); await app.inject({ method: "GET", url: "/provider-estimated-rewards" }); @@ -208,7 +217,11 @@ describe("GET /provider-estimated-rewards", () => { it("for historical blocks, fetches prices only for denoms actually in rewards", async () => { (extractBaseDenoms as ReturnType).mockResolvedValue(new Set(["lava", "atom"])); - (buildHistoricalPriceMap as ReturnType).mockResolvedValue({ lava: 0.035, atom: 8.2 }); + // LAVA fetch (kicked off in parallel with fan-out) returns lava; the + // follow-up non-LAVA fetch resolves the remaining denoms. + (buildHistoricalPriceMap as ReturnType) + .mockResolvedValueOnce({ lava: 0.035 }) + .mockResolvedValueOnce({ atom: 8.2 }); const app = await buildApp(); const res = await app.inject({ @@ -217,15 +230,15 @@ describe("GET /provider-estimated-rewards", () => { }); expect(res.statusCode).toBe(200); - // buildHistoricalPriceMap called with ONLY the relevant denoms (not all 22) - expect(buildHistoricalPriceMap).toHaveBeenCalledWith( - expect.any(Date), - expect.arrayContaining(["lava", "atom"]), - ); - const passedDenoms = (buildHistoricalPriceMap as ReturnType).mock.calls[0]![1] as string[]; - expect(passedDenoms.length).toBe(2); + // Two calls total: ['lava'] upfront + any non-LAVA denoms after fan-out. + // Never all 22 known denoms (that used to blow the gateway timeout). + expect(buildHistoricalPriceMap).toHaveBeenCalledTimes(2); + const first = (buildHistoricalPriceMap as ReturnType).mock.calls[0]![1] as string[]; + expect(first).toEqual(["lava"]); + const second = (buildHistoricalPriceMap as ReturnType).mock.calls[1]![1] as string[]; + expect(second).toEqual(["atom"]); - // Price overrides passed down to processRawProviderRewards + // Merged price overrides passed down to processRawProviderRewards for (const call of (processRawProviderRewards as ReturnType).mock.calls) { expect(call[2]).toEqual({ lava: 0.035, atom: 8.2 }); } @@ -248,20 +261,28 @@ describe("GET /provider-estimated-rewards", () => { expect(fetchLavaUsdPrice).not.toHaveBeenCalled(); }); - it("snapshots the provider set at the historical block", async () => { + it("uses the indexer's all-time provider list (includes unstaked historical providers)", async () => { + // Indexer has a provider that isn't currently staked (no moniker). Route + // should still query it at the historical block and render it with "-". + (gqlSafe as ReturnType).mockResolvedValue({ + allProviders: { nodes: [{ addr: "lava@1abc" }, { addr: "lava@2def" }, { addr: "lava@3ghost" }] }, + }); + mockProcessPerProvider(); (fetchBlockTime as ReturnType).mockResolvedValue("2026-03-17T15:00:00Z"); const app = await buildApp(); - await app.inject({ method: "GET", url: "/provider-estimated-rewards?block=4697952" }); - expect(fetchProvidersWithSpecs).toHaveBeenCalledWith(4697952); + const res = await app.inject({ method: "GET", url: "/provider-estimated-rewards?block=4697952" }); + expect(res.statusCode).toBe(200); + expect(fetchRawProviderRewards).toHaveBeenCalledTimes(3); + const queriedAddrs = (fetchRawProviderRewards as ReturnType).mock.calls.map((c) => c[0]); + expect(queriedAddrs).toEqual(expect.arrayContaining(["lava@1abc", "lava@2def", "lava@3ghost"])); }); - it("uses live prices + current provider set when ?block= is omitted", async () => { + it("uses live prices when ?block= is omitted", async () => { (fetchLavaUsdPrice as ReturnType).mockResolvedValue(0.025); const app = await buildApp(); const res = await app.inject({ method: "GET", url: "/provider-estimated-rewards" }); const body = JSON.parse(res.body); expect(body.meta.priceLavaUsd).toBe(0.025); - expect(fetchProvidersWithSpecs).toHaveBeenCalledWith(undefined); expect(buildHistoricalPriceMap).not.toHaveBeenCalled(); for (const call of (processRawProviderRewards as ReturnType).mock.calls) { expect(call[2]).toBeUndefined(); From bad05f4a66651e65b3717dbe8d1f1c8941f1320c Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Tue, 21 Apr 2026 11:15:27 +0300 Subject: [PATCH 4/4] fix(api): tighten chain upstream-error classification + route fetchBlockTime through fetchRest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related correctness bugs for historical /provider-estimated-rewards: 1. Back-to-back cold fetches of the same block produced different provider counts and total_usd (32 / \$35k vs 33 / \$45k for block 3928945). Root cause: a bare "collections: not found" can come from two places: - legit: provider has no accrued rewards at this block - degraded replica: partially-pruned state can't look up the key classifyChainUpstreamError used to treat both as passthrough, silently dropping providers from the response under cold-fetch LB variance. Now only an explicit "cannot get claimable rewards after distribution" marker qualifies as passthrough — everything else retries. Degraded replicas now force a fail-loud 5xx (which the cache plugin skips), matching the same "never cache incomplete data" discipline as the LAVA price 503 path. 2. fetchBlockTime hit tendermint-RPC directly (/block?height=N) which gateway-times-out on pruned replicas and returns {result:{success:false, message:"gateway_timeout: ..."}} — crashing our caller with "Cannot read properties of undefined (reading 'header')" at block.ts:21. Switched to the cosmos-REST equivalent /cosmos/base/tendermint/v1beta1/blocks/{height} so the call goes through fetchRest and inherits archive-header routing + retries. Verified: block 4895283 still returns 50 providers / \$13,199.08 in 54s. Block 4697952 (1 month old) now 5xx'es cleanly after exhausting retries against pruned replicas, instead of serving a partial 200. --- apps/api/src/rpc/block.ts | 18 +++++++++--------- apps/api/src/rpc/rest.ts | 34 ++++++++++++++++++++++------------ 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/apps/api/src/rpc/block.ts b/apps/api/src/rpc/block.ts index da49b8c..5e377b4 100644 --- a/apps/api/src/rpc/block.ts +++ b/apps/api/src/rpc/block.ts @@ -1,4 +1,5 @@ import { config } from "../config.js"; +import { fetchRest } from "./rest.js"; // Lava mainnet genesis timestamp (block 1) — used to narrow binary search range. const LAVA_GENESIS_UNIX = 1_713_350_000; // ~2024-04-17 @@ -9,16 +10,15 @@ const BLOCK_AT_TS_CACHE_MAX = 200; /** Fetch the timestamp of a given block height (ISO 8601 string). * Used for historical pricing — lets callers query CoinGecko for LAVA - * price at the block's date instead of using the current price. */ + * price at the block's date instead of using the current price. + * Routes through fetchRest so retries + the archive header kick in for + * old blocks; otherwise the tendermint-RPC proxy gateway-times-out on + * pruned replicas and leaves callers with a malformed response. */ export async function fetchBlockTime(height: number): Promise { - const res = await fetch(`${config.lava.rpcUrl}/block?height=${height}`, { - signal: AbortSignal.timeout(10_000), - }); - if (!res.ok) throw new Error(`RPC ${res.status}`); - const data = (await res.json()) as { - result: { block: { header: { time: string } } }; - }; - return data.result.block.header.time; + const data = await fetchRest<{ block: { header: { time: string } } }>( + `/cosmos/base/tendermint/v1beta1/blocks/${height}`, + ); + return data.block.header.time; } export async function fetchLatestBlockHeight(): Promise<{ diff --git a/apps/api/src/rpc/rest.ts b/apps/api/src/rpc/rest.ts index 118f5c3..af88546 100644 --- a/apps/api/src/rpc/rest.ts +++ b/apps/api/src/rpc/rest.ts @@ -84,13 +84,20 @@ function shouldRetry(statusOrNull: number | null): boolean { /** * Lava REST returns a 200 OK with `{success:false, message:"upstream_error: ..."}` * for two very different conditions: - * 1. The replica doesn't have the requested block in archive (pruned). Retry - * on a fresh connection — another replica may have it. - * 2. The chain genuinely has no data for the request (provider not found, - * empty rewards collection, etc). Don't retry — return the body so - * downstream processors handle it as "no data". + * 1. The replica can't serve the request at this block (pruned state, + * version mismatch, generic upstream failure). Retry on a fresh + * connection — another replica may have it. + * 2. The chain genuinely has no data for the request (e.g. the provider has + * no accrued rewards left after distribution). Don't retry — return the + * body so downstream processors handle it as "no data". * - * Returns "retry" for archive misses, "passthrough" for legitimate empty + * Default is RETRY. Passthrough requires an explicit whitelist match — a + * bare `collections: not found` is also what a partially-pruned replica + * returns when it can't read state, and treating that as legit-empty caused + * providers to silently drop from cached responses (same block, back-to-back + * cold fetches produced different provider counts). + * + * Returns "retry" for replica issues, "passthrough" for known legit-empty * answers, or null if the body isn't an error wrapper at all. */ function classifyChainUpstreamError(body: unknown): "retry" | "passthrough" | null { @@ -102,13 +109,16 @@ function classifyChainUpstreamError(body: unknown): "retry" | "passthrough" | nu return null; } const msg = ((body as { message?: unknown }).message as string | undefined) ?? ""; - // Archive pruning markers — replica doesn't have this block. - if (msg.includes("version does not exist") || msg.includes("version mismatch")) { - return "retry"; + // Explicit legit-empty markers. Only patterns the chain emits when the + // request itself has a well-formed "nothing to return" answer qualify as + // passthrough — everything else could be a silently-degraded replica and + // must retry. + // "cannot get claimable rewards after distribution" → provider has no + // remaining accrued amount at this block (normal state) + if (msg.includes("cannot get claimable rewards after distribution")) { + return "passthrough"; } - // Anything else is treated as a legitimate "no data" — passthrough so the - // caller sees an empty payload rather than a 5xx after wasted retries. - return "passthrough"; + return "retry"; } function backoffWithJitter(attempt: number): number {