diff --git a/apps/api/src/__tests__/routes-provider-estimated-rewards.test.ts b/apps/api/src/__tests__/routes-provider-estimated-rewards.test.ts index 5b5e9a2..4af7a0b 100644 --- a/apps/api/src/__tests__/routes-provider-estimated-rewards.test.ts +++ b/apps/api/src/__tests__/routes-provider-estimated-rewards.test.ts @@ -5,7 +5,8 @@ import { errorHandlerPlugin } from "../plugins/error-handler.js"; vi.mock("../rpc/lava.js", () => ({ RPC_BATCH_SIZE: 5, prewarmPriceCache: vi.fn(), - fetchProvidersWithSpecs: vi.fn(), + fetchAllSpecs: vi.fn(), + fetchAllProviderMonikers: vi.fn(), fetchRawProviderRewards: vi.fn(), extractBaseDenoms: vi.fn(), processRawProviderRewards: vi.fn(), @@ -15,9 +16,14 @@ vi.mock("../rpc/lava.js", () => ({ buildHistoricalPriceMap: vi.fn(), })); +vi.mock("../graphql/client.js", () => ({ + gqlSafe: vi.fn(), +})); + const { prewarmPriceCache, - fetchProvidersWithSpecs, + fetchAllSpecs, + fetchAllProviderMonikers, fetchRawProviderRewards, extractBaseDenoms, processRawProviderRewards, @@ -26,6 +32,7 @@ const { fetchLavaUsdPrice, buildHistoricalPriceMap, } = await import("../rpc/lava.js"); +const { gqlSafe } = await import("../graphql/client.js"); const { providerEstimatedRewardsRoutes } = await import("../routes/provider-estimated-rewards.js"); async function buildApp() { @@ -35,12 +42,16 @@ async function buildApp() { return app; } -const MOCK_PROVIDERS_WITH_SPECS = { - providers: new Map([ - ["lava@1abc", { moniker: "AlphaProvider", identity: "", commission: "50", specs: [] }], - ["lava@2def", { moniker: "BetaProvider", identity: "", commission: "30", specs: [] }], - ]), - specNames: new Map([["ETH1", "Ethereum Mainnet"], ["NEAR", "Near"]]), +const MOCK_SPECS = [ + { index: "ETH1", name: "Ethereum Mainnet" }, + { index: "NEAR", name: "Near" }, +]; +const MOCK_MONIKERS = new Map([ + ["lava@1abc", "AlphaProvider"], + ["lava@2def", "BetaProvider"], +]); +const MOCK_INDEXER_PROVIDERS = { + allProviders: { nodes: [{ addr: "lava@1abc" }, { addr: "lava@2def" }] }, }; const TOK = (amount: string, usd: string) => ({ @@ -90,7 +101,9 @@ beforeEach(() => { vi.resetAllMocks(); (prewarmPriceCache as ReturnType).mockResolvedValue(undefined); (fetchLavaUsdPrice as ReturnType).mockResolvedValue(0.12); - (fetchProvidersWithSpecs as ReturnType).mockResolvedValue(MOCK_PROVIDERS_WITH_SPECS); + (fetchAllSpecs as ReturnType).mockResolvedValue(MOCK_SPECS); + (fetchAllProviderMonikers as ReturnType).mockResolvedValue(MOCK_MONIKERS); + (gqlSafe as ReturnType).mockResolvedValue(MOCK_INDEXER_PROVIDERS); (fetchRawProviderRewards as ReturnType).mockResolvedValue(STUB_RAW); (extractBaseDenoms as ReturnType).mockResolvedValue(new Set(["lava"])); (processRawProviderRewards as ReturnType).mockImplementation( @@ -179,13 +192,9 @@ describe("GET /provider-estimated-rewards", () => { }); it("batches provider RPC calls 5 at a time", async () => { - const providers = new Map(); - for (let i = 0; i < 12; i++) { - providers.set(`lava@p${i}`, { moniker: `P${i}`, identity: "", commission: "50", specs: [] }); - } - (fetchProvidersWithSpecs as ReturnType).mockResolvedValue({ - providers, specNames: new Map(), - }); + const nodes = Array.from({ length: 12 }, (_, i) => ({ addr: `lava@p${i}` })); + (gqlSafe as ReturnType).mockResolvedValue({ allProviders: { nodes } }); + (fetchAllProviderMonikers as ReturnType).mockResolvedValue(new Map()); (processRawProviderRewards as ReturnType).mockResolvedValue([]); const app = await buildApp(); await app.inject({ method: "GET", url: "/provider-estimated-rewards" }); @@ -208,7 +217,11 @@ describe("GET /provider-estimated-rewards", () => { it("for historical blocks, fetches prices only for denoms actually in rewards", async () => { (extractBaseDenoms as ReturnType).mockResolvedValue(new Set(["lava", "atom"])); - (buildHistoricalPriceMap as ReturnType).mockResolvedValue({ lava: 0.035, atom: 8.2 }); + // LAVA fetch (kicked off in parallel with fan-out) returns lava; the + // follow-up non-LAVA fetch resolves the remaining denoms. + (buildHistoricalPriceMap as ReturnType) + .mockResolvedValueOnce({ lava: 0.035 }) + .mockResolvedValueOnce({ atom: 8.2 }); const app = await buildApp(); const res = await app.inject({ @@ -217,15 +230,15 @@ describe("GET /provider-estimated-rewards", () => { }); expect(res.statusCode).toBe(200); - // buildHistoricalPriceMap called with ONLY the relevant denoms (not all 22) - expect(buildHistoricalPriceMap).toHaveBeenCalledWith( - expect.any(Date), - expect.arrayContaining(["lava", "atom"]), - ); - const passedDenoms = (buildHistoricalPriceMap as ReturnType).mock.calls[0]![1] as string[]; - expect(passedDenoms.length).toBe(2); + // Two calls total: ['lava'] upfront + any non-LAVA denoms after fan-out. + // Never all 22 known denoms (that used to blow the gateway timeout). + expect(buildHistoricalPriceMap).toHaveBeenCalledTimes(2); + const first = (buildHistoricalPriceMap as ReturnType).mock.calls[0]![1] as string[]; + expect(first).toEqual(["lava"]); + const second = (buildHistoricalPriceMap as ReturnType).mock.calls[1]![1] as string[]; + expect(second).toEqual(["atom"]); - // Price overrides passed down to processRawProviderRewards + // Merged price overrides passed down to processRawProviderRewards for (const call of (processRawProviderRewards as ReturnType).mock.calls) { expect(call[2]).toEqual({ lava: 0.035, atom: 8.2 }); } @@ -248,20 +261,28 @@ describe("GET /provider-estimated-rewards", () => { expect(fetchLavaUsdPrice).not.toHaveBeenCalled(); }); - it("snapshots the provider set at the historical block", async () => { + it("uses the indexer's all-time provider list (includes unstaked historical providers)", async () => { + // Indexer has a provider that isn't currently staked (no moniker). Route + // should still query it at the historical block and render it with "-". + (gqlSafe as ReturnType).mockResolvedValue({ + allProviders: { nodes: [{ addr: "lava@1abc" }, { addr: "lava@2def" }, { addr: "lava@3ghost" }] }, + }); + mockProcessPerProvider(); (fetchBlockTime as ReturnType).mockResolvedValue("2026-03-17T15:00:00Z"); const app = await buildApp(); - await app.inject({ method: "GET", url: "/provider-estimated-rewards?block=4697952" }); - expect(fetchProvidersWithSpecs).toHaveBeenCalledWith(4697952); + const res = await app.inject({ method: "GET", url: "/provider-estimated-rewards?block=4697952" }); + expect(res.statusCode).toBe(200); + expect(fetchRawProviderRewards).toHaveBeenCalledTimes(3); + const queriedAddrs = (fetchRawProviderRewards as ReturnType).mock.calls.map((c) => c[0]); + expect(queriedAddrs).toEqual(expect.arrayContaining(["lava@1abc", "lava@2def", "lava@3ghost"])); }); - it("uses live prices + current provider set when ?block= is omitted", async () => { + it("uses live prices when ?block= is omitted", async () => { (fetchLavaUsdPrice as ReturnType).mockResolvedValue(0.025); const app = await buildApp(); const res = await app.inject({ method: "GET", url: "/provider-estimated-rewards" }); const body = JSON.parse(res.body); expect(body.meta.priceLavaUsd).toBe(0.025); - expect(fetchProvidersWithSpecs).toHaveBeenCalledWith(undefined); expect(buildHistoricalPriceMap).not.toHaveBeenCalled(); for (const call of (processRawProviderRewards as ReturnType).mock.calls) { expect(call[2]).toBeUndefined(); diff --git a/apps/api/src/routes/provider-estimated-rewards.ts b/apps/api/src/routes/provider-estimated-rewards.ts index 5cfa884..122b76e 100644 --- a/apps/api/src/routes/provider-estimated-rewards.ts +++ b/apps/api/src/routes/provider-estimated-rewards.ts @@ -4,16 +4,18 @@ import { RPC_BATCH_SIZE, buildHistoricalPriceMap, extractBaseDenoms, + fetchAllProviderMonikers, + fetchAllSpecs, fetchBlockAtTimestamp, fetchBlockTime, fetchLavaUsdPrice, - fetchProvidersWithSpecs, fetchRawProviderRewards, prewarmPriceCache, processRawProviderRewards, type EstimatedRewardsResponse, type RewardsBySpecEntry, } from "../rpc/lava.js"; +import { gqlSafe } from "../graphql/client.js"; import { sendApiError } from "../plugins/error-handler.js"; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -138,9 +140,6 @@ export async function providerEstimatedRewardsRoutes(app: FastifyInstance) { await prewarmPriceCache(); - // Compute block metadata upfront (needed for both path branches) and - // snapshot the provider set AT the block (when set) so providers who have - // since unstaked / deregistered still appear in historical responses. let priceTimestamp: string; let priceLavaUsd: number | undefined; const blockTimeIso = block ? await fetchBlockTime(block) : null; @@ -151,13 +150,40 @@ export async function providerEstimatedRewardsRoutes(app: FastifyInstance) { priceLavaUsd = await fetchLavaUsdPrice(); } - const { providers: providerMap, specNames } = await fetchProvidersWithSpecs(block); - const addresses = Array.from(providerMap.keys()); + // ── Resolve addresses ──────────────────────────────────────────────── + // Source of truth is the indexer's `app.providers` table — every provider + // that's ever been paid a relay, which is a superset of anyone who can + // have estimated rewards at a historical block. Monikers come from the + // current chain stake list (fast, cached); providers who've since unstaked + // won't have a moniker and render as "-" in the FE. + const [specs, indexerProviders, monikerMap] = await Promise.all([ + fetchAllSpecs(), + gqlSafe<{ allProviders: { nodes: Array<{ addr: string }> } } | null>( + `query { allProviders { nodes { addr } } }`, + undefined, + null, + ), + fetchAllProviderMonikers(), + ]); + const specNames = new Map(specs.map((s) => [s.index, s.name])); + const providerMap = new Map(); + const addresses: string[] = []; + for (const node of indexerProviders?.allProviders.nodes ?? []) { + addresses.push(node.addr); + providerMap.set(node.addr, { moniker: monikerMap.get(node.addr) ?? "" }); + } + + // ── LAVA price: kicked off here so it overlaps with the chain fan-out. + // ~99% of reward denoms are ulava; non-LAVA denoms that actually appear + // are fetched after fan-out inside the rare-path below. On live (non-block) + // queries we use the live cache, no override needed. + let priceOverrides: Record | undefined; + const lavaPricePromise: Promise> | null = + block && blockTimeIso + ? buildHistoricalPriceMap(new Date(blockTimeIso), ["lava"]) + : null; - // ── Pass 1: fetch raw rewards for every provider ───────────────────── - // Keeps the chain response so we can (a) inspect which denoms actually - // appear and (b) format the final output with the right prices, without - // paying for a second chain-RPC round. + // ── Chain fan-out: per-provider estimated_provider_rewards ────────── const rawByAddr = new Map(); for (let i = 0; i < addresses.length; i += RPC_BATCH_SIZE) { const batch = addresses.slice(i, i + RPC_BATCH_SIZE); @@ -165,38 +191,30 @@ export async function providerEstimatedRewardsRoutes(app: FastifyInstance) { batch.forEach((addr, j) => rawByAddr.set(addr, raws[j]!)); } - // ── Price overrides ────────────────────────────────────────────────── - // For historical blocks, fetch prices ONLY for denoms that appear in the - // rewards (typically just LAVA + 0-2 IBC denoms). Fetching all 22 known - // denoms sequentially with retry backoff would exceed the gateway timeout - // under any CoinGecko throttling. - let priceOverrides: Record | undefined; - if (block && blockTimeIso) { + if (lavaPricePromise) { + try { + priceOverrides = await lavaPricePromise; + } catch (err) { + // Cache plugin skips 4xx/5xx — refusing to ship avoids poisoning the + // block-keyed cache with a wrong price for a year. + return sendApiError( + reply, 503, + `historical LAVA price unavailable for block ${block} (${blockTimeIso}); please retry: ${(err as Error).message}`, + ); + } + + // Rare non-LAVA denoms that actually appeared in this block. IBC + // resolution hits chain denom_trace (cached), so this is cheap. const relevantDenoms = await extractBaseDenoms([...rawByAddr.values()]); - if (relevantDenoms.size === 0) { - // No priceable rewards in this block — nothing to override. - priceOverrides = {}; - priceLavaUsd = 0; - } else { - try { - priceOverrides = await buildHistoricalPriceMap( - new Date(blockTimeIso), - [...relevantDenoms], - ); - } catch (err) { - // LAVA historical price unavailable. Refuse to ship a response — - // the cache plugin skips 4xx/5xx so we don't poison the block-keyed - // cache with a wrong price for a year. - return sendApiError( - reply, 503, - `historical LAVA price unavailable for block ${block} (${blockTimeIso}); please retry: ${(err as Error).message}`, - ); - } - priceLavaUsd = priceOverrides.lava ?? await fetchLavaUsdPrice(); + const missing = [...relevantDenoms].filter((d) => priceOverrides![d] === undefined); + if (missing.length > 0) { + const extra = await buildHistoricalPriceMap(new Date(blockTimeIso!), missing); + priceOverrides = { ...priceOverrides, ...extra }; } + priceLavaUsd = priceOverrides.lava ?? priceLavaUsd ?? await fetchLavaUsdPrice(); } - // ── Pass 2: format each provider's raw response with the right prices ─ + // ── Format each provider's raw response ───────────────────────────── const results: Array<{ provider: string; moniker: string; diff --git a/apps/api/src/rpc/block.ts b/apps/api/src/rpc/block.ts index da49b8c..5e377b4 100644 --- a/apps/api/src/rpc/block.ts +++ b/apps/api/src/rpc/block.ts @@ -1,4 +1,5 @@ import { config } from "../config.js"; +import { fetchRest } from "./rest.js"; // Lava mainnet genesis timestamp (block 1) — used to narrow binary search range. const LAVA_GENESIS_UNIX = 1_713_350_000; // ~2024-04-17 @@ -9,16 +10,15 @@ const BLOCK_AT_TS_CACHE_MAX = 200; /** Fetch the timestamp of a given block height (ISO 8601 string). * Used for historical pricing — lets callers query CoinGecko for LAVA - * price at the block's date instead of using the current price. */ + * price at the block's date instead of using the current price. + * Routes through fetchRest so retries + the archive header kick in for + * old blocks; otherwise the tendermint-RPC proxy gateway-times-out on + * pruned replicas and leaves callers with a malformed response. */ export async function fetchBlockTime(height: number): Promise { - const res = await fetch(`${config.lava.rpcUrl}/block?height=${height}`, { - signal: AbortSignal.timeout(10_000), - }); - if (!res.ok) throw new Error(`RPC ${res.status}`); - const data = (await res.json()) as { - result: { block: { header: { time: string } } }; - }; - return data.result.block.header.time; + const data = await fetchRest<{ block: { header: { time: string } } }>( + `/cosmos/base/tendermint/v1beta1/blocks/${height}`, + ); + return data.block.header.time; } export async function fetchLatestBlockHeight(): Promise<{ diff --git a/apps/api/src/rpc/providers.ts b/apps/api/src/rpc/providers.ts index 6b0bc89..31adfcf 100644 --- a/apps/api/src/rpc/providers.ts +++ b/apps/api/src/rpc/providers.ts @@ -119,10 +119,13 @@ export async function fetchProvidersWithSpecs( for (let i = 0; i < allSpecs.length; i += RPC_BATCH_SIZE) { const batch = allSpecs.slice(i, i + RPC_BATCH_SIZE); + // No .catch() — fetchRest now retries transient failures (5xx, 429, + // network) up to 3 times. If it still throws, we'd rather 500 loudly + // than silently drop providers and cache an incomplete response. The + // cache plugin skips 4xx/5xx so a failed request never poisons Redis. const results = await Promise.all( batch.map((s) => fetchProvidersForSpec(s.index, blockHeight) - .then((ps) => ps.map((p) => ({ ...p, specId: s.index, specName: s.name }))) - .catch(() => [] as Array)), + .then((ps) => ps.map((p) => ({ ...p, specId: s.index, specName: s.name })))), ); for (const providers of results) { @@ -222,6 +225,9 @@ async function fetchAllProvidersImpl(): Promise { for (let i = 0; i < specs.length; i += RPC_BATCH_SIZE) { const batch = specs.slice(i, i + RPC_BATCH_SIZE); const results = await Promise.all( + // Same rationale as fetchProvidersWithSpecs: no .catch(). fetchRest + // retries transient failures; if it still throws, fail loud rather + // than silently drop providers from /providers or /all_providers_apr. batch.map((s) => fetchProvidersForSpec(s.index) .then((providers) => providers.map((p): SpecEntry => ({ @@ -232,8 +238,7 @@ async function fetchAllProvidersImpl(): Promise { delegate_total: p.delegate_total, delegate_commission: p.delegate_commission, specId: s.index, - }))) - .catch(() => [] as SpecEntry[]), + }))), ), ); specProviders.push(...results); diff --git a/apps/api/src/rpc/rest.ts b/apps/api/src/rpc/rest.ts index 47e275f..af88546 100644 --- a/apps/api/src/rpc/rest.ts +++ b/apps/api/src/rpc/rest.ts @@ -1,8 +1,23 @@ import pino from "pino"; +import { Agent } from "undici"; import { config } from "../config.js"; export const logger = pino({ name: "rpc" }); +// One-shot dispatcher used only for retry attempts after a pruned-replica +// error. The default global dispatcher pools 32 sockets per origin for 30s, +// which means every retry inside that window reuses the SAME TCP connection +// and lands on the same backend (the one that doesn't have the block). A +// dispatcher with `keepAliveTimeout: 0` and `pipelining: 0` opens — and +// closes — one connection per request, giving Cloudflare a fresh routing +// decision and a chance at a healthy archive replica. +const noKeepAliveDispatcher = new Agent({ + connections: 1, + pipelining: 0, + keepAliveTimeout: 1, + keepAliveMaxTimeout: 1, +}); + /** * Concurrent in-flight RPC calls per batch loop. Operators running dedicated * RPC endpoints can raise this (env: RPC_BATCH_SIZE). Public endpoints should @@ -13,6 +28,163 @@ export const RPC_BATCH_SIZE = config.lava.rpcBatchSize; // Request coalescing: concurrent fetches for the same path share one in-flight request const inflightRpc = new Map>(); +// Archive-routing threshold. Recent blocks live on every replica; older blocks +// risk hitting a non-archive replica that has pruned the state and responds +// with `{success:false, "version does not exist"}`. Empirically the public LB +// starts losing blocks well within a week, so keep a tight 7-day window — +// archive nodes are heavier so we still want to avoid them for fresh queries. +// Lava produces ~1 block per 30s → 7 days ≈ 20_160 blocks. +const ARCHIVE_THRESHOLD_BLOCKS = 20_160; +const LATEST_HEIGHT_CACHE_MS = 60_000; +let latestHeightCache: { height: number; at: number } | null = null; + +async function getLatestHeight(): Promise { + if (latestHeightCache && Date.now() - latestHeightCache.at < LATEST_HEIGHT_CACHE_MS) { + return latestHeightCache.height; + } + // No blockHeight → won't trigger the archive-routing logic, no recursion. + const data = await fetchRest<{ block: { header: { height: string } } }>( + "/cosmos/base/tendermint/v1beta1/blocks/latest", + ); + const height = parseInt(data.block.header.height, 10); + latestHeightCache = { height, at: Date.now() }; + return height; +} + +// ── Retry policy ───────────────────────────────────────────────────────────── +// Transient failures on the public chain RPC (network blips, rate limits on +// concurrent historical queries, archive-node slow paths) previously returned +// errors that upstream callers silently swallowed via `.catch(() => [])`, +// dropping providers from historical rewards responses. Retry with exponential +// backoff so transient blips don't leak into user-visible output. +// +// Retry on "likely transient" failures: +// • Network error (fetch rejected — timeout, connection reset, DNS, etc.) +// • 408 Request Timeout +// • 429 Too Many Requests (honor Retry-After if present) +// • 5xx Server Error +// • Chain LB body-level error (200 OK with `{success:false, message:"upstream_error: ..."}`) +// The lava REST gateway fronts multiple archive nodes; some have older +// blocks pruned and respond with this wrapper instead of the expected +// payload. Retries on a fresh connection (Connection: close + jittered +// backoff) usually land on a healthy replica. +// 4xx client errors (including 404) are NOT retried — wrong path/params, +// retrying won't help. +const RPC_MAX_ATTEMPTS = 10; +const RPC_MAX_BACKOFF_MS = 5_000; +const RPC_REQUEST_TIMEOUT_MS = 20_000; + +function shouldRetry(statusOrNull: number | null): boolean { + if (statusOrNull === null) return true; // network-level error + if (statusOrNull === 408 || statusOrNull === 429) return true; + if (statusOrNull >= 500 && statusOrNull < 600) return true; + return false; +} + +/** + * Lava REST returns a 200 OK with `{success:false, message:"upstream_error: ..."}` + * for two very different conditions: + * 1. The replica can't serve the request at this block (pruned state, + * version mismatch, generic upstream failure). Retry on a fresh + * connection — another replica may have it. + * 2. The chain genuinely has no data for the request (e.g. the provider has + * no accrued rewards left after distribution). Don't retry — return the + * body so downstream processors handle it as "no data". + * + * Default is RETRY. Passthrough requires an explicit whitelist match — a + * bare `collections: not found` is also what a partially-pruned replica + * returns when it can't read state, and treating that as legit-empty caused + * providers to silently drop from cached responses (same block, back-to-back + * cold fetches produced different provider counts). + * + * Returns "retry" for replica issues, "passthrough" for known legit-empty + * answers, or null if the body isn't an error wrapper at all. + */ +function classifyChainUpstreamError(body: unknown): "retry" | "passthrough" | null { + if ( + typeof body !== "object" || + body === null || + (body as { success?: unknown }).success !== false + ) { + return null; + } + const msg = ((body as { message?: unknown }).message as string | undefined) ?? ""; + // Explicit legit-empty markers. Only patterns the chain emits when the + // request itself has a well-formed "nothing to return" answer qualify as + // passthrough — everything else could be a silently-degraded replica and + // must retry. + // "cannot get claimable rewards after distribution" → provider has no + // remaining accrued amount at this block (normal state) + if (msg.includes("cannot get claimable rewards after distribution")) { + return "passthrough"; + } + return "retry"; +} + +function backoffWithJitter(attempt: number): number { + const base = Math.min(RPC_MAX_BACKOFF_MS, 2 ** attempt * 300); + return base + Math.floor(Math.random() * 500); +} + +async function fetchWithRetry(url: string, baseHeaders: Record): Promise { + let lastErr: unknown; + for (let attempt = 1; attempt <= RPC_MAX_ATTEMPTS; attempt++) { + // After the first attempt force a brand-new TCP connection via a no-pool + // dispatcher. The default global pool keeps sockets alive for ~30s; reusing + // any of them means landing on the same backend that just rejected us. + const fetchOpts: RequestInit = { + headers: attempt === 1 ? baseHeaders : { ...baseHeaders, Connection: "close" }, + signal: AbortSignal.timeout(RPC_REQUEST_TIMEOUT_MS), + }; + if (attempt > 1) (fetchOpts as { dispatcher?: unknown }).dispatcher = noKeepAliveDispatcher; + try { + const res = await fetch(url, fetchOpts); + if (res.ok) { + const body = (await res.json()) as unknown; + const cls = classifyChainUpstreamError(body); + if (cls === "retry") { + const message = (body as { message?: string }).message; + if (attempt === RPC_MAX_ATTEMPTS) { + throw new Error( + `chain upstream error after ${RPC_MAX_ATTEMPTS} attempts: ${message ?? "(no message)"}`, + ); + } + const backoffMs = backoffWithJitter(attempt); + logger.warn( + { url, attempt, backoffMs, message }, + "chain upstream returned success=false (pruned replica), retrying on fresh connection", + ); + await new Promise((r) => setTimeout(r, backoffMs)); + continue; + } + // cls === "passthrough" → legitimate empty answer; cls === null → normal payload. + return body as T; + } + if (!shouldRetry(res.status)) { + throw new Error(`RPC ${res.status}: ${res.statusText}`); + } + + if (attempt === RPC_MAX_ATTEMPTS) { + throw new Error(`RPC ${res.status}: ${res.statusText}`); + } + + const retryAfter = parseInt(res.headers.get("retry-after") ?? "0", 10); + const backoffMs = retryAfter > 0 + ? Math.min(RPC_MAX_BACKOFF_MS, retryAfter * 1000) + : backoffWithJitter(attempt); + logger.warn({ url, status: res.status, attempt, backoffMs }, "chain RPC retryable error, backing off"); + await new Promise((r) => setTimeout(r, backoffMs)); + } catch (e) { + lastErr = e; + if (attempt === RPC_MAX_ATTEMPTS) throw e; + const backoffMs = backoffWithJitter(attempt); + logger.warn({ url, attempt, err: String(e) }, "chain RPC error, retrying"); + await new Promise((r) => setTimeout(r, backoffMs)); + } + } + throw lastErr ?? new Error("fetchWithRetry: unreachable"); +} + export async function fetchRest(path: string, blockHeight?: number): Promise { const cacheKey = blockHeight ? `${path}@${blockHeight}` : path; const existing = inflightRpc.get(cacheKey); @@ -20,14 +192,19 @@ export async function fetchRest(path: string, blockHeight?: number): Promise< const promise = (async () => { const headers: Record = {}; - if (blockHeight) headers["x-cosmos-block-height"] = String(blockHeight); - - const res = await fetch(`${config.lava.restUrl}${path}`, { - headers, - signal: AbortSignal.timeout(15_000), - }); - if (!res.ok) throw new Error(`RPC ${res.status}: ${res.statusText}`); - return (await res.json()) as T; + if (blockHeight) { + headers["x-cosmos-block-height"] = String(blockHeight); + // Pin OLD historical queries to archive replicas. Without the header the + // public LB freely routes to non-archive replicas that have pruned the + // requested block and return `{success:false, "version does not exist"}`. + // Recent blocks are still on every replica, and archive nodes are heavier + // / under more load — so only opt in when the block is genuinely old. + const latest = await getLatestHeight().catch(() => null); + if (latest !== null && latest - blockHeight > ARCHIVE_THRESHOLD_BLOCKS) { + headers["lava-extension"] = "archive"; + } + } + return await fetchWithRetry(`${config.lava.restUrl}${path}`, headers); })(); inflightRpc.set(cacheKey, promise); diff --git a/apps/api/src/rpc/rewards.ts b/apps/api/src/rpc/rewards.ts index ca039e6..cd67375 100644 --- a/apps/api/src/rpc/rewards.ts +++ b/apps/api/src/rpc/rewards.ts @@ -149,19 +149,19 @@ export interface RewardsBySpecEntry { } /** Fetch the raw estimated_provider_rewards chain response for one provider. - * Returns an empty structure on error so callers can iterate without throwing. */ + * Throws on failure (after fetchRest's internal retries are exhausted) so the + * caller can fail loud — historical rewards responses are cached for a year, + * so silently dropping a provider on transient RPC flakes would bake a wrong + * response into the cache. The cache plugin skips 4xx/5xx, so the next caller + * retries from scratch. */ export async function fetchRawProviderRewards( address: string, blockHeight?: number, ): Promise { - try { - return await fetchRest( - `/lavanet/lava/subscription/estimated_provider_rewards/${address}/`, - blockHeight, - ); - } catch { - return { info: [], total: [] }; - } + return await fetchRest( + `/lavanet/lava/subscription/estimated_provider_rewards/${address}/`, + blockHeight, + ); } /** Extract the set of base denoms (post-IBC-resolution) that appear in one or @@ -276,13 +276,11 @@ export async function processRawProviderRewards( } /** - * Fetch a provider's actual earned rewards (no benchmark amount) and group by spec. - * Composes fetchRawProviderRewards + processRawProviderRewards for callers that - * don't need the raw/extract/process split (e.g. APR's computeAllProvidersApr). - * - * Pass blockHeight to query historical chain state (archive node required). - * Pass priceOverrides (keyed by base denom, e.g. "lava") to price tokens at a - * specific point in time. + * Best-effort variant for live-dashboard consumers (APR). Catches per-provider + * RPC failures and returns [] so one flaky provider doesn't 500 the whole APR + * computation. The new historical-rewards route calls fetchRawProviderRewards + * directly and lets failures propagate, because incomplete data would be + * cached for a year under CACHE_TTL.IMMUTABLE. */ export async function fetchRewardsBySpec( address: string, @@ -290,8 +288,12 @@ export async function fetchRewardsBySpec( blockHeight?: number, priceOverrides?: Record, ): Promise { - const raw = await fetchRawProviderRewards(address, blockHeight); - return processRawProviderRewards(raw, specNameMap, priceOverrides); + try { + const raw = await fetchRawProviderRewards(address, blockHeight); + return await processRawProviderRewards(raw, specNameMap, priceOverrides); + } catch { + return []; + } } export interface ClaimableRewardEntry {