From bed596f12591a9056a144cb88cd4d330cf576060 Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Sat, 18 Apr 2026 16:41:24 +0300 Subject: [PATCH] feat: honest gap accounting + in-run dead-letter retry + tip-first interleave --- cmd/indexer/main.go | 28 +++++++++ config.example.yml | 24 ++++++++ internal/config/config.go | 46 ++++++++++++--- internal/pipeline/pipeline.go | 95 +++++++++++++++++++++++++----- internal/state/state.go | 104 +++++++++++++++++++++++++++++++++ internal/web/assets/index.html | 15 ++++- internal/web/server.go | 57 +++++++++++++++--- 7 files changed, 339 insertions(+), 30 deletions(-) diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go index 87e64fa..7bd12aa 100644 --- a/cmd/indexer/main.go +++ b/cmd/indexer/main.go @@ -173,6 +173,34 @@ func main() { slog.Warn("aggregates apply failed (non-fatal)", "err", err) } + // Optional: drop state for heights outside the current window. Used + // when the operator has narrowed start_height (or dropped earliest_mode) + // after a previous run populated things below the new window. One-shot + // and idempotent — if there's nothing to prune the counts are 0. + if cfg.Indexer.PruneOutsideWindow && !cfg.Indexer.EarliestMode { + pruneEnd := cfg.Indexer.EndHeight + if pruneEnd == 0 { + if tip, err := client.Tip(ctx); err == nil { + pruneEnd = tip + } + } + if pruneEnd > 0 { + for _, h := range reg.All() { + rr, ff, err := st.PruneOutsideWindow(ctx, h.Name(), cfg.Indexer.StartHeight, pruneEnd) + if err != nil { + slog.Warn("prune outside window failed", "handler", h.Name(), "err", err) + continue + } + if rr > 0 || ff > 0 { + slog.Info("pruned state outside window", + "handler", h.Name(), + "start", cfg.Indexer.StartHeight, "end", pruneEnd, + "ranges_deleted", rr, "failures_deleted", ff) + } + } + } + } + pipe := pipeline.New(cfg, pool, client, st, reg) if sizer != nil { pipe = pipe.WithSizer(sizer) diff --git a/config.example.yml b/config.example.yml index 4950a9b..85050ae 100644 --- a/config.example.yml +++ b/config.example.yml @@ -88,6 +88,30 @@ indexer: # How often the tip-follower polls /status after catching up. tip_poll_interval: 3s + # Dead-letter retry loop: how often the pipeline sweeps app.indexer_failures + # and re-enqueues failed heights. 0 disables (dead-letters then only get + # another chance on restart or when tip-follow re-picks them up). Default + # 60s catches transient saturation without churning the DB. + failure_retry_interval: 60s + + # Max failed heights retried per sweep. Smaller = gentler on the RPC fleet, + # larger = faster drain. + failure_retry_batch: 200 + + # If true, on startup delete indexer_ranges + indexer_failures rows for + # heights OUTSIDE [start_height, end_height|tip]. Useful when narrowing + # start_height after a previous wider run — otherwise the stale state + # shows up as zombie "covered ranges" on the timeline that will never + # be touched again. + prune_outside_window: false + + # Gap scheduler interleave. Most batches go to the highest-cursor gap + # (tip-first) but every Nth batch goes to the LOWEST-cursor gap — so a + # big historical-backfill gap can't starve behind thousands of + # tip-close fragments. 0 or 1 = pure tip-first (fine if you only ever + # have one gap). Default 5 ≈ 80/20 split in favour of tip work. + backfill_interleave: 5 + database: host: postgres port: 5432 diff --git a/internal/config/config.go b/internal/config/config.go index 39e5547..2cb4b0d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -84,6 +84,33 @@ type Indexer struct { WriteBatchRows int `yaml:"write_batch_rows"` QueueDepth int `yaml:"queue_depth"` TipPollInterval time.Duration `yaml:"tip_poll_interval"` + + // PruneOutsideWindow, if true, deletes any indexer_ranges and + // indexer_failures entries for heights outside [StartHeight, EndHeight|tip] + // on startup. Useful when the operator narrowed the window after a + // previous (wider) run and wants the stale state dropped so gap math + // reflects the new window. One-shot per startup; safe to leave on. + PruneOutsideWindow bool `yaml:"prune_outside_window"` + + // FailureRetryInterval controls how often the pipeline sweeps the + // dead-letter table and re-queues failed heights during an active + // run. 0 disables in-run retry (old behaviour: dead-letters wait + // until a restart or until tip-follow re-picks them up). Default 60s + // catches transient saturation failures without hammering the DB. + FailureRetryInterval time.Duration `yaml:"failure_retry_interval"` + + // FailureRetryBatch is the max number of failed heights retried per + // sweep. Smaller = gentler on the RPC endpoints, larger = faster + // drain. Default 200. + FailureRetryBatch int `yaml:"failure_retry_batch"` + + // BackfillInterleave controls how strictly tip-first the gap scheduler + // is. Every Nth batch goes to the OLDEST-cursor gap instead of the + // highest, so a large backfill gap can't starve behind tip-close + // fragments. 0 or 1 disables the interleave (pure tip-first — good + // when you only ever have one gap). Default 5 → 80/20 split in + // favour of tip-close work. + BackfillInterleave int `yaml:"backfill_interleave"` } // WantsHandler reports whether `name` should be registered given the @@ -137,14 +164,17 @@ func Load(path string) (*Config, error) { func defaults() *Config { return &Config{ Indexer: Indexer{ - StartHeight: 1, - EndHeight: 0, - TipConfirmations: 0, // Cosmos/CometBFT has deterministic finality - FetchWorkers: 16, - FetchBatchSize: 10, - WriteBatchRows: 20000, - QueueDepth: 200, - TipPollInterval: 3 * time.Second, + StartHeight: 1, + EndHeight: 0, + TipConfirmations: 0, // Cosmos/CometBFT has deterministic finality + FetchWorkers: 16, + FetchBatchSize: 10, + WriteBatchRows: 20000, + QueueDepth: 200, + TipPollInterval: 3 * time.Second, + FailureRetryInterval: 60 * time.Second, + FailureRetryBatch: 200, + BackfillInterleave: 5, }, Database: Database{ Host: "localhost", diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index 75d21cd..abfc043 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -205,33 +205,49 @@ func (p *Pipeline) indexGaps(ctx context.Context, gaps []state.Range) error { g, ctx := errgroup.WithContext(ctx) - // Single producer, tip-first: on every emit we pick the gap whose - // current cursor is HIGHEST — i.e. closest to the chain tip — and - // dispatch the next batch from there. Consequences: - // - With a historical-backfill gap (e.g. [6mo-ago, pruning-horizon]) - // plus a live tip-extension gap, the live gap always wins. The - // backfill only gets worker attention once the tip gap is caught - // up, so the observable "latest indexed height" tracks the chain - // head even while history is still being filled. - // - When tip-follower extends the top gap on each poll, those new - // heights immediately jump to the front of the queue; they don't - // sit behind half a million backfill blocks. - // - Within a single gap the behaviour is unchanged: bottom-up walk - // from gap.From to gap.To. + // Single producer, tip-first with interleave: most batches go to the + // gap with the HIGHEST cursor (closest to tip), but every Nth batch + // goes to the LOWEST-cursor gap instead, so a big historical gap + // can't starve behind thousands of tip-close fragments. + // + // Why not pure tip-first: if gaps are highly fragmented near tip + // (dead-letter holes × thousands) the tip-first preference keeps + // rotating through those fragments for hours while a single large + // bottom gap gets zero cycles. Interleave every N batches lets the + // bottom gap drain in parallel at ~(1/N) of the throughput without + // losing the "tip tracks head" property. + // + // BackfillInterleave: 0 or 1 → pure tip-first. Default 5 → ~80/20. + // Within a single gap the walk is still bottom-up From → To. g.Go(func() error { defer close(jobs) cursors := make([]int64, len(gaps)) for i := range gaps { cursors[i] = gaps[i].From } + interleave := cfg.BackfillInterleave + batchIdx := 0 for { + // Every `interleave`-th batch, pick the LOWEST-cursor gap + // (oldest work) instead of the highest. + pickLowest := interleave > 1 && (batchIdx%interleave) == interleave-1 best := -1 for i := range gaps { if cursors[i] > gaps[i].To { continue } - if best < 0 || cursors[i] > cursors[best] { + if best < 0 { best = i + continue + } + if pickLowest { + if cursors[i] < cursors[best] { + best = i + } + } else { + if cursors[i] > cursors[best] { + best = i + } } } if best < 0 { @@ -248,9 +264,60 @@ func (p *Pipeline) indexGaps(ctx context.Context, gaps []state.Range) error { case jobs <- job{from: cursors[best], to: to}: } cursors[best] = to + 1 + batchIdx++ } }) + // Dead-letter retry sweep: the pipeline's processWithSplit records any + // size-1 block that exhausts its retries into app.indexer_failures and + // moves on. Historically those heights stayed "failed" until the process + // restarted — no good when a multi-day backfill briefly saturates the + // endpoints and racks up thousands of transient holes. + // + // This goroutine periodically pops the newest-first batch of failures, + // deletes their rows, and runs them back through processWithSplit. If + // the retry succeeds the height lands in indexer_ranges (so it stops + // being a gap); if it fails again, processWithSplit re-inserts the + // row with retries+1 and it gets another chance on the next sweep. + // + // Runs in parallel with the main tip-first producer — uses the same + // pgxpool + processWithSplit, so it shares the AIMD budget and advisory + // locks naturally. No separate worker pool needed; dead-letter retry + // is rarely the bottleneck. + if cfg.FailureRetryInterval > 0 && cfg.FailureRetryBatch > 0 { + g.Go(func() error { + ticker := time.NewTicker(cfg.FailureRetryInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + for _, handler := range p.registry.All() { + heights, err := p.state.PopFailures(ctx, handler.Name(), cfg.FailureRetryBatch) + if err != nil { + slog.Warn("dead-letter sweep: pop failed", "handler", handler.Name(), "err", err) + continue + } + if len(heights) == 0 { + continue + } + slog.Info("dead-letter sweep: retrying heights", + "handler", handler.Name(), "count", len(heights)) + for _, h := range heights { + if ctx.Err() != nil { + return nil + } + // Swallow errors: processWithSplit only returns ctx + // errors; real failures re-insert into indexer_failures. + _ = p.processWithSplit(ctx, h, h) + } + } + } + }) + } + // Consumers. start := time.Now() var progressBlocks atomic.Int64 diff --git a/internal/state/state.go b/internal/state/state.go index adb342a..fd2b6b2 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -104,6 +104,110 @@ func (s *State) FailureCount(ctx context.Context, handler string) (count int64, return } +// PopFailures removes up to `limit` dead-letter heights for `handler` +// (highest heights first — tip-first just like the main producer) and +// returns them. Atomic: the SELECT + DELETE happens inside a single +// DELETE…RETURNING so a second caller can't pop the same height. +// +// Heights returned here are candidates for in-run retry. If the retry +// eventually succeeds, the height lands in indexer_ranges and the failure +// is gone for good. If it fails again, processWithSplit re-inserts it +// with retries+1, putting it back in the dead-letter pool for the next +// cycle. +func (s *State) PopFailures(ctx context.Context, handler string, limit int) ([]int64, error) { + if limit <= 0 { + return nil, nil + } + rows, err := s.pool.Query(ctx, fmt.Sprintf(` + DELETE FROM %[1]s.indexer_failures + WHERE (handler, height) IN ( + SELECT handler, height + FROM %[1]s.indexer_failures + WHERE handler = $1 + ORDER BY height DESC + LIMIT $2 + ) + RETURNING height`, s.schema), handler, limit) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]int64, 0, limit) + for rows.Next() { + var h int64 + if err := rows.Scan(&h); err != nil { + return nil, err + } + out = append(out, h) + } + return out, rows.Err() +} + +// PruneOutsideWindow deletes indexer_ranges + indexer_failures entries +// that fall entirely outside [start, end] for the given handler. Used on +// startup when the operator narrows the start_height and wants the stale +// state from a previous (wider) run removed so the UI + gap math stay +// honest about the new window. Ranges that STRADDLE the boundary are +// left untouched — they still contain blocks inside the new window. +// +// Returns (ranges_deleted, failures_deleted). +func (s *State) PruneOutsideWindow(ctx context.Context, handler string, start, end int64) (int64, int64, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return 0, 0, err + } + defer tx.Rollback(ctx) + + // Advisory lock: prevents the pipeline's range-merge from racing with + // the prune on the same handler. + if _, err := tx.Exec(ctx, + `SELECT pg_advisory_xact_lock(hashtext($1))`, handler); err != nil { + return 0, 0, err + } + + rTag, err := tx.Exec(ctx, fmt.Sprintf(` + DELETE FROM %s.indexer_ranges + WHERE handler = $1 AND (to_height < $2 OR from_height > $3)`, s.schema), + handler, start, end) + if err != nil { + return 0, 0, err + } + fTag, err := tx.Exec(ctx, fmt.Sprintf(` + DELETE FROM %s.indexer_failures + WHERE handler = $1 AND (height < $2 OR height > $3)`, s.schema), + handler, start, end) + if err != nil { + return 0, 0, err + } + if err := tx.Commit(ctx); err != nil { + return 0, 0, err + } + return rTag.RowsAffected(), fTag.RowsAffected(), nil +} + +// FailureHeights returns the individual dead-letter heights in [start, end] +// for a handler. Used by the web layer to subtract them from "untouched" +// gaps so the UI distinguishes never-tried gaps from retry-waiting ones. +func (s *State) FailureHeights(ctx context.Context, handler string, start, end int64) ([]int64, error) { + rows, err := s.pool.Query(ctx, fmt.Sprintf(` + SELECT height FROM %s.indexer_failures + WHERE handler = $1 AND height BETWEEN $2 AND $3 + ORDER BY height`, s.schema), handler, start, end) + if err != nil { + return nil, err + } + defer rows.Close() + out := make([]int64, 0, 256) + for rows.Next() { + var h int64 + if err := rows.Scan(&h); err != nil { + return nil, err + } + out = append(out, h) + } + return out, rows.Err() +} + // FailureRanges coalesces individual dead-letter heights into contiguous // ranges so the UI can paint them as red bars on the timeline instead of // 1-pixel spikes. diff --git a/internal/web/assets/index.html b/internal/web/assets/index.html index 3a5d796..5758b36 100644 --- a/internal/web/assets/index.html +++ b/internal/web/assets/index.html @@ -174,6 +174,9 @@ details.ranges-detail .list span.gap { color: var(--warn); border-color: #3b2a0d; } + details.ranges-detail .list span.failed-range { + color: var(--danger); border-color: #3b0d0d; + } footer { padding: 18px 24px; color: var(--muted); font-size: 11px; border-top: 1px solid var(--border); margin-top: 40px; text-align: center; } @@ -1048,10 +1051,20 @@

Handlers

card.axisDateR.textContent = tipDateText; card.axisDateR.style.display = tipDateText ? '' : 'none'; - card.summary.textContent = `${ranges.length} covered ranges, ${(h.gaps || []).length} gap${(h.gaps||[]).length === 1 ? '' : 's'}`; + // Two separate counts: untouched gaps (never attempted) and dead-letter + // holes (attempted, failed, waiting for retry). Historically both were + // lumped as "gaps" which made a transiently-saturated fleet look like + // a huge unindexed hole when actually most of the window WAS attempted. + const gapCount = (h.gaps || []).length; + const failedCount = (h.failed_ranges || []).length; + const parts = [`${fmt(ranges.length)} covered range${ranges.length === 1 ? '' : 's'}`]; + if (gapCount) parts.push(`${fmt(gapCount)} untouched`); + if (failedCount) parts.push(`${fmt(failedCount)} dead-letter range${failedCount === 1 ? '' : 's'}`); + card.summary.textContent = parts.join(' · '); card.list.innerHTML = ''; ranges.forEach(r => card.list.appendChild(el('span', {}, `${fmt(r.from)} → ${fmt(r.to)}`))); (h.gaps || []).forEach(r => card.list.appendChild(el('span', { class: 'gap' }, `gap ${fmt(r.from)} → ${fmt(r.to)}`))); + (h.failed_ranges || []).forEach(r => card.list.appendChild(el('span', { class: 'failed-range' }, `dead-letter ${fmt(r.from)}${r.from !== r.to ? ' → ' + fmt(r.to) : ''}`))); } function renderHeader(s) { diff --git a/internal/web/server.go b/internal/web/server.go index 83a390d..b1f8dc9 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -195,6 +195,9 @@ type HandlerStatus struct { Name string `json:"name"` EventTypes []string `json:"event_types"` Ranges []RangeJSON `json:"ranges"` + // Gaps is "never-tried" heights only: uncovered AND not currently in + // the dead-letter pool. FailedRanges holds the dead-letter side so + // the UI can report them separately with different semantics. Gaps []RangeJSON `json:"gaps"` WindowPct float64 `json:"window_pct"` // % of configured [start, end∨tip] ChainPct float64 `json:"chain_pct"` // % of full chain [1, tip] @@ -205,10 +208,10 @@ type HandlerStatus struct { RowCounts map[string]int64 `json:"row_counts"` // Dead-letter: heights that exhausted their retry budget. Still gaps - // in indexer_ranges, so a future run picks them up automatically. - FailureCount int64 `json:"failure_count"` - MaxRetries int `json:"max_retries"` - FailedRanges []RangeJSON `json:"failed_ranges,omitempty"` + // in indexer_ranges, so a future sweep (or restart) picks them up. + FailureCount int64 `json:"failure_count"` + MaxRetries int `json:"max_retries"` + FailedRanges []RangeJSON `json:"failed_ranges,omitempty"` } type RangeJSON struct { @@ -340,11 +343,20 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { hs.CoveredStart = hs.Ranges[0].From hs.CoveredEnd = hs.Ranges[len(hs.Ranges)-1].To } - // Gaps within [start, windowEnd] + // Gaps = uncovered AND not already dead-lettered, within [start, windowEnd]. + // The UI shows dead-lettered heights separately via FailedRanges. gaps, err := s.State.Gaps(ctx, h.Name(), s.Start, windowEnd) if err == nil { - for _, rg := range gaps { - hs.Gaps = append(hs.Gaps, RangeJSON{From: rg.From, To: rg.To}) + failed, ferr := s.State.FailureHeights(ctx, h.Name(), s.Start, windowEnd) + if ferr != nil { + // Non-fatal: if we can't read failures, fall back to raw gaps. + for _, rg := range gaps { + hs.Gaps = append(hs.Gaps, RangeJSON{From: rg.From, To: rg.To}) + } + } else { + for _, rg := range subtractHeightsFromRanges(gaps, failed) { + hs.Gaps = append(hs.Gaps, RangeJSON{From: rg.From, To: rg.To}) + } } } hs.RowCounts = s.rowCountsFor(ctx, h.Name()) @@ -474,6 +486,37 @@ func (s *Server) dataTimeRangeFor(ctx context.Context, handler string) (time.Tim // coverageRatio returns the fraction of [from, to] that is covered by ranges. // 0 if to < from. +// subtractHeightsFromRanges splits each input range at every height in +// `heights`, returning the sub-ranges that remain after the heights are +// removed. `heights` must be sorted ascending (PopFailures / FailureHeights +// both produce sorted output). Used by the status handler to return +// "never-tried" gaps: raw gaps minus dead-lettered heights. +func subtractHeightsFromRanges(ranges []state.Range, heights []int64) []state.Range { + if len(heights) == 0 || len(ranges) == 0 { + return ranges + } + out := make([]state.Range, 0, len(ranges)) + h := 0 + for _, rg := range ranges { + // Advance h past heights below this range — they're irrelevant. + for h < len(heights) && heights[h] < rg.From { + h++ + } + cur := rg.From + for h < len(heights) && heights[h] <= rg.To { + if heights[h] > cur { + out = append(out, state.Range{From: cur, To: heights[h] - 1}) + } + cur = heights[h] + 1 + h++ + } + if cur <= rg.To { + out = append(out, state.Range{From: cur, To: rg.To}) + } + } + return out +} + func coverageRatio(ranges []RangeJSON, from, to int64) float64 { if to < from { return 0