Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions cmd/indexer/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,34 @@ func main() {
slog.Warn("aggregates apply failed (non-fatal)", "err", err)
}

// Optional: drop state for heights outside the current window. Used
// when the operator has narrowed start_height (or dropped earliest_mode)
// after a previous run populated things below the new window. One-shot
// and idempotent — if there's nothing to prune the counts are 0.
if cfg.Indexer.PruneOutsideWindow && !cfg.Indexer.EarliestMode {
pruneEnd := cfg.Indexer.EndHeight
if pruneEnd == 0 {
if tip, err := client.Tip(ctx); err == nil {
pruneEnd = tip
}
}
if pruneEnd > 0 {
for _, h := range reg.All() {
rr, ff, err := st.PruneOutsideWindow(ctx, h.Name(), cfg.Indexer.StartHeight, pruneEnd)
if err != nil {
slog.Warn("prune outside window failed", "handler", h.Name(), "err", err)
continue
}
if rr > 0 || ff > 0 {
slog.Info("pruned state outside window",
"handler", h.Name(),
"start", cfg.Indexer.StartHeight, "end", pruneEnd,
"ranges_deleted", rr, "failures_deleted", ff)
}
}
}
}

pipe := pipeline.New(cfg, pool, client, st, reg)
if sizer != nil {
pipe = pipe.WithSizer(sizer)
Expand Down
24 changes: 24 additions & 0 deletions config.example.yml
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,30 @@ indexer:
# How often the tip-follower polls /status after catching up.
tip_poll_interval: 3s

# Dead-letter retry loop: how often the pipeline sweeps app.indexer_failures
# and re-enqueues failed heights. 0 disables (dead-letters then only get
# another chance on restart or when tip-follow re-picks them up). Default
# 60s catches transient saturation without churning the DB.
failure_retry_interval: 60s

# Max failed heights retried per sweep. Smaller = gentler on the RPC fleet,
# larger = faster drain.
failure_retry_batch: 200

# If true, on startup delete indexer_ranges + indexer_failures rows for
# heights OUTSIDE [start_height, end_height|tip]. Useful when narrowing
# start_height after a previous wider run — otherwise the stale state
# shows up as zombie "covered ranges" on the timeline that will never
# be touched again.
prune_outside_window: false

# Gap scheduler interleave. Most batches go to the highest-cursor gap
# (tip-first) but every Nth batch goes to the LOWEST-cursor gap — so a
# big historical-backfill gap can't starve behind thousands of
# tip-close fragments. 0 or 1 = pure tip-first (fine if you only ever
# have one gap). Default 5 ≈ 80/20 split in favour of tip work.
backfill_interleave: 5

database:
host: postgres
port: 5432
Expand Down
46 changes: 38 additions & 8 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,33 @@ type Indexer struct {
WriteBatchRows int `yaml:"write_batch_rows"`
QueueDepth int `yaml:"queue_depth"`
TipPollInterval time.Duration `yaml:"tip_poll_interval"`

// PruneOutsideWindow, if true, deletes any indexer_ranges and
// indexer_failures entries for heights outside [StartHeight, EndHeight|tip]
// on startup. Useful when the operator narrowed the window after a
// previous (wider) run and wants the stale state dropped so gap math
// reflects the new window. One-shot per startup; safe to leave on.
PruneOutsideWindow bool `yaml:"prune_outside_window"`

// FailureRetryInterval controls how often the pipeline sweeps the
// dead-letter table and re-queues failed heights during an active
// run. 0 disables in-run retry (old behaviour: dead-letters wait
// until a restart or until tip-follow re-picks them up). Default 60s
// catches transient saturation failures without hammering the DB.
FailureRetryInterval time.Duration `yaml:"failure_retry_interval"`

// FailureRetryBatch is the max number of failed heights retried per
// sweep. Smaller = gentler on the RPC endpoints, larger = faster
// drain. Default 200.
FailureRetryBatch int `yaml:"failure_retry_batch"`

// BackfillInterleave controls how strictly tip-first the gap scheduler
// is. Every Nth batch goes to the OLDEST-cursor gap instead of the
// highest, so a large backfill gap can't starve behind tip-close
// fragments. 0 or 1 disables the interleave (pure tip-first — good
// when you only ever have one gap). Default 5 → 80/20 split in
// favour of tip-close work.
BackfillInterleave int `yaml:"backfill_interleave"`
}

// WantsHandler reports whether `name` should be registered given the
Expand Down Expand Up @@ -137,14 +164,17 @@ func Load(path string) (*Config, error) {
func defaults() *Config {
return &Config{
Indexer: Indexer{
StartHeight: 1,
EndHeight: 0,
TipConfirmations: 0, // Cosmos/CometBFT has deterministic finality
FetchWorkers: 16,
FetchBatchSize: 10,
WriteBatchRows: 20000,
QueueDepth: 200,
TipPollInterval: 3 * time.Second,
StartHeight: 1,
EndHeight: 0,
TipConfirmations: 0, // Cosmos/CometBFT has deterministic finality
FetchWorkers: 16,
FetchBatchSize: 10,
WriteBatchRows: 20000,
QueueDepth: 200,
TipPollInterval: 3 * time.Second,
FailureRetryInterval: 60 * time.Second,
FailureRetryBatch: 200,
BackfillInterleave: 5,
},
Database: Database{
Host: "localhost",
Expand Down
95 changes: 81 additions & 14 deletions internal/pipeline/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,33 +205,49 @@ func (p *Pipeline) indexGaps(ctx context.Context, gaps []state.Range) error {

g, ctx := errgroup.WithContext(ctx)

// Single producer, tip-first: on every emit we pick the gap whose
// current cursor is HIGHEST — i.e. closest to the chain tip — and
// dispatch the next batch from there. Consequences:
// - With a historical-backfill gap (e.g. [6mo-ago, pruning-horizon])
// plus a live tip-extension gap, the live gap always wins. The
// backfill only gets worker attention once the tip gap is caught
// up, so the observable "latest indexed height" tracks the chain
// head even while history is still being filled.
// - When tip-follower extends the top gap on each poll, those new
// heights immediately jump to the front of the queue; they don't
// sit behind half a million backfill blocks.
// - Within a single gap the behaviour is unchanged: bottom-up walk
// from gap.From to gap.To.
// Single producer, tip-first with interleave: most batches go to the
// gap with the HIGHEST cursor (closest to tip), but every Nth batch
// goes to the LOWEST-cursor gap instead, so a big historical gap
// can't starve behind thousands of tip-close fragments.
//
// Why not pure tip-first: if gaps are highly fragmented near tip
// (dead-letter holes × thousands) the tip-first preference keeps
// rotating through those fragments for hours while a single large
// bottom gap gets zero cycles. Interleave every N batches lets the
// bottom gap drain in parallel at ~(1/N) of the throughput without
// losing the "tip tracks head" property.
//
// BackfillInterleave: 0 or 1 → pure tip-first. Default 5 → ~80/20.
// Within a single gap the walk is still bottom-up From → To.
g.Go(func() error {
defer close(jobs)
cursors := make([]int64, len(gaps))
for i := range gaps {
cursors[i] = gaps[i].From
}
interleave := cfg.BackfillInterleave
batchIdx := 0
for {
// Every `interleave`-th batch, pick the LOWEST-cursor gap
// (oldest work) instead of the highest.
pickLowest := interleave > 1 && (batchIdx%interleave) == interleave-1
best := -1
for i := range gaps {
if cursors[i] > gaps[i].To {
continue
}
if best < 0 || cursors[i] > cursors[best] {
if best < 0 {
best = i
continue
}
if pickLowest {
if cursors[i] < cursors[best] {
best = i
}
} else {
if cursors[i] > cursors[best] {
best = i
}
}
}
if best < 0 {
Expand All @@ -248,9 +264,60 @@ func (p *Pipeline) indexGaps(ctx context.Context, gaps []state.Range) error {
case jobs <- job{from: cursors[best], to: to}:
}
cursors[best] = to + 1
batchIdx++
}
})

// Dead-letter retry sweep: the pipeline's processWithSplit records any
// size-1 block that exhausts its retries into app.indexer_failures and
// moves on. Historically those heights stayed "failed" until the process
// restarted — no good when a multi-day backfill briefly saturates the
// endpoints and racks up thousands of transient holes.
//
// This goroutine periodically pops the newest-first batch of failures,
// deletes their rows, and runs them back through processWithSplit. If
// the retry succeeds the height lands in indexer_ranges (so it stops
// being a gap); if it fails again, processWithSplit re-inserts the
// row with retries+1 and it gets another chance on the next sweep.
//
// Runs in parallel with the main tip-first producer — uses the same
// pgxpool + processWithSplit, so it shares the AIMD budget and advisory
// locks naturally. No separate worker pool needed; dead-letter retry
// is rarely the bottleneck.
if cfg.FailureRetryInterval > 0 && cfg.FailureRetryBatch > 0 {
g.Go(func() error {
ticker := time.NewTicker(cfg.FailureRetryInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return nil
case <-ticker.C:
}
for _, handler := range p.registry.All() {
heights, err := p.state.PopFailures(ctx, handler.Name(), cfg.FailureRetryBatch)
if err != nil {
slog.Warn("dead-letter sweep: pop failed", "handler", handler.Name(), "err", err)
continue
}
if len(heights) == 0 {
continue
}
slog.Info("dead-letter sweep: retrying heights",
"handler", handler.Name(), "count", len(heights))
for _, h := range heights {
if ctx.Err() != nil {
return nil
}
// Swallow errors: processWithSplit only returns ctx
// errors; real failures re-insert into indexer_failures.
_ = p.processWithSplit(ctx, h, h)
}
}
}
})
}

// Consumers.
start := time.Now()
var progressBlocks atomic.Int64
Expand Down
104 changes: 104 additions & 0 deletions internal/state/state.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,110 @@ func (s *State) FailureCount(ctx context.Context, handler string) (count int64,
return
}

// PopFailures removes up to `limit` dead-letter heights for `handler`
// (highest heights first — tip-first just like the main producer) and
// returns them. Atomic: the SELECT + DELETE happens inside a single
// DELETE…RETURNING so a second caller can't pop the same height.
//
// Heights returned here are candidates for in-run retry. If the retry
// eventually succeeds, the height lands in indexer_ranges and the failure
// is gone for good. If it fails again, processWithSplit re-inserts it
// with retries+1, putting it back in the dead-letter pool for the next
// cycle.
func (s *State) PopFailures(ctx context.Context, handler string, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
rows, err := s.pool.Query(ctx, fmt.Sprintf(`
DELETE FROM %[1]s.indexer_failures
WHERE (handler, height) IN (
SELECT handler, height
FROM %[1]s.indexer_failures
WHERE handler = $1
ORDER BY height DESC
LIMIT $2
)
RETURNING height`, s.schema), handler, limit)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]int64, 0, limit)
for rows.Next() {
var h int64
if err := rows.Scan(&h); err != nil {
return nil, err
}
out = append(out, h)
}
return out, rows.Err()
}

// PruneOutsideWindow deletes indexer_ranges + indexer_failures entries
// that fall entirely outside [start, end] for the given handler. Used on
// startup when the operator narrows the start_height and wants the stale
// state from a previous (wider) run removed so the UI + gap math stay
// honest about the new window. Ranges that STRADDLE the boundary are
// left untouched — they still contain blocks inside the new window.
//
// Returns (ranges_deleted, failures_deleted).
func (s *State) PruneOutsideWindow(ctx context.Context, handler string, start, end int64) (int64, int64, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return 0, 0, err
}
defer tx.Rollback(ctx)

// Advisory lock: prevents the pipeline's range-merge from racing with
// the prune on the same handler.
if _, err := tx.Exec(ctx,
`SELECT pg_advisory_xact_lock(hashtext($1))`, handler); err != nil {
return 0, 0, err
}

rTag, err := tx.Exec(ctx, fmt.Sprintf(`
DELETE FROM %s.indexer_ranges
WHERE handler = $1 AND (to_height < $2 OR from_height > $3)`, s.schema),
handler, start, end)
if err != nil {
return 0, 0, err
}
fTag, err := tx.Exec(ctx, fmt.Sprintf(`
DELETE FROM %s.indexer_failures
WHERE handler = $1 AND (height < $2 OR height > $3)`, s.schema),
handler, start, end)
if err != nil {
return 0, 0, err
}
if err := tx.Commit(ctx); err != nil {
return 0, 0, err
}
return rTag.RowsAffected(), fTag.RowsAffected(), nil
}

// FailureHeights returns the individual dead-letter heights in [start, end]
// for a handler. Used by the web layer to subtract them from "untouched"
// gaps so the UI distinguishes never-tried gaps from retry-waiting ones.
func (s *State) FailureHeights(ctx context.Context, handler string, start, end int64) ([]int64, error) {
rows, err := s.pool.Query(ctx, fmt.Sprintf(`
SELECT height FROM %s.indexer_failures
WHERE handler = $1 AND height BETWEEN $2 AND $3
ORDER BY height`, s.schema), handler, start, end)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]int64, 0, 256)
for rows.Next() {
var h int64
if err := rows.Scan(&h); err != nil {
return nil, err
}
out = append(out, h)
}
return out, rows.Err()
}

// FailureRanges coalesces individual dead-letter heights into contiguous
// ranges so the UI can paint them as red bars on the timeline instead of
// 1-pixel spikes.
Expand Down
15 changes: 14 additions & 1 deletion internal/web/assets/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,9 @@
details.ranges-detail .list span.gap {
color: var(--warn); border-color: #3b2a0d;
}
details.ranges-detail .list span.failed-range {
color: var(--danger); border-color: #3b0d0d;
}

footer { padding: 18px 24px; color: var(--muted); font-size: 11px; border-top: 1px solid var(--border); margin-top: 40px; text-align: center; }

Expand Down Expand Up @@ -1048,10 +1051,20 @@ <h2>Handlers</h2>
card.axisDateR.textContent = tipDateText;
card.axisDateR.style.display = tipDateText ? '' : 'none';

card.summary.textContent = `${ranges.length} covered ranges, ${(h.gaps || []).length} gap${(h.gaps||[]).length === 1 ? '' : 's'}`;
// Two separate counts: untouched gaps (never attempted) and dead-letter
// holes (attempted, failed, waiting for retry). Historically both were
// lumped as "gaps" which made a transiently-saturated fleet look like
// a huge unindexed hole when actually most of the window WAS attempted.
const gapCount = (h.gaps || []).length;
const failedCount = (h.failed_ranges || []).length;
const parts = [`${fmt(ranges.length)} covered range${ranges.length === 1 ? '' : 's'}`];
if (gapCount) parts.push(`${fmt(gapCount)} untouched`);
if (failedCount) parts.push(`${fmt(failedCount)} dead-letter range${failedCount === 1 ? '' : 's'}`);
card.summary.textContent = parts.join(' · ');
card.list.innerHTML = '';
ranges.forEach(r => card.list.appendChild(el('span', {}, `${fmt(r.from)} → ${fmt(r.to)}`)));
(h.gaps || []).forEach(r => card.list.appendChild(el('span', { class: 'gap' }, `gap ${fmt(r.from)} → ${fmt(r.to)}`)));
(h.failed_ranges || []).forEach(r => card.list.appendChild(el('span', { class: 'failed-range' }, `dead-letter ${fmt(r.from)}${r.from !== r.to ? ' → ' + fmt(r.to) : ''}`)));
}

function renderHeader(s) {
Expand Down
Loading
Loading