Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions drivers/postgres/internal/cdc.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,13 @@ func (p *Postgres) PreCDC(ctx context.Context, streams []types.StreamInterface)

globalState := p.state.GetGlobal()
if globalState == nil || globalState.State == nil {
p.state.SetGlobal(waljs.WALState{LSN: slot.CurrentLSN.String()})
p.state.ResetStreams()
// advance before seeding state: AdvanceLSN returns only once the slot has confirmed
// the target, so the state file and confirmed_flush_lsn agree by construction
if err := waljs.AdvanceLSN(ctx, p.client, p.cdcConfig.ReplicationSlot, slot.CurrentLSN.String()); err != nil {
return err
}
p.state.SetGlobal(waljs.WALState{LSN: slot.CurrentLSN.String()})
p.state.ResetStreams()
}
p.streams = streams
return nil
Expand Down
33 changes: 27 additions & 6 deletions pkg/waljs/replicator.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,14 +130,35 @@ func NewReplicator(ctx context.Context, config *Config, slot ReplicationSlot, re
}
}

// advanceLSN advances the logical replication position to the current WAL position.
// AdvanceLSN moves the slot to currentWalPos and retries until postgres confirms it there:
// pg_replication_slot_advance silently clamps its target to the flushed WAL position.
func AdvanceLSN(ctx context.Context, db *sqlx.DB, slot, currentWalPos string) error {
// Get replication slot position
if _, err := db.ExecContext(ctx, fmt.Sprintf(AdvanceLSNTemplate, slot, currentWalPos)); err != nil {
return fmt.Errorf("failed to advance replication slot: %w", err)
target, err := pglogrepl.ParseLSN(currentWalPos)
if err != nil {
return fmt.Errorf("failed to parse advance target lsn[%s]: %s", currentWalPos, err)
}

timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
for {
var slotName string
var endLSN pglogrepl.LSN
Comment on lines +144 to +145

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we remove this slotName variable from here if it is redundant?

if err := db.QueryRowContext(timeoutCtx, fmt.Sprintf(AdvanceLSNTemplate, slot, currentWalPos)).Scan(&slotName, &endLSN); err != nil {
return fmt.Errorf("failed to advance replication slot: %w", err)
}
Comment on lines +146 to +148

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I understand why we're using timeoutCtx here, but I noticed that elsewhere in the codebase, QueryRowContext is used with the regular ctx. Do we need the timeout specifically here, or should we keep it consistent with the rest of the codebase? Is there any specific reason for using timeoutCtx here?

if endLSN >= target {
logger.Debugf("advanced LSN to %s", currentWalPos)
Comment on lines +149 to +150

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here we have >= check but when > check will be possible over here ?
And suppose it is possible than shouldn't we set endLSN in globalState instead of target ?

return nil
}
// end_lsn short of the request means the advance was clamped to the flushed position;
// wait for the walwriter to catch up (bounded by ~3x wal_writer_delay) and re-advance
logger.Debugf("slot advance clamped at %s (target %s), retrying", endLSN, target)
select {
case <-timeoutCtx.Done():
return fmt.Errorf("slot advance stopped at %s and could not reach target %s: %w", endLSN, target, timeoutCtx.Err())
case <-time.After(100 * time.Millisecond):
Comment on lines +158 to +159

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We do not use %w in our codebase for normal error,we use it for non-retryable error?If changing also look for other error messages as well.

}
}
logger.Debugf("advanced LSN to %s", currentWalPos)
return nil
}

// Confirm that Logs has been recorded
Expand Down
Loading