Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 21 additions & 9 deletions protocol/chainlib/consumer_websocket_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -695,21 +695,23 @@ func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) {
utils.LogAttr("params", protocolMessage.GetRPCMessage().GetParams()),
)

// A client the manager dropped for not keeping up is told the one way a WebSocket
// can tell it: its socket closes with 1013 (try again later), so it reconnects and
// resubscribes instead of waiting on a socket nothing will ever push to again. The
// frame goes through the writer like any other, and the writer then tears the
// connection down, so the deferred UnsubscribeAll ends the client's other
// subscriptions on it. Any other end of a subscription leaves the socket open.
if errors.Is(cwm.subscriptionEndError(dappID, userIp), ErrSubscriptionClientTooSlow) {
utils.LavaFormatDebug("websocket subscription client did not keep up, closing its connection",
// A client the manager dropped for not keeping up, or whose subscription the router
// lost and could not restore, is told the one way a WebSocket can tell it: its
// socket closes with 1013 (try again later), so it reconnects and resubscribes
// instead of waiting on a socket nothing will ever push to again. The frame goes
// through the writer like any other, and the writer then tears the connection
// down, so the deferred UnsubscribeAll ends the client's other subscriptions on it.
// Any other end of a subscription leaves the socket open.
if endErr := cwm.subscriptionEndError(dappID, userIp); subscriptionEndClosesSocket(endErr) {
utils.LavaFormatDebug("websocket subscription ended for a reason the client must hear, closing its connection",
utils.LogAttr("GUID", frameCtx),
utils.LogAttr("dappID", dappID),
utils.LogAttr("userIp", userIp),
utils.LogAttr("reason", endErr.Error()),
)
sendWS(webSocketMsgWithType{
messageType: websocket.CloseMessage,
msg: websocket.FormatCloseMessage(websocket.CloseTryAgainLater, ErrSubscriptionClientTooSlow.Error()),
msg: websocket.FormatCloseMessage(websocket.CloseTryAgainLater, endErr.Error()),
final: true,
})
}
Expand Down Expand Up @@ -737,6 +739,16 @@ func (cwm *ConsumerWebsocketManager) reportSubscriptionRelay(ctx context.Context
}
}

// subscriptionEndClosesSocket reports whether the reason a manager gave for ending a subscription
// is one the client must hear by its socket closing with 1013 (try again later): the client did not
// keep up with the pushes, or the router lost the subscription and could not restore it (MAG-5022).
// Either way nothing will push on that subscription again, and the client reconnects and
// resubscribes. The set is closed on purpose: a reason the ingress does not know leaves the socket
// open, as before.
func subscriptionEndClosesSocket(endErr error) bool {
return errors.Is(endErr, ErrSubscriptionClientTooSlow) || errors.Is(endErr, ErrSubscriptionUpstreamLost)
}

// subscriptionEndError asks the subscription manager why it ended a subscription of this
// connection, when the manager reports that at all; nil for an ordinary end.
func (cwm *ConsumerWebsocketManager) subscriptionEndError(dappID, userIp string) error {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,19 @@ const subscribeFrame = `{"jsonrpc":"2.0","method":"eth_subscribe","params":["new
// client's socket with 1013 (try again later), through the connection's writer, so the client
// reconnects instead of waiting on a socket nothing will push to again.
func TestConsumerWebsocketManager_SlowSubscriberIsClosedWithTryAgainLater(t *testing.T) {
client := dialSubscriptionListener(t, endingSubscriptions{endErr: ErrSubscriptionClientTooSlow})
requireClosedWithTryAgainLater(t, ErrSubscriptionClientTooSlow)
}

// MAG-5022: a subscription the router gave up (it was lost and could not be restored) closes the
// client's socket the same way, with the reason the gRPC manager ends such a stream with. Before,
// the client kept a socket nothing would ever push on again.
func TestConsumerWebsocketManager_GivenUpSubscriptionIsClosedWithTryAgainLater(t *testing.T) {
requireClosedWithTryAgainLater(t, ErrSubscriptionUpstreamLost)
}

func requireClosedWithTryAgainLater(t *testing.T, endErr error) {
t.Helper()
client := dialSubscriptionListener(t, endingSubscriptions{endErr: endErr})
require.NoError(t, client.WriteMessage(websocket.TextMessage, []byte(subscribeFrame)))

for frames := 0; ; frames++ {
Expand All @@ -72,7 +84,7 @@ func TestConsumerWebsocketManager_SlowSubscriberIsClosedWithTryAgainLater(t *tes
var closeErr *websocket.CloseError
require.ErrorAs(t, err, &closeErr, "the client got %v instead of a close frame", err)
require.Equal(t, websocket.CloseTryAgainLater, closeErr.Code)
require.Equal(t, ErrSubscriptionClientTooSlow.Error(), closeErr.Text)
require.Equal(t, endErr.Error(), closeErr.Text)
return
}
}
Expand Down
11 changes: 9 additions & 2 deletions protocol/chainlib/ws_subscription_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,14 +61,21 @@ type WSSubscriptionManager interface {
// reason in the close frame the client gets.
var ErrSubscriptionClientTooSlow = errors.New("subscription client did not keep up")

// ErrSubscriptionUpstreamLost is what a WSSubscriptionEndReporter reports for a connection whose
// subscription the router gave up: the upstream subscription was lost and could not be restored
// (the reconnect or the re-subscribe failed, or the restores without progress ran out). Nothing
// will push on it again. Its text is the reason in the close frame the client gets, the same words
// the gRPC manager ends such a stream with (codes.Unavailable) (MAG-5022).
var ErrSubscriptionUpstreamLost = errors.New("upstream subscription lost")

// WSSubscriptionEndReporter is an optional interface on WSSubscriptionManager, the WebSocket
// counterpart of GRPCSubscriptionEndReporter. A WebSocket subscription has no end of its own to
// carry a reason: its reply channel just closes. A manager that ends a client's subscription for
// a reason the client must hear reports it here, and the ingress tells the client by closing the
// socket. Any other end leaves the socket open, as before.
type WSSubscriptionEndReporter interface {
// SubscriptionEndError returns why the manager ended a subscription of this connection
// (ErrSubscriptionClientTooSlow), or nil for an ordinary end. The ingress reads it after a
// reply channel has closed; a reason is read once.
// (ErrSubscriptionClientTooSlow, ErrSubscriptionUpstreamLost), or nil for an ordinary end. The
// ingress reads it after a reply channel has closed; a reason is read once.
SubscriptionEndError(dappID string, consumerIp string, webSocketConnectionUniqueId string) error
}
111 changes: 62 additions & 49 deletions protocol/chaintracker/chain_tracker.go
Original file line number Diff line number Diff line change
Expand Up @@ -934,47 +934,79 @@ func (cs *ChainTracker) waitOutRateLimitHoldoff(ctx context.Context) error {
return sleepCtx(ctx, wait)
}

func (cs *ChainTracker) fetchInitDataWithRetry(ctx context.Context) (err error) {
var newLatestBlock int64
for idx := 0; idx < initRetriesCount+1; idx++ {
// retryInitFetch runs one init step, fetch, until it answers or its attempts are spent: the
// first attempt and initRetriesCount retries. Both init steps (the head, then the previous
// blocks) go through here, so they get the same attempts and the same spacing; they used to be
// two loops that had drifted one attempt apart (MAG-5022).
//
// MAG-2159 finding 3: failed attempts on a per-endpoint (flat) tracker are spaced
// flatPollInterval apart so a failing endpoint cannot burst the upstream at startup.
// Cancellation stays prompt (sleepCtx returns at once on ctx.Done), and there is no delay after
// the last attempt. Every outcome informs the rate-limit floor and the hold-off registry, as a
// poll's does, and a rate-limited attempt ends the burst at once: another attempt now would re-arm
// a ban-type limit. The applied hold-off rides the error as its Retry-After, so the caller retrying
// the start (endpointstate.startTrackerWithRetry) waits it out with the tracker in retrying_start
// naming the 429, not sleeping inside StartAndServe where the state would read "starting" with no
// error for the whole wait.
//
// The spent-attempts failure is returned, not logged at ERROR. The one production caller
// retries the start for as long as the endpoint is down and warns once per attempt with this
// error (startTrackerWithRetry), so an ERROR here described a dead endpoint, a non-fatal
// condition the router handles, as "critical" every 30 s or less for the whole outage.
//
// The returned error is built here and only logged at debug. A LavaFormat* call whose level is
// off returns the bare cause, so at the router's default --log-level (info) the step, the
// attempts and the timeout suggestion would have reached neither that WARN nor the tracker's
// retrying_start state. The endpoint stays out of the text: the WARN names it.
func (cs *ChainTracker) retryInitFetch(ctx context.Context, step string, fetch func(context.Context) error) error {
var err error
for attempt := 0; attempt <= initRetriesCount; attempt++ {
if err = cs.waitOutRateLimitHoldoff(ctx); err != nil {
return err
}
newLatestBlock, err = cs.iChainFetcherWrapper.FetchLatestBlockNum(ctx)
// Every init outcome informs the rate-limit floor and the registry, as a poll's does.
err = fetch(ctx)
heldFor := cs.noteFetchOutcome(err)
if err == nil {
break
return nil
}
if errors.Is(err, common.StatusCodeError429) {
// The upstream refused for rate: the burst ends here. Another attempt now would
// re-arm a ban-type limit. The applied hold-off rides the error as its Retry-After, so
// the caller retrying the start (endpointstate.startTrackerWithRetry) waits it out with
// the tracker in retrying_start naming the 429 — not sleeping inside StartAndServe,
// where the state would read "starting" with no error for the whole wait.
return utils.LavaFormatWarning("chain tracker init: upstream rate-limited the first fetch, backing off", common.RateLimited(err, heldFor), utils.Attribute{Key: "endpoint", Value: cs.endpoint.String()})
return utils.LavaFormatWarning("chain tracker init: upstream rate-limited the "+step+", backing off", common.RateLimited(err, heldFor), utils.Attribute{Key: "endpoint", Value: cs.endpoint.String()})
}
if attempt == initRetriesCount {
break
}
utils.LavaFormatDebug("failed fetching block num data on chain tracker init, retry", utils.Attribute{Key: "retry Num", Value: idx}, utils.Attribute{Key: "endpoint", Value: cs.endpoint})
// MAG-2159 finding 3: space out failed init retries for per-endpoint (flat) trackers
// so a failing endpoint cannot burst the upstream at startup. Cancellation stays
// prompt (sleepCtx returns immediately on ctx.Done). No delay after the last attempt.
if cs.flatPollInterval > 0 && idx < initRetriesCount {
utils.LavaFormatDebug("chain tracker init: "+step+" failed, retrying",
utils.Attribute{Key: "attempt", Value: attempt + 1},
utils.Attribute{Key: "error", Value: err},
utils.Attribute{Key: "endpoint", Value: cs.endpoint.String()},
)
if cs.flatPollInterval > 0 {
if ctxErr := sleepCtx(ctx, cs.flatPollInterval); ctxErr != nil {
return ctxErr
}
}
}
failure := fmt.Errorf("chain tracker init: failed fetching data from the node: %s failed after %d attempts: %w", step, initRetriesCount+1, err)
if errors.Is(err, common.ContextDeadlineExceededError) {
failure = fmt.Errorf("%w (suggestion: consider increasing the timeout configuration in the 'node-url' config option; the initial HTTPS/WSS communication sometimes takes a long time to establish a connection)", failure)
}
utils.LavaFormatDebug(failure.Error(), utils.Attribute{Key: "endpoint", Value: cs.endpoint.String()})
return failure
}

func (cs *ChainTracker) fetchInitDataWithRetry(ctx context.Context) error {
var newLatestBlock int64
err := cs.retryInitFetch(ctx, "head fetch", func(ctx context.Context) (err error) {
newLatestBlock, err = cs.iChainFetcherWrapper.FetchLatestBlockNum(ctx)
return err
})
if err != nil {
// Add suggestion if error is due to context deadline exceeded
if errors.Is(err, common.ContextDeadlineExceededError) {
utils.LavaFormatError("suggestion -- If you encounter a 'context deadline exceeded' error, consider increasing the timeout configuration in the 'node-url' config option. Sometimes, the initial HTTPS/WSS communication takes a long time to establish a connection.", nil)
}
return utils.LavaFormatError("critical -- failed fetching data from the node, chain tracker creation error", err, utils.Attribute{Key: "endpoint", Value: cs.endpoint})
return err
}
// Head-only (MAG-2218): there is no hash to seed a block queue with, and retrying
// fetchAllPreviousBlocks would exhaust its attempts and return the "chain tracker creation
// error" below — which start() propagates, leaving startTrackerWithRetry looping forever on a
// chain that is in fact healthy. Publish the head we just fetched and finish init.
// fetchAllPreviousBlocks would exhaust its attempts and return the init failure below, which
// start() propagates, leaving startTrackerWithRetry looping forever on a chain that is in fact
// healthy. Publish the head we just fetched and finish init.
if cs.headOnly {
// A reply can parse cleanly and still carry no usable height: an SVM result object with
// no context.slot unmarshals to slot 0 with a nil error. Nothing below this branch would
Expand All @@ -991,31 +1023,12 @@ func (cs *ChainTracker) fetchInitDataWithRetry(ctx context.Context) (err error)
return nil
}

for idx := 0; idx < initRetriesCount; idx++ {
_, err = cs.fetchAllPreviousBlocks(ctx, newLatestBlock)
if err == nil {
break
}
if errors.Is(err, common.StatusCodeError429) {
// Same rule as the head fetch: a rate limit ends the burst, and the applied hold-off
// rides the error for the start retry to wait out.
heldFor := cs.noteFetchOutcome(err)
return utils.LavaFormatWarning("chain tracker init: upstream rate-limited the block fetch, backing off", common.RateLimited(err, heldFor), utils.Attribute{Key: "endpoint", Value: cs.endpoint.String()})
}
utils.LavaFormatDebug("failed fetching data on chain tracker init, retry", utils.Attribute{Key: "retry Num", Value: idx}, utils.Attribute{Key: "endpoint", Value: cs.endpoint.String()})
// MAG-2159 finding 3: same startup spacing for the previous-blocks init retries.
if cs.flatPollInterval > 0 && idx < initRetriesCount-1 {
if ctxErr := sleepCtx(ctx, cs.flatPollInterval); ctxErr != nil {
return ctxErr
}
}
}
err = cs.retryInitFetch(ctx, "block fetch", func(ctx context.Context) error {
_, err := cs.fetchAllPreviousBlocks(ctx, newLatestBlock)
return err
})
if err != nil {
// Add suggestion if error is due to context deadline exceeded
if errors.Is(err, common.ContextDeadlineExceededError) {
utils.LavaFormatError("suggestion -- If you encounter a 'context deadline exceeded' error, consider increasing the timeout configuration in the 'node-url' config option. Sometimes, the initial HTTPS/WSS communication takes a long time to establish a connection.", nil)
}
return utils.LavaFormatError("critical -- failed fetching data from the node, chain tracker creation error", err, utils.Attribute{Key: "endpoint", Value: cs.endpoint})
return err
}
cs.lastRealPoll = time.Now() // the init fetch is this tracker's first real poll
return nil
Expand Down
Loading