Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 21 additions & 9 deletions protocol/chainlib/consumer_websocket_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -695,21 +695,23 @@ func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) {
utils.LogAttr("params", protocolMessage.GetRPCMessage().GetParams()),
)

// A client the manager dropped for not keeping up is told the one way a WebSocket
// can tell it: its socket closes with 1013 (try again later), so it reconnects and
// resubscribes instead of waiting on a socket nothing will ever push to again. The
// frame goes through the writer like any other, and the writer then tears the
// connection down, so the deferred UnsubscribeAll ends the client's other
// subscriptions on it. Any other end of a subscription leaves the socket open.
if errors.Is(cwm.subscriptionEndError(dappID, userIp), ErrSubscriptionClientTooSlow) {
utils.LavaFormatDebug("websocket subscription client did not keep up, closing its connection",
// A client the manager dropped for not keeping up, or whose subscription the router
// lost and could not restore, is told the one way a WebSocket can tell it: its
// socket closes with 1013 (try again later), so it reconnects and resubscribes
// instead of waiting on a socket nothing will ever push to again. The frame goes
// through the writer like any other, and the writer then tears the connection
// down, so the deferred UnsubscribeAll ends the client's other subscriptions on it.
// Any other end of a subscription leaves the socket open.
if endErr := cwm.subscriptionEndError(dappID, userIp); subscriptionEndClosesSocket(endErr) {
utils.LavaFormatDebug("websocket subscription ended for a reason the client must hear, closing its connection",
utils.LogAttr("GUID", frameCtx),
utils.LogAttr("dappID", dappID),
utils.LogAttr("userIp", userIp),
utils.LogAttr("reason", endErr.Error()),
)
sendWS(webSocketMsgWithType{
messageType: websocket.CloseMessage,
msg: websocket.FormatCloseMessage(websocket.CloseTryAgainLater, ErrSubscriptionClientTooSlow.Error()),
msg: websocket.FormatCloseMessage(websocket.CloseTryAgainLater, endErr.Error()),
final: true,
})
}
Expand Down Expand Up @@ -737,6 +739,16 @@ func (cwm *ConsumerWebsocketManager) reportSubscriptionRelay(ctx context.Context
}
}

// subscriptionEndClosesSocket reports whether the reason a manager gave for ending a subscription
// is one the client must hear by its socket closing with 1013 (try again later): the client did not
// keep up with the pushes, or the router lost the subscription and could not restore it (MAG-5022).
// Either way nothing will push on that subscription again, and the client reconnects and
// resubscribes. The set is closed on purpose: a reason the ingress does not know leaves the socket
// open, as before.
func subscriptionEndClosesSocket(endErr error) bool {
return errors.Is(endErr, ErrSubscriptionClientTooSlow) || errors.Is(endErr, ErrSubscriptionUpstreamLost)
}

// subscriptionEndError asks the subscription manager why it ended a subscription of this
// connection, when the manager reports that at all; nil for an ordinary end.
func (cwm *ConsumerWebsocketManager) subscriptionEndError(dappID, userIp string) error {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,19 @@ const subscribeFrame = `{"jsonrpc":"2.0","method":"eth_subscribe","params":["new
// client's socket with 1013 (try again later), through the connection's writer, so the client
// reconnects instead of waiting on a socket nothing will push to again.
func TestConsumerWebsocketManager_SlowSubscriberIsClosedWithTryAgainLater(t *testing.T) {
client := dialSubscriptionListener(t, endingSubscriptions{endErr: ErrSubscriptionClientTooSlow})
requireClosedWithTryAgainLater(t, ErrSubscriptionClientTooSlow)
}

// MAG-5022: a subscription the router gave up (it was lost and could not be restored) closes the
// client's socket the same way, with the reason the gRPC manager ends such a stream with. Before,
// the client kept a socket nothing would ever push on again.
func TestConsumerWebsocketManager_GivenUpSubscriptionIsClosedWithTryAgainLater(t *testing.T) {
requireClosedWithTryAgainLater(t, ErrSubscriptionUpstreamLost)
}

func requireClosedWithTryAgainLater(t *testing.T, endErr error) {
t.Helper()
client := dialSubscriptionListener(t, endingSubscriptions{endErr: endErr})
require.NoError(t, client.WriteMessage(websocket.TextMessage, []byte(subscribeFrame)))

for frames := 0; ; frames++ {
Expand All @@ -72,7 +84,7 @@ func TestConsumerWebsocketManager_SlowSubscriberIsClosedWithTryAgainLater(t *tes
var closeErr *websocket.CloseError
require.ErrorAs(t, err, &closeErr, "the client got %v instead of a close frame", err)
require.Equal(t, websocket.CloseTryAgainLater, closeErr.Code)
require.Equal(t, ErrSubscriptionClientTooSlow.Error(), closeErr.Text)
require.Equal(t, endErr.Error(), closeErr.Text)
return
}
}
Expand Down
11 changes: 9 additions & 2 deletions protocol/chainlib/ws_subscription_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,14 +61,21 @@ type WSSubscriptionManager interface {
// reason in the close frame the client gets.
var ErrSubscriptionClientTooSlow = errors.New("subscription client did not keep up")

// ErrSubscriptionUpstreamLost is what a WSSubscriptionEndReporter reports for a connection whose
// subscription the router gave up: the upstream subscription was lost and could not be restored
// (the reconnect or the re-subscribe failed, or the restores without progress ran out). Nothing
// will push on it again. Its text is the reason in the close frame the client gets, the same words
// the gRPC manager ends such a stream with (codes.Unavailable) (MAG-5022).
var ErrSubscriptionUpstreamLost = errors.New("upstream subscription lost")

// WSSubscriptionEndReporter is an optional interface on WSSubscriptionManager, the WebSocket
// counterpart of GRPCSubscriptionEndReporter. A WebSocket subscription has no end of its own to
// carry a reason: its reply channel just closes. A manager that ends a client's subscription for
// a reason the client must hear reports it here, and the ingress tells the client by closing the
// socket. Any other end leaves the socket open, as before.
type WSSubscriptionEndReporter interface {
// SubscriptionEndError returns why the manager ended a subscription of this connection
// (ErrSubscriptionClientTooSlow), or nil for an ordinary end. The ingress reads it after a
// reply channel has closed; a reason is read once.
// (ErrSubscriptionClientTooSlow, ErrSubscriptionUpstreamLost), or nil for an ordinary end. The
// ingress reads it after a reply channel has closed; a reason is read once.
SubscriptionEndError(dappID string, consumerIp string, webSocketConnectionUniqueId string) error
}
109 changes: 60 additions & 49 deletions protocol/chaintracker/chain_tracker.go
Original file line number Diff line number Diff line change
Expand Up @@ -934,47 +934,77 @@ func (cs *ChainTracker) waitOutRateLimitHoldoff(ctx context.Context) error {
return sleepCtx(ctx, wait)
}

func (cs *ChainTracker) fetchInitDataWithRetry(ctx context.Context) (err error) {
var newLatestBlock int64
for idx := 0; idx < initRetriesCount+1; idx++ {
// retryInitFetch runs one init step, fetch, until it answers or its attempts are spent: the
// first attempt and initRetriesCount retries. Both init steps (the head, then the previous
// blocks) go through here, so they get the same attempts and the same spacing; they used to be
// two loops that had drifted one attempt apart (MAG-5022).
//
// MAG-2159 finding 3: failed attempts on a per-endpoint (flat) tracker are spaced
// flatPollInterval apart so a failing endpoint cannot burst the upstream at startup.
// Cancellation stays prompt (sleepCtx returns at once on ctx.Done), and there is no delay after
// the last attempt. Every outcome informs the rate-limit floor and the hold-off registry, as a
// poll's does, and a rate-limited attempt ends the burst at once: another attempt now would re-arm
// a ban-type limit. The applied hold-off rides the error as its Retry-After, so the caller retrying
// the start (endpointstate.startTrackerWithRetry) waits it out with the tracker in retrying_start
// naming the 429, not sleeping inside StartAndServe where the state would read "starting" with no
// error for the whole wait.
//
// The spent-attempts failure is returned, not logged at ERROR. The one production caller
// retries the start for as long as the endpoint is down and warns once per attempt with this
// error (startTrackerWithRetry), so an ERROR here described a dead endpoint, a non-fatal
// condition the router handles, as "critical" every 30 s or less for the whole outage.
func (cs *ChainTracker) retryInitFetch(ctx context.Context, step string, fetch func(context.Context) error) error {
var err error
for attempt := 0; attempt <= initRetriesCount; attempt++ {
if err = cs.waitOutRateLimitHoldoff(ctx); err != nil {
return err
}
newLatestBlock, err = cs.iChainFetcherWrapper.FetchLatestBlockNum(ctx)
// Every init outcome informs the rate-limit floor and the registry, as a poll's does.
err = fetch(ctx)
heldFor := cs.noteFetchOutcome(err)
if err == nil {
break
return nil
}
if errors.Is(err, common.StatusCodeError429) {
// The upstream refused for rate: the burst ends here. Another attempt now would
// re-arm a ban-type limit. The applied hold-off rides the error as its Retry-After, so
// the caller retrying the start (endpointstate.startTrackerWithRetry) waits it out with
// the tracker in retrying_start naming the 429 — not sleeping inside StartAndServe,
// where the state would read "starting" with no error for the whole wait.
return utils.LavaFormatWarning("chain tracker init: upstream rate-limited the first fetch, backing off", common.RateLimited(err, heldFor), utils.Attribute{Key: "endpoint", Value: cs.endpoint.String()})
return utils.LavaFormatWarning("chain tracker init: upstream rate-limited the "+step+", backing off", common.RateLimited(err, heldFor), utils.Attribute{Key: "endpoint", Value: cs.endpoint.String()})
}
if attempt == initRetriesCount {
break
}
utils.LavaFormatDebug("failed fetching block num data on chain tracker init, retry", utils.Attribute{Key: "retry Num", Value: idx}, utils.Attribute{Key: "endpoint", Value: cs.endpoint})
// MAG-2159 finding 3: space out failed init retries for per-endpoint (flat) trackers
// so a failing endpoint cannot burst the upstream at startup. Cancellation stays
// prompt (sleepCtx returns immediately on ctx.Done). No delay after the last attempt.
if cs.flatPollInterval > 0 && idx < initRetriesCount {
utils.LavaFormatDebug("chain tracker init: "+step+" failed, retrying",
utils.Attribute{Key: "attempt", Value: attempt + 1},
utils.Attribute{Key: "error", Value: err},
utils.Attribute{Key: "endpoint", Value: cs.endpoint.String()},
)
if cs.flatPollInterval > 0 {
if ctxErr := sleepCtx(ctx, cs.flatPollInterval); ctxErr != nil {
return ctxErr
}
}
}
attributes := []utils.Attribute{
{Key: "step", Value: step},
{Key: "attempts", Value: initRetriesCount + 1},
{Key: "endpoint", Value: cs.endpoint.String()},
}
if errors.Is(err, common.ContextDeadlineExceededError) {
attributes = append(attributes, utils.Attribute{Key: "suggestion", Value: "If you encounter a 'context deadline exceeded' error, consider increasing the timeout configuration in the 'node-url' config option. Sometimes, the initial HTTPS/WSS communication takes a long time to establish a connection."})
}
return utils.LavaFormatDebugErr("chain tracker init: failed fetching data from the node", err, attributes...)
}

func (cs *ChainTracker) fetchInitDataWithRetry(ctx context.Context) error {
var newLatestBlock int64
err := cs.retryInitFetch(ctx, "head fetch", func(ctx context.Context) (err error) {
newLatestBlock, err = cs.iChainFetcherWrapper.FetchLatestBlockNum(ctx)
return err
})
if err != nil {
// Add suggestion if error is due to context deadline exceeded
if errors.Is(err, common.ContextDeadlineExceededError) {
utils.LavaFormatError("suggestion -- If you encounter a 'context deadline exceeded' error, consider increasing the timeout configuration in the 'node-url' config option. Sometimes, the initial HTTPS/WSS communication takes a long time to establish a connection.", nil)
}
return utils.LavaFormatError("critical -- failed fetching data from the node, chain tracker creation error", err, utils.Attribute{Key: "endpoint", Value: cs.endpoint})
return err
}
// Head-only (MAG-2218): there is no hash to seed a block queue with, and retrying
// fetchAllPreviousBlocks would exhaust its attempts and return the "chain tracker creation
// error" below — which start() propagates, leaving startTrackerWithRetry looping forever on a
// chain that is in fact healthy. Publish the head we just fetched and finish init.
// fetchAllPreviousBlocks would exhaust its attempts and return the init failure below, which
// start() propagates, leaving startTrackerWithRetry looping forever on a chain that is in fact
// healthy. Publish the head we just fetched and finish init.
if cs.headOnly {
// A reply can parse cleanly and still carry no usable height: an SVM result object with
// no context.slot unmarshals to slot 0 with a nil error. Nothing below this branch would
Expand All @@ -991,31 +1021,12 @@ func (cs *ChainTracker) fetchInitDataWithRetry(ctx context.Context) (err error)
return nil
}

for idx := 0; idx < initRetriesCount; idx++ {
_, err = cs.fetchAllPreviousBlocks(ctx, newLatestBlock)
if err == nil {
break
}
if errors.Is(err, common.StatusCodeError429) {
// Same rule as the head fetch: a rate limit ends the burst, and the applied hold-off
// rides the error for the start retry to wait out.
heldFor := cs.noteFetchOutcome(err)
return utils.LavaFormatWarning("chain tracker init: upstream rate-limited the block fetch, backing off", common.RateLimited(err, heldFor), utils.Attribute{Key: "endpoint", Value: cs.endpoint.String()})
}
utils.LavaFormatDebug("failed fetching data on chain tracker init, retry", utils.Attribute{Key: "retry Num", Value: idx}, utils.Attribute{Key: "endpoint", Value: cs.endpoint.String()})
// MAG-2159 finding 3: same startup spacing for the previous-blocks init retries.
if cs.flatPollInterval > 0 && idx < initRetriesCount-1 {
if ctxErr := sleepCtx(ctx, cs.flatPollInterval); ctxErr != nil {
return ctxErr
}
}
}
err = cs.retryInitFetch(ctx, "block fetch", func(ctx context.Context) error {
_, err := cs.fetchAllPreviousBlocks(ctx, newLatestBlock)
return err
})
if err != nil {
// Add suggestion if error is due to context deadline exceeded
if errors.Is(err, common.ContextDeadlineExceededError) {
utils.LavaFormatError("suggestion -- If you encounter a 'context deadline exceeded' error, consider increasing the timeout configuration in the 'node-url' config option. Sometimes, the initial HTTPS/WSS communication takes a long time to establish a connection.", nil)
}
return utils.LavaFormatError("critical -- failed fetching data from the node, chain tracker creation error", err, utils.Attribute{Key: "endpoint", Value: cs.endpoint})
return err
}
cs.lastRealPoll = time.Now() // the init fetch is this tracker's first real poll
return nil
Expand Down
129 changes: 129 additions & 0 deletions protocol/chaintracker/init_retry_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
package chaintracker

import (
"bytes"
"context"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"

"github.com/magma-Devs/smart-router/utils"
"github.com/rs/zerolog"
"github.com/stretchr/testify/require"
)

// stepFailingFetcher fails the head fetch headFailures times and the block-hash fetch
// blockFailures times before answering, counting every call of each.
type stepFailingFetcher struct {
advancingFetcher
headFailures int32
blockFailures int32
headCalls atomic.Int32
blockCalls atomic.Int32
}

func (f *stepFailingFetcher) FetchLatestBlockNum(ctx context.Context) (int64, error) {
if f.headCalls.Add(1) <= f.headFailures {
return 0, fmt.Errorf("head fetch refused")
}
return f.advancingFetcher.FetchLatestBlockNum(ctx)
}

func (f *stepFailingFetcher) FetchBlockHashByNum(ctx context.Context, blockNum int64) (string, error) {
if f.blockCalls.Add(1) <= f.blockFailures {
return "", fmt.Errorf("block fetch refused")
}
return f.advancingFetcher.FetchBlockHashByNum(ctx, blockNum)
}

func newInitRetryTracker(t *testing.T, fetcher ChainFetcher) *ChainTracker {
t.Helper()
tracker := newCustomChainTracker(fetcher, ChainTrackerConfig{
BlocksToSave: 1,
AverageBlockTime: 20 * time.Millisecond,
ServerBlockMemory: 100,
ChainId: "ETH1",
ParseDirectiveEnabled: true,
FlatPollInterval: time.Millisecond,
})
ct, ok := tracker.(*ChainTracker)
require.True(t, ok)
return ct
}

// The two init steps used to be two loops that had drifted apart: the head fetch got
// initRetriesCount+1 attempts, the block fetch initRetriesCount, under a comment claiming the
// same spacing. A node that answers on the last attempt of either step must start (MAG-5022).
func TestFetchInitDataWithRetry_BothStepsGetTheSameAttempts(t *testing.T) {
fetcher := &stepFailingFetcher{headFailures: initRetriesCount, blockFailures: initRetriesCount}
fetcher.block.Store(100)
ct := newInitRetryTracker(t, fetcher)

require.NoError(t, ct.fetchInitDataWithRetry(context.Background()),
"a node that answers on the last attempt of each step must start")
require.Equal(t, int32(initRetriesCount+1), fetcher.headCalls.Load(), "head fetch: one attempt and initRetriesCount retries")
require.Equal(t, int32(initRetriesCount+1), fetcher.blockCalls.Load(), "block fetch: the same attempts as the head fetch")
}

// A node that never answers fails the start after the same attempts on either step, and the
// failure reaches the caller with its cause intact.
func TestFetchInitDataWithRetry_SpentAttemptsFailTheStep(t *testing.T) {
for name, fetcher := range map[string]*stepFailingFetcher{
"head fetch": {headFailures: initRetriesCount + 1},
"block fetch": {blockFailures: initRetriesCount + 1},
} {
t.Run(name, func(t *testing.T) {
fetcher.block.Store(100)
ct := newInitRetryTracker(t, fetcher)
err := ct.fetchInitDataWithRetry(context.Background())
require.Error(t, err)
require.Contains(t, err.Error(), name+" refused", "the step's own failure is the cause")
calls := fetcher.headCalls.Load()
if name == "block fetch" {
calls = fetcher.blockCalls.Load()
}
require.Equal(t, int32(initRetriesCount+1), calls, "the step stops after its attempts, not before and not after")
})
}
}

// A dead endpoint used to log "critical ... chain tracker creation error" at ERROR on every
// start attempt, although the start is retried for as long as the endpoint is down and the
// retry loop already warns once per attempt. The spent-attempts failure is now returned to that
// loop and logged no higher than debug (MAG-5022).
func TestFetchInitDataWithRetry_SpentAttemptsLogNoError(t *testing.T) {
var logged bytes.Buffer
restore := utils.SetStdLogger(zerolog.New(&logged).Level(zerolog.WarnLevel))
t.Cleanup(restore)

fetcher := &stepFailingFetcher{headFailures: initRetriesCount + 1}
ct := newInitRetryTracker(t, fetcher)
require.Error(t, ct.fetchInitDataWithRetry(context.Background()))

for _, line := range strings.Split(strings.TrimSpace(logged.String()), "\n") {
if line == "" {
continue
}
require.NotContains(t, line, `"level":"error"`, "a retried start failure is not an ERROR: %s", line)
}
}

// Cancellation during the spacing between attempts returns at once with the context's error,
// not the step's.
func TestFetchInitDataWithRetry_CancelBetweenAttempts(t *testing.T) {
fetcher := &stepFailingFetcher{headFailures: initRetriesCount + 1}
ct := newInitRetryTracker(t, fetcher)
ct.flatPollInterval = time.Hour
ctx, cancel := context.WithCancel(context.Background())
go func() {
for fetcher.headCalls.Load() == 0 {
time.Sleep(time.Millisecond)
}
cancel()
}()
err := ct.fetchInitDataWithRetry(ctx)
require.ErrorIs(t, err, context.Canceled)
require.Equal(t, int32(1), fetcher.headCalls.Load(), "the cancelled wait makes no further attempt")
}
Loading