Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
bd6cbbb
fix(rpcsmartrouter): let a WebSocket teardown win over a restore in f…
avitenzer Oct 12, 2026
3b13631
fix(rpcsmartrouter): join a WebSocket subscription that became active…
avitenzer Oct 12, 2026
ec12e1e
Merge remote-tracking branch 'origin/fix/MAG-2546-ws-subscription-cou…
avitenzer Oct 13, 2026
21fd439
Merge remote-tracking branch 'origin/fix/MAG-2547-ws-reconnect-vs-cli…
avitenzer Oct 13, 2026
1edee77
fix(rpcsmartrouter): drop a slow subscription client instead of losin…
avitenzer Oct 13, 2026
22b9fe6
fix(rpcsmartrouter): retire a dead pooled WebSocket connection so the…
avitenzer Oct 13, 2026
cfb4738
fix(rpcsmartrouter): share one gRPC stream among concurrent identical…
avitenzer Oct 13, 2026
cc87221
Merge main into fix/MAG-4242-subscriptions
AnnaR-prog Oct 17, 2026
0f0ad88
refactor(rpcsmartrouter): drop retirement's JSON-RPC error branch the…
AnnaR-prog Oct 15, 2026
569e4d6
test(rpcsmartrouter): race the unsubscribe against a restore on a con…
AnnaR-prog Oct 15, 2026
624a956
fix(rpcsmartrouter): release an unsubscribing client without calling …
AnnaR-prog Oct 15, 2026
4776966
fix(rpcsmartrouter): count a slow consumer only when its channel is f…
AnnaR-prog Oct 15, 2026
cc15d50
fix(rpcsmartrouter): wait briefly for a full subscriber before droppi…
AnnaR-prog Oct 15, 2026
0b84dc9
docs(metrics): document smartrouter_subscription_clients_dropped_tota…
AnnaR-prog Oct 15, 2026
727179f
test(rpcsmartrouter): say why the outage test counts subscribe calls …
AnnaR-prog Oct 15, 2026
3b0c0ef
fix(rpcsmartrouter): close a dropped WebSocket client's socket with 1…
AnnaR-prog Oct 17, 2026
01903fb
fix(chainlib): read the websocket rate-limit flags once per connectio…
AnnaR-prog Oct 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/METRICS.md
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,7 @@ They split into **endpoint-scoped** (`rpc_endpoint_*`) and **router-scoped**
| `smartrouter_ws_connections_active` | Gauge | `spec`, `apiInterface` | Active WebSocket connections. |
| `smartrouter_ws_subscriptions_total` | Counter | `spec`, `apiInterface` | Total WebSocket subscription requests. |
| `smartrouter_ws_subscription_errors_total` | Counter | `spec`, `apiInterface` | Failed WebSocket subscription requests. |
| `smartrouter_subscription_clients_dropped_total` | Counter | `spec`, `apiInterface` | Clients dropped from a shared subscription for falling behind its pushes: the client's reply buffer stayed full for the send grace (100 pushes and 10 ms today, both code constants). Counts WebSocket and gRPC subscriptions alike; `apiInterface` tells them apart. A dropped gRPC client's stream ends with `UNAVAILABLE`; a dropped WebSocket client's socket is closed with close code 1013 (try again later), which ends its other subscriptions on that socket too. |

#### Request breakdown

Expand Down
45 changes: 40 additions & 5 deletions protocol/chainlib/consumer_websocket_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -391,8 +391,14 @@ func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) {
idleFor := atomic.Int64{}
idleFor.Store(time.Now().Unix())
requestsPerSecond := &atomic.Uint64{}
// The rate-limit settings are process-wide flags set once at startup. Read them once here, in
// the connection's own goroutine, so the checker below and the read loop work off one snapshot:
// the checker used to read the globals itself, from a goroutine nothing waits for, which raced
// any later write of them (a test that builds the root command registers the flags again).
wsRateLimit := WebSocketRateLimit
wsBanDuration := WebSocketBanDuration
go func() {
if WebSocketRateLimit <= 0 && cwm.headerRateLimit <= 0 && GetMaxIdleTimeInSeconds() <= 0 {
if wsRateLimit <= 0 && cwm.headerRateLimit <= 0 && GetMaxIdleTimeInSeconds() <= 0 {
return
}
ticker := time.NewTicker(time.Second) // rate limit per second.
Expand Down Expand Up @@ -426,15 +432,15 @@ func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) {
return
}
}
if cwm.headerRateLimit > 0 || WebSocketRateLimit > 0 {
if cwm.headerRateLimit > 0 || wsRateLimit > 0 {
// check if rate limit reached, and ban is required
currentRequestsPerSecondLoad := requestsPerSecond.Load()
if WebSocketBanDuration > 0 && (currentRequestsPerSecondLoad > cwm.headerRateLimit || currentRequestsPerSecondLoad > uint64(WebSocketRateLimit)) {
if wsBanDuration > 0 && (currentRequestsPerSecondLoad > cwm.headerRateLimit || currentRequestsPerSecondLoad > uint64(wsRateLimit)) {
// wait the ban duration before resetting the store.
select {
case <-webSocketCtx.Done():
return
case <-time.After(WebSocketBanDuration): // just continue
case <-time.After(wsBanDuration): // just continue
}
}
requestsPerSecond.Store(0)
Expand Down Expand Up @@ -462,7 +468,7 @@ func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) {
// Check rate limit is met
currentRequestsPerSecond := requestsPerSecond.Add(1)
if (cwm.headerRateLimit > 0 && currentRequestsPerSecond > cwm.headerRateLimit) ||
(WebSocketRateLimit > 0 && currentRequestsPerSecond > uint64(WebSocketRateLimit)) {
(wsRateLimit > 0 && currentRequestsPerSecond > uint64(wsRateLimit)) {
rateLimitResponse, err := cwm.handleRateLimitReached(msg)
if err == nil {
sendWS(webSocketMsgWithType{messageType: messageType, msg: rateLimitResponse})
Expand Down Expand Up @@ -674,6 +680,25 @@ func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) {
utils.LogAttr("userIp", userIp),
utils.LogAttr("params", protocolMessage.GetRPCMessage().GetParams()),
)

// A client the manager dropped for not keeping up is told the one way a WebSocket
// can tell it: its socket closes with 1013 (try again later), so it reconnects and
// resubscribes instead of waiting on a socket nothing will ever push to again. The
// frame goes through the writer like any other, and the writer then tears the
// connection down, so the deferred UnsubscribeAll ends the client's other
// subscriptions on it. Any other end of a subscription leaves the socket open.
if errors.Is(cwm.subscriptionEndError(dappID, userIp), ErrSubscriptionClientTooSlow) {
utils.LavaFormatDebug("websocket subscription client did not keep up, closing its connection",
utils.LogAttr("GUID", webSocketCtx),
utils.LogAttr("dappID", dappID),
utils.LogAttr("userIp", userIp),
)
sendWS(webSocketMsgWithType{
messageType: websocket.CloseMessage,
msg: websocket.FormatCloseMessage(websocket.CloseTryAgainLater, ErrSubscriptionClientTooSlow.Error()),
final: true,
})
}
}()
}

Expand All @@ -689,6 +714,16 @@ func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) {
}
}

// subscriptionEndError asks the subscription manager why it ended a subscription of this
// connection, when the manager reports that at all; nil for an ordinary end.
func (cwm *ConsumerWebsocketManager) subscriptionEndError(dappID, userIp string) error {
reporter, ok := cwm.wsSubscriptionManager.(WSSubscriptionEndReporter)
if !ok {
return nil
}
return reporter.SubscriptionEndError(dappID, userIp, cwm.WebsocketConnectionUID)
}

// buildUnsubscribeSuccessReply synthesizes a JSON-RPC 2.0 success response for an
// eth_unsubscribe / unsubscribe that the consumer satisfied locally — provider
// relays do not surface an unsubscribe acknowledgement frame, so we fabricate
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package chainlib

import (
"context"
"errors"
"net/http"
"testing"
"time"

"github.com/gorilla/websocket"
"github.com/magma-Devs/smart-router/protocol/common"
"github.com/magma-Devs/smart-router/protocol/metrics"
pairingtypes "github.com/magma-Devs/smart-router/types/relay"
"github.com/stretchr/testify/require"
)

// endingSubscriptions acknowledges every subscribe, pushes once and ends the subscription, and
// reports endErr for the connection as a WSSubscriptionEndReporter does.
type endingSubscriptions struct {
WSSubscriptionManager
endErr error
}

func (endingSubscriptions) StartSubscription(context.Context, ProtocolMessage, string, string, string, *metrics.RelayMetrics) (*pairingtypes.RelayReply, <-chan *pairingtypes.RelayReply, error) {
replies := make(chan *pairingtypes.RelayReply, 1)
replies <- &pairingtypes.RelayReply{Data: []byte(`{"jsonrpc":"2.0","method":"eth_subscription","params":{"subscription":"0x1","result":{"number":1}}}`)}
close(replies)
return &pairingtypes.RelayReply{Data: []byte(`{"jsonrpc":"2.0","id":1,"result":"0x1"}`)}, replies, nil
}

// UnsubscribeAll runs when the connection closes; there is nothing to tear down.
func (endingSubscriptions) UnsubscribeAll(context.Context, string, string, string, *metrics.RelayMetrics) error {
return nil
}

func (m endingSubscriptions) SubscriptionEndError(string, string, string) error { return m.endErr }

func dialSubscriptionListener(t *testing.T, manager WSSubscriptionManager) *websocket.Conn {
t.Helper()
serveCtx, cancelServe := context.WithCancel(context.Background())
t.Cleanup(cancelServe)
logger, err := metrics.NewRPCConsumerLogs(nil, nil, nil)
require.NoError(t, err)
listener, addr := startTestJsonRPCListenerWithSubscriptions(t, serveCtx, common.DEFAULT_HEALTH_PATH, wsSubscribeStub{}, logger, manager)
t.Cleanup(func() {
shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 5*time.Second)
defer cancelShutdown()
_ = listener.Shutdown(shutdownCtx)
})
client, _, err := websocket.DefaultDialer.Dial("ws://"+addr+"/ws", http.Header{"project-id": {"acme"}})
require.NoError(t, err)
t.Cleanup(func() { _ = client.Close() })
return client
}

const subscribeFrame = `{"jsonrpc":"2.0","method":"eth_subscribe","params":["newHeads"],"id":1}`

// MAG-4242: a subscription the manager ended because the client did not keep up closes the
// client's socket with 1013 (try again later), through the connection's writer, so the client
// reconnects instead of waiting on a socket nothing will push to again.
func TestConsumerWebsocketManager_SlowSubscriberIsClosedWithTryAgainLater(t *testing.T) {
client := dialSubscriptionListener(t, endingSubscriptions{endErr: ErrSubscriptionClientTooSlow})
require.NoError(t, client.WriteMessage(websocket.TextMessage, []byte(subscribeFrame)))

for frames := 0; ; frames++ {
_ = client.SetReadDeadline(time.Now().Add(3 * time.Second))
_, _, err := client.ReadMessage()
if err == nil {
require.Less(t, frames, 2, "an acknowledgement and one push, then the close")
continue
}
var closeErr *websocket.CloseError
require.ErrorAs(t, err, &closeErr, "the client got %v instead of a close frame", err)
require.Equal(t, websocket.CloseTryAgainLater, closeErr.Code)
require.Equal(t, ErrSubscriptionClientTooSlow.Error(), closeErr.Text)
return
}
}

// Every other end of a subscription leaves the socket open: the client subscribes again on it.
func TestConsumerWebsocketManager_OrdinarySubscriptionEndKeepsTheSocket(t *testing.T) {
for _, endErr := range []error{nil, errors.New("some other end")} {
client := dialSubscriptionListener(t, endingSubscriptions{endErr: endErr})
for round := range 2 {
require.NoError(t, client.WriteMessage(websocket.TextMessage, []byte(subscribeFrame)))
for frame := range 2 { // the acknowledgement and the push, in either order
_ = client.SetReadDeadline(time.Now().Add(3 * time.Second))
_, _, err := client.ReadMessage()
require.NoError(t, err, "end %v, round %d, frame %d: the socket must stay open", endErr, round, frame)
}
}
}
}
18 changes: 18 additions & 0 deletions protocol/chainlib/ws_subscription_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package chainlib

import (
"context"
"errors"

"github.com/magma-Devs/smart-router/protocol/metrics"
pairingtypes "github.com/magma-Devs/smart-router/types/relay"
Expand Down Expand Up @@ -54,3 +55,20 @@ type WSSubscriptionManager interface {
metricsData *metrics.RelayMetrics,
) error
}

// ErrSubscriptionClientTooSlow is what a WSSubscriptionEndReporter reports for a connection it
// dropped a subscription of because the client did not keep up with the pushes. Its text is the
// reason in the close frame the client gets.
var ErrSubscriptionClientTooSlow = errors.New("subscription client did not keep up")

// WSSubscriptionEndReporter is an optional interface on WSSubscriptionManager, the WebSocket
// counterpart of GRPCSubscriptionEndReporter. A WebSocket subscription has no end of its own to
// carry a reason: its reply channel just closes. A manager that ends a client's subscription for
// a reason the client must hear reports it here, and the ingress tells the client by closing the
// socket. Any other end leaves the socket open, as before.
type WSSubscriptionEndReporter interface {
// SubscriptionEndError returns why the manager ended a subscription of this connection
// (ErrSubscriptionClientTooSlow), or nil for an ordinary end. The ingress reads it after a
// reply channel has closed; a reason is read once.
SubscriptionEndError(dappID string, consumerIp string, webSocketConnectionUniqueId string) error
}
54 changes: 54 additions & 0 deletions protocol/common/safe_channel_sender.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,60 @@ func (scs *SafeChannelSender[T]) LockAndSendAsynchronously(msg T) {
}()
}

// TrySendOutcome is what TrySend did with a message.
type TrySendOutcome int

const (
// TrySendDelivered: the receiver had room, and has the message.
TrySendDelivered TrySendOutcome = iota
// TrySendFull: the receiver's channel is full, so it is not keeping up.
TrySendFull
// TrySendClosed: the sender was closed or its context ended, so nobody is receiving. The
// receiver left; it was not slow.
TrySendClosed
)

// TrySend delivers msg if the receiver has room for it and reports what happened. It never
// waits and never retries: a receiver that is not keeping up is reported, not waited for, so
// one slow client cannot stall the fan-out to every other client of a shared subscription
// (MAG-4242). A closed sender, or one whose context has ended, reports TrySendClosed: a caller
// that drops slow receivers must not take a receiver that left for one.
func (scs *SafeChannelSender[T]) TrySend(msg T) TrySendOutcome {
return scs.SendWithin(msg, 0)
}

// SendWithin is TrySend with a bounded wait: when the receiver's channel is full it waits up to
// grace for room before reporting TrySendFull. A receiver that is reading, however slowly, frees
// a slot within the grace and gets the message; only one that is a full channel behind and frees
// nothing for grace is reported full. The wait holds the sender's lock, as Send's retries do, so
// a Close waits for it: keep grace short. A context that ends during the wait reports
// TrySendClosed.
func (scs *SafeChannelSender[T]) SendWithin(msg T, grace time.Duration) TrySendOutcome {
scs.lock.Lock()
defer scs.lock.Unlock()
if scs.closed || scs.ctx.Err() != nil {
return TrySendClosed
}
select {
case scs.ch <- msg:
return TrySendDelivered
default:
}
if grace <= 0 {
return TrySendFull
}
timer := time.NewTimer(grace)
defer timer.Stop()
select {
case scs.ch <- msg:
return TrySendDelivered
case <-scs.ctx.Done():
return TrySendClosed
case <-timer.C:
return TrySendFull
}
}

// Used when you need to wait for the other side to receive the message.
func (scs *SafeChannelSender[T]) Send(msg T) {
scs.lock.Lock()
Expand Down
45 changes: 45 additions & 0 deletions protocol/common/safe_channel_sender_trysend_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package common

import (
"context"
"testing"

"github.com/stretchr/testify/require"
)

// TestSafeChannelSender_TrySend pins the non-waiting send the subscription fan-out uses
// (MAG-4242): a push lands when there is room, is refused as full when the receiver is behind,
// and is refused as closed after the sender closed or its context ended, so a caller that drops
// slow receivers can tell one that left from one that fell behind. None of it ever blocks.
func TestSafeChannelSender_TrySend(t *testing.T) {
t.Run("lands while there is room, full once the receiver is behind", func(t *testing.T) {
ch := make(chan int, 2)
sender := NewSafeChannelSender(context.Background(), ch)
require.Equal(t, TrySendDelivered, sender.TrySend(1))
require.Equal(t, TrySendDelivered, sender.TrySend(2))
require.Equal(t, TrySendFull, sender.TrySend(3), "a full receiver is reported, not waited for")
require.Equal(t, 1, <-ch)
require.Equal(t, TrySendDelivered, sender.TrySend(4), "room again once the receiver caught up")
})
t.Run("closed after close", func(t *testing.T) {
ch := make(chan int, 1)
sender := NewSafeChannelSender(context.Background(), ch)
sender.Close()
require.Equal(t, TrySendClosed, sender.TrySend(1))
})
t.Run("closed after its context ended, even with room", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
ch := make(chan int, 1)
sender := NewSafeChannelSender(ctx, ch)
cancel()
require.Equal(t, TrySendClosed, sender.TrySend(1))
})
t.Run("closed after its context ended, with the receiver behind", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
ch := make(chan int, 1)
sender := NewSafeChannelSender(ctx, ch)
require.Equal(t, TrySendDelivered, sender.TrySend(1))
cancel()
require.Equal(t, TrySendClosed, sender.TrySend(2), "a receiver that left is not a slow one")
})
}
4 changes: 4 additions & 0 deletions protocol/metrics/consumer_metrics_manager_inf.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ func (NoOpConsumerMetrics) RecordStickyClaim(string, string, string)
func (NoOpConsumerMetrics) SetCSMReportedProvidersCount(string, string, int) {}
func (NoOpConsumerMetrics) SetWsSubscriptionRequestMetric(string, string) {}
func (NoOpConsumerMetrics) SetFailedWsSubscriptionRequestMetric(string, string) {}
func (NoOpConsumerMetrics) SetSubscriptionClientDroppedMetric(string, string) {}
func (NoOpConsumerMetrics) SetWebSocketConnectionActive(string, string, bool) {}
func (NoOpConsumerMetrics) SetVersion(string) {}
func (NoOpConsumerMetrics) StartSelectionStatsUpdater(context.Context, time.Duration) {}
Expand Down Expand Up @@ -129,6 +130,9 @@ type ConsumerMetricsManagerInf interface {
// --- WebSocket (DirectWSSubscriptionManager) ---
SetWsSubscriptionRequestMetric(chainId string, apiInterface string)
SetFailedWsSubscriptionRequestMetric(chainId string, apiInterface string)
// SetSubscriptionClientDroppedMetric counts a client dropped from a shared subscription
// (WebSocket or gRPC) because it did not keep up with the pushes.
SetSubscriptionClientDroppedMetric(chainId string, apiInterface string)
SetWebSocketConnectionActive(chainId string, apiInterface string, add bool)

// --- Misc (RPCConsumerLogs / rpcsmartrouter.go) ---
Expand Down
15 changes: 15 additions & 0 deletions protocol/metrics/smartrouter_metrics_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ type SmartRouterMetricsManager struct {
routerWsConnectionsActive *prometheus.GaugeVec
routerWsSubscriptionsTotal *prometheus.CounterVec
routerWsSubscriptionErrors *prometheus.CounterVec
subscriptionClientsDropped *prometheus.CounterVec

// Cross-validation group metrics
crossValidationRequestsTotalMetric *prometheus.CounterVec // smartrouter_cross_validation_requests_total {spec, apiInterface, method}
Expand Down Expand Up @@ -462,6 +463,11 @@ func NewSmartRouterMetricsManager(options SmartRouterMetricsManagerOptions) *Sma
Help: "Total failed WebSocket subscription requests on the smart router.",
}, []string{"spec", "apiInterface"})

subscriptionClientsDropped := prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "smartrouter_subscription_clients_dropped_total",
Help: "Clients dropped from a shared subscription (WebSocket or gRPC) because they did not keep up with its pushes.",
}, []string{"spec", "apiInterface"})

crossValidationLabels := []string{"spec", "apiInterface", "method"}
crossValidationProviderLabels := []string{"spec", "apiInterface", "method", "provider_address"}
crossValidationRequestsTotalMetric := prometheus.NewCounterVec(prometheus.CounterOpts{
Expand Down Expand Up @@ -746,6 +752,7 @@ func NewSmartRouterMetricsManager(options SmartRouterMetricsManagerOptions) *Sma
routerWsConnectionsActive = registerOrReuse(routerWsConnectionsActive)
routerWsSubscriptionsTotal = registerOrReuse(routerWsSubscriptionsTotal)
routerWsSubscriptionErrors = registerOrReuse(routerWsSubscriptionErrors)
subscriptionClientsDropped = registerOrReuse(subscriptionClientsDropped)
endpointEndToEndLatency = registerOrReuse(endpointEndToEndLatency)
crossValidationRequestsTotalMetric = registerOrReuse(crossValidationRequestsTotalMetric)
crossValidationSuccessTotalMetric = registerOrReuse(crossValidationSuccessTotalMetric)
Expand Down Expand Up @@ -842,6 +849,7 @@ func NewSmartRouterMetricsManager(options SmartRouterMetricsManagerOptions) *Sma
routerWsConnectionsActive: routerWsConnectionsActive,
routerWsSubscriptionsTotal: routerWsSubscriptionsTotal,
routerWsSubscriptionErrors: routerWsSubscriptionErrors,
subscriptionClientsDropped: subscriptionClientsDropped,

// Cross-validation group
crossValidationRequestsTotalMetric: crossValidationRequestsTotalMetric,
Expand Down Expand Up @@ -2088,6 +2096,13 @@ func (m *SmartRouterMetricsManager) SetFailedWsSubscriptionRequestMetric(chainId
m.routerWsSubscriptionErrors.WithLabelValues(chainId, apiInterface).Inc()
}

func (m *SmartRouterMetricsManager) SetSubscriptionClientDroppedMetric(chainId, apiInterface string) {
if m == nil {
return
}
m.subscriptionClientsDropped.WithLabelValues(chainId, apiInterface).Inc()
}

func (m *SmartRouterMetricsManager) SetWebSocketConnectionActive(chainId, apiInterface string, add bool) {
if m == nil {
return
Expand Down
Loading