From 743bd7b818590be1d07a0fe5b1cb094b1e144846 Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Wed, 16 Sep 2026 22:23:42 +0300 Subject: [PATCH 1/8] fix(provideroptimizer): bound the per-provider relay time history updateRelayTime appended a time.Time to a per-provider slice on every relay, success or failure, and nothing ever truncated it: only the debug reset endpoints clear the cache. That is 24 bytes per relay for the life of the process, so a busy pod's heap climbed for weeks until the container limit killed it. Each reallocation also copied the whole history under the provider's stripe lock, and the slice's Location pointers made the GC walk all of it every cycle. calculateHalfTime reads exactly one element, the median, so the history is now a fixed 128-slot ring in arrival order. Its median means "the time this provider takes to serve half a window of relays", which is the input the half-life heuristic wants; the unbounded median drifted to half the uptime and the MaxHalfTime clamp discarded it anyway. The ring is mutated in place under the stripe lock, so the cached pointer never needs a replacement Set. Jira ticket: MAG-3722 --- .../provideroptimizer/provider_optimizer.go | 78 ++++++++++++++----- .../relay_time_window_test.go | 76 ++++++++++++++++++ 2 files changed, 136 insertions(+), 18 deletions(-) create mode 100644 protocol/provideroptimizer/relay_time_window_test.go diff --git a/protocol/provideroptimizer/provider_optimizer.go b/protocol/provideroptimizer/provider_optimizer.go index 47bf0712a..6f569a4a1 100644 --- a/protocol/provideroptimizer/provider_optimizer.go +++ b/protocol/provideroptimizer/provider_optimizer.go @@ -877,15 +877,53 @@ func (po *ProviderOptimizer) updateDecayingWeightedAverage(providerData Provider return providerData, nil } -// updateRelayTime adds a relay sample time to a provider's data +// relayStatsWindowSize bounds how many relay sample times are kept per provider. +// +// calculateHalfTime reads exactly one of them, the median, so the window only has +// to be wide enough for that median to mean "the time it takes this provider to +// serve half a window of relays". Unbounded, the slice grew by 24 bytes on every +// relay for the life of the process and its median drifted to half the uptime, +// which the MaxHalfTime clamp then discarded anyway (MAG-3722). +const relayStatsWindowSize = 128 + +// relayTimeWindow holds a provider's most recent relay sample times in arrival order. +// It is mutated in place under the provider's stripe lock, so the pointer stored in +// providerRelayStats never needs a replacement Set once it is in the cache. +type relayTimeWindow struct { + times [relayStatsWindowSize]time.Time + next int // slot the next sample lands in + count int // samples held, at most len(times) +} + +// add records one sample, overwriting the oldest once the window is full. +func (w *relayTimeWindow) add(sampleTime time.Time) { + w.times[w.next] = sampleTime + w.next = (w.next + 1) % len(w.times) + if w.count < len(w.times) { + w.count++ + } +} + +// median returns the middle sample by arrival order, which for monotonic sample +// times is the median time. ok is false while the window is empty. +func (w *relayTimeWindow) median() (median time.Time, ok bool) { + if w.count == 0 { + return time.Time{}, false + } + oldest := (w.next - w.count + len(w.times)) % len(w.times) + return w.times[(oldest+(w.count-1)/2)%len(w.times)], true +} + +// updateRelayTime adds a relay sample time to a provider's window. func (po *ProviderOptimizer) updateRelayTime(providerAddress string, sampleTime time.Time) { - times := po.getRelayStatsTimes(providerAddress) - if len(times) == 0 { - po.providerRelayStats.Set(providerAddress, []time.Time{sampleTime}, 1) + window := po.getRelayStatsWindow(providerAddress) + if window == nil { + window = &relayTimeWindow{} + window.add(sampleTime) + po.providerRelayStats.Set(providerAddress, window, 1) return } - times = append(times, sampleTime) - po.providerRelayStats.Set(providerAddress, times, 1) + window.add(sampleTime) } // calculateHalfTime calculates a provider's half life time for a relay sampled in sampleTime @@ -901,13 +939,16 @@ func (po *ProviderOptimizer) calculateHalfTime(providerAddress string, sampleTim return halfTime } -// getRelayStatsTimeDiff returns the time passed since the provider optimizer's saved relay times median +// getRelayStatsTimeDiff returns the time passed since the median of the provider's recent relay times func (po *ProviderOptimizer) getRelayStatsTimeDiff(providerAddress string, sampleTime time.Time) time.Duration { - times := po.getRelayStatsTimes(providerAddress) - if len(times) == 0 { + window := po.getRelayStatsWindow(providerAddress) + if window == nil { + return 0 + } + medianTime, ok := window.median() + if !ok { return 0 } - medianTime := times[(len(times)-1)/2] if medianTime.Before(sampleTime) { return sampleTime.Sub(medianTime) } @@ -919,16 +960,17 @@ func (po *ProviderOptimizer) getRelayStatsTimeDiff(providerAddress string, sampl return time.Since(medianTime) } -func (po *ProviderOptimizer) getRelayStatsTimes(providerAddress string) []time.Time { +// getRelayStatsWindow returns the provider's relay time window, or nil when none is cached yet. +func (po *ProviderOptimizer) getRelayStatsWindow(providerAddress string) *relayTimeWindow { storedVal, found := po.providerRelayStats.Get(providerAddress) - if found { - times, ok := storedVal.([]time.Time) - if !ok { - utils.LavaFormatFatal("invalid usage of optimizer relay stats cache", nil, utils.Attribute{Key: "storedVal", Value: storedVal}) - } - return times + if !found { + return nil + } + window, ok := storedVal.(*relayTimeWindow) + if !ok { + utils.LavaFormatFatal("invalid usage of optimizer relay stats cache", nil, utils.Attribute{Key: "storedVal", Value: storedVal}) } - return nil + return window } func NewProviderOptimizer(strategy Strategy, averageBlockTIme time.Duration, wantedNumProvidersInConcurrency uint, consumerOptimizerQoSClient consumerOptimizerQoSClientInf, chainId string) *ProviderOptimizer { diff --git a/protocol/provideroptimizer/relay_time_window_test.go b/protocol/provideroptimizer/relay_time_window_test.go new file mode 100644 index 000000000..6e9585cf7 --- /dev/null +++ b/protocol/provideroptimizer/relay_time_window_test.go @@ -0,0 +1,76 @@ +package provideroptimizer + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestRelayTimeWindow_BoundedAndMedian pins the two properties calculateHalfTime relies on: +// the window never holds more than relayStatsWindowSize samples, and its median is the +// middle sample of what it holds, by arrival order. +func TestRelayTimeWindow_BoundedAndMedian(t *testing.T) { + base := time.Date(2026, 9, 16, 0, 0, 0, 0, time.UTC) + w := &relayTimeWindow{} + + _, ok := w.median() + require.False(t, ok, "an empty window has no median") + + w.add(base) + median, ok := w.median() + require.True(t, ok) + require.Equal(t, base, median, "one sample is its own median") + + const total = 10 * relayStatsWindowSize + for i := 1; i < total; i++ { + w.add(base.Add(time.Duration(i) * time.Second)) + } + require.Equal(t, relayStatsWindowSize, w.count, "the window must stop growing at its size") + + // The window holds samples [total-size, total). Its median is the one at offset + // (size-1)/2 from the oldest retained sample. + oldest := total - relayStatsWindowSize + want := base.Add(time.Duration(oldest+(relayStatsWindowSize-1)/2) * time.Second) + median, ok = w.median() + require.True(t, ok) + require.Equal(t, want, median) +} + +// TestUpdateRelayTime_DoesNotGrowWithRelays is the regression for MAG-3722: every relay +// used to append 24 bytes to a per-provider slice that nothing ever truncated, so a busy +// pod's heap grew for the life of the process. The window keeps the half-life input +// meaningful, "the time this provider takes to serve half a window of relays", while +// holding a fixed number of samples no matter how many relays arrive. +func TestUpdateRelayTime_DoesNotGrowWithRelays(t *testing.T) { + po := setupProviderOptimizer(1) + const addr = "provider-under-load" + base := time.Date(2026, 9, 16, 0, 0, 0, 0, time.UTC) + + const relays = 10_000 + for i := 0; i < relays; i++ { + po.updateRelayTime(addr, base.Add(time.Duration(i)*time.Second)) + } + po.providerRelayStats.Wait() // ristretto Set is asynchronous + + window := po.getRelayStatsWindow(addr) + require.NotNil(t, window, "the provider's window must be cached") + require.Equal(t, relayStatsWindowSize, window.count, + "after %d relays the window must hold exactly its size, not one entry per relay", relays) + + // The median of the last relayStatsWindowSize one-second samples sits + // (size-1)/2 seconds after the oldest retained sample, so the age of the median + // seen from the newest sample is size/2 seconds. + newest := base.Add(time.Duration(relays-1) * time.Second) + wantAge := time.Duration(relayStatsWindowSize/2) * time.Second + require.Equal(t, wantAge, po.getRelayStatsTimeDiff(addr, newest)) + + // A busy provider therefore keeps the default half-life; only a provider slower + // than a window per DefaultHalfLifeTime stretches it. + require.Equal(t, po.calculateHalfTime(addr, newest), po.calculateHalfTime("never-seen", newest), + "a provider serving relays every second must decay on the default half-life") + + po.ResetState() + po.providerRelayStats.Wait() + require.Nil(t, po.getRelayStatsWindow(addr), "ResetState must drop the window") +} From 9e6d1bb3ee6eeabd637af2a854193fa7b4599303 Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Wed, 16 Sep 2026 22:23:42 +0300 Subject: [PATCH 2/8] fix(ws): release a client's subscriptions and writer when its connection ends Three ways a websocket client's departure left state behind for the life of the process, all on the same path. The subscription manager never released a JOINED client. A second subscriber to a query joins the first one's upstream subscription, and only the creating client had a disconnect watcher: when a joiner closed its socket its reply channel stayed open, which parks the ingress forwarder goroutine ranging over it, it kept counting as connected, and the upstream subscription outlived every real subscriber. The creator's watcher had the mirror omission: it deleted the client's sender without closing it, so its forwarder parked too. Every client now attaches through one helper and leaves through one helper that closes the channel, drops the bookkeeping and wakes the watcher, so no path can skip a step again. An explicit unsubscribe wakes the watcher instead of leaving it parked until the socket closes, and UnsubscribeAll now tells the node when the last client leaves, which it used to skip. The shared subscription's context was derived from the creating client's connection, so the creator leaving ended the upstream listener underneath everyone who had joined. It is now owned by the manager, the rule the gRPC manager already applies. The ingress writer goroutine returned on a failed write without telling anyone. Every later frame sender then blocked forever, and the read loop's own error path is one of them, so the handler never returned and fasthttp kept the 128 KiB read buffer, the handler goroutine, the ticker goroutine and the per-IP limiter slot. The writer now closes a done channel on the way out, senders select on it, and a failed write sets an immediate read deadline so the read loop wakes. The handler also releases every subscription it holds through UnsubscribeAll on return, synchronously, rather than relying on a cancelled context being noticed. Smaller entries that never went away: the id mapper's per-client counter (one per connection that ever subscribed) is dropped with the client, and rate limiters idle for their refill time are swept, since such an entry decides exactly as a fresh one would. Jira ticket: MAG-3722 --- .../chainlib/consumer_websocket_manager.go | 133 +++++--- .../consumer_websocket_writer_test.go | 92 ++++++ .../direct_ws_subscription_manager.go | 284 +++++++++++------- .../direct_ws_subscription_release_test.go | 211 +++++++++++++ .../rpcsmartrouter/subscription_id_mapper.go | 17 ++ .../subscription_id_mapper_leak_test.go | 29 ++ protocol/rpcsmartrouter/websocket_config.go | 77 ++++- .../websocket_config_sweep_test.go | 36 +++ 8 files changed, 708 insertions(+), 171 deletions(-) create mode 100644 protocol/chainlib/consumer_websocket_writer_test.go create mode 100644 protocol/rpcsmartrouter/direct_ws_subscription_release_test.go create mode 100644 protocol/rpcsmartrouter/subscription_id_mapper_leak_test.go create mode 100644 protocol/rpcsmartrouter/websocket_config_sweep_test.go diff --git a/protocol/chainlib/consumer_websocket_manager.go b/protocol/chainlib/consumer_websocket_manager.go index 355664aa4..d9c4deeef 100644 --- a/protocol/chainlib/consumer_websocket_manager.go +++ b/protocol/chainlib/consumer_websocket_manager.go @@ -105,6 +105,75 @@ func (cwm *ConsumerWebsocketManager) handleRateLimitReached(inpData []byte) ([]b return bytesRateLimitError, nil } +// webSocketMsgWithType is one frame queued for the connection's writer goroutine. +type webSocketMsgWithType struct { + messageType int + msg []byte +} + +// websocketFrameWriter is the slice of the connection the writer goroutine touches. +// *websocket.Conn satisfies it; tests hand in a fake whose writes fail on demand. +type websocketFrameWriter interface { + WriteMessage(messageType int, data []byte) error + SetReadDeadline(t time.Time) error +} + +// runWebsocketWriter is the connection's single writer. Single-writer discipline +// guarantees WriteMessage is never called concurrently, which the underlying +// gorilla/fasthttp websocket library does not allow. +// +// It serves regular frames and the server-shutdown close frame, and closes writerDone +// on its way out. A failed write ends it, and that used to leave every later +// enqueueWebsocketFrame caller blocked forever: the read loop's own error path sends a +// frame before it can return, so the handler never returned, and fasthttp kept the +// 128 KiB read buffer, the handler goroutine and the per-IP limiter slot for the life +// of the process (MAG-3722). Now a failed write also sets an immediate read deadline so +// the read loop wakes and the handler tears the connection down. +func runWebsocketWriter(ctx, webSocketCtx context.Context, conn websocketFrameWriter, frames <-chan webSocketMsgWithType, writerDone chan<- struct{}) { + defer close(writerDone) + for { + select { + case <-ctx.Done(): + // Server-wide shutdown — send a CloseGoingAway (1001) frame so the client can + // distinguish an intentional shutdown from a crash, then unblock the read loop by + // setting an immediate read deadline rather than Close()-ing the conn here. + // + // Close() from this goroutine raced gofiber's handler cleanup: it unblocks ReadMessage, + // the handler returns, and gofiber recycles the Conn (fasthttp pooling) WHILE this + // goroutine is still inside Close() — a use-after-recycle data race. SetReadDeadline + // returns immediately (the read loop breaks on the resulting error) and lets the handler, + // the conn's sole owner, do the single Close on return. + _ = conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseGoingAway, "server shutting down")) + _ = conn.SetReadDeadline(time.Now()) + return + case <-webSocketCtx.Done(): + utils.LavaFormatTrace("websocket's context cancelled", utils.LogAttr("GUID", webSocketCtx)) + return + case msg, ok := <-frames: + if !ok { + return + } + if err := conn.WriteMessage(msg.messageType, msg.msg); err != nil { + utils.LavaFormatTrace("error writing msg to the websocket", utils.LogAttr("err", err)) + _ = conn.SetReadDeadline(time.Now()) + return + } + } + } +} + +// enqueueWebsocketFrame hands a frame to the writer goroutine, but gives up as soon as +// ctx (server shutdown) or webSocketCtx (per-connection cleanup) cancels or the writer +// is gone: a plain send would park the caller for good. +func enqueueWebsocketFrame(ctx, webSocketCtx context.Context, writerDone <-chan struct{}, frames chan<- webSocketMsgWithType, msg webSocketMsgWithType) { + select { + case frames <- msg: + case <-ctx.Done(): + case <-webSocketCtx.Done(): + case <-writerDone: + } +} + func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) { // adding metrics for how many active connections we have. cwm.rpcConsumerLogs.SetWebSocketConnectionActive(cwm.chainId, cwm.apiInterface, true) @@ -116,12 +185,9 @@ func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) { err error ) - type webSocketMsgWithType struct { - messageType int - msg []byte - } - websocketConnWriteChan := make(chan webSocketMsgWithType) + // Closed by the writer goroutine on its way out. + writerDone := make(chan struct{}) websocketConn := cwm.websocketConn logger := cwm.rpcConsumerLogs @@ -132,57 +198,26 @@ func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) { webSocketCtx = utils.WithUniqueIdentifier(webSocketCtx, guid) utils.LavaFormatDebug("consumer websocket manager started", utils.LogAttr("GUID", webSocketCtx)) defer func() { + // The connection is over. Release every subscription this client still holds + // right here, synchronously: reply channels close and the forwarder goroutines + // below end now, not whenever a cancelled context happens to be noticed. dappID + // and the remote address are per-connection, so this key matches every + // subscribe the loop registered. + if cwm.wsSubscriptionManager != nil { + dappID, _ := websocketConn.Locals(ProjectIDHeader).(string) + if err := cwm.wsSubscriptionManager.UnsubscribeAll(webSocketCtx, dappID, websocketConn.RemoteAddr().String(), cwm.WebsocketConnectionUID, nil); err != nil { + utils.LavaFormatDebug("error releasing subscriptions on websocket close", utils.LogAttr("err", err), utils.LogAttr("GUID", webSocketCtx)) + } + } cancelWebSocketCtx() // In case there's a problem make sure to cancel the connection utils.LavaFormatDebug("consumer websocket manager stopped", utils.LogAttr("GUID", webSocketCtx)) }() - // sendWS enqueues a frame for the writer goroutine, but unblocks if either - // ctx (server shutdown) or webSocketCtx (per-conn cleanup) cancels first — - // the writer exits on those signals, so a plain send would deadlock the - // caller and leave the connection goroutine stuck (preventing wsWG.Done() - // from firing during graceful shutdown). sendWS := func(msg webSocketMsgWithType) { - select { - case websocketConnWriteChan <- msg: - case <-ctx.Done(): - case <-webSocketCtx.Done(): - } + enqueueWebsocketFrame(ctx, webSocketCtx, writerDone, websocketConnWriteChan, msg) } - // Single writer goroutine: serves both regular messages and the server-shutdown - // close frame. Single-writer discipline guarantees we never call WriteMessage - // concurrently — the underlying gorilla/fasthttp websocket library is not safe - // for concurrent writes. - go func() { - for { - select { - case <-ctx.Done(): - // Server-wide shutdown — send a CloseGoingAway (1001) frame so the client can - // distinguish an intentional shutdown from a crash, then unblock the read loop by - // setting an immediate read deadline rather than Close()-ing the conn here. - // - // Close() from this goroutine raced gofiber's handler cleanup: it unblocks ReadMessage, - // the handler returns, and gofiber recycles the Conn (fasthttp pooling) WHILE this - // goroutine is still inside Close() — a use-after-recycle data race. SetReadDeadline - // returns immediately (the read loop breaks on the resulting error) and lets the handler, - // the conn's sole owner, do the single Close on return. - _ = cwm.websocketConn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseGoingAway, "server shutting down")) - _ = cwm.websocketConn.SetReadDeadline(time.Now()) - return - case <-webSocketCtx.Done(): - utils.LavaFormatTrace("websocket's context cancelled", utils.LogAttr("GUID", webSocketCtx)) - return - case msg, ok := <-websocketConnWriteChan: - if !ok { - return - } - if err := cwm.websocketConn.WriteMessage(msg.messageType, msg.msg); err != nil { - utils.LavaFormatTrace("error writing msg to the websocket") - return - } - } - } - }() + go runWebsocketWriter(ctx, webSocketCtx, cwm.websocketConn, websocketConnWriteChan, writerDone) // set up a routine to check for rate limits or idle time idleFor := atomic.Int64{} diff --git a/protocol/chainlib/consumer_websocket_writer_test.go b/protocol/chainlib/consumer_websocket_writer_test.go new file mode 100644 index 000000000..a1c186e7d --- /dev/null +++ b/protocol/chainlib/consumer_websocket_writer_test.go @@ -0,0 +1,92 @@ +package chainlib + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// failingFrameWriter fails every write and records whether the read loop was woken. +type failingFrameWriter struct { + mu sync.Mutex + writes int + readDeadline bool +} + +func (f *failingFrameWriter) WriteMessage(int, []byte) error { + f.mu.Lock() + defer f.mu.Unlock() + f.writes++ + return errors.New("broken pipe") +} + +func (f *failingFrameWriter) SetReadDeadline(time.Time) error { + f.mu.Lock() + defer f.mu.Unlock() + f.readDeadline = true + return nil +} + +// TestWebsocketWriterFailure_UnblocksSenders is the regression for MAG-3722: when the +// writer goroutine died on a failed write, every later frame sender blocked forever, and +// the read loop's own error path is one of them, so the handler never returned and the +// connection's buffers, goroutines and per-IP limiter slot were held for good. +func TestWebsocketWriterFailure_UnblocksSenders(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + webSocketCtx, cancelWebSocketCtx := context.WithCancel(context.Background()) + defer cancelWebSocketCtx() + + conn := &failingFrameWriter{} + frames := make(chan webSocketMsgWithType) + writerDone := make(chan struct{}) + go runWebsocketWriter(ctx, webSocketCtx, conn, frames, writerDone) + + // The first frame reaches the writer, whose write fails and ends it. + enqueueWebsocketFrame(ctx, webSocketCtx, writerDone, frames, webSocketMsgWithType{messageType: 1, msg: []byte("a")}) + select { + case <-writerDone: + case <-time.After(2 * time.Second): + t.Fatal("a failed write must end the writer goroutine") + } + + conn.mu.Lock() + require.Equal(t, 1, conn.writes) + require.True(t, conn.readDeadline, "a failed write must wake the read loop so the handler can return") + conn.mu.Unlock() + + // Every later sender, the read loop's error reply included, must return at once. + returned := make(chan struct{}) + go func() { + defer close(returned) + enqueueWebsocketFrame(ctx, webSocketCtx, writerDone, frames, webSocketMsgWithType{messageType: 1, msg: []byte("b")}) + }() + select { + case <-returned: + case <-time.After(2 * time.Second): + t.Fatal("a frame sent after the writer died must not block") + } +} + +// TestWebsocketWriter_ExitsOnConnectionContext keeps the normal teardown path honest: +// cancelling the per-connection context ends the writer without touching the conn. +func TestWebsocketWriter_ExitsOnConnectionContext(t *testing.T) { + webSocketCtx, cancelWebSocketCtx := context.WithCancel(context.Background()) + conn := &failingFrameWriter{} + writerDone := make(chan struct{}) + go runWebsocketWriter(context.Background(), webSocketCtx, conn, make(chan webSocketMsgWithType), writerDone) + + cancelWebSocketCtx() + select { + case <-writerDone: + case <-time.After(2 * time.Second): + t.Fatal("the writer must end when the connection context is cancelled") + } + conn.mu.Lock() + require.Equal(t, 0, conn.writes) + conn.mu.Unlock() +} diff --git a/protocol/rpcsmartrouter/direct_ws_subscription_manager.go b/protocol/rpcsmartrouter/direct_ws_subscription_manager.go index a9f32e0dc..6e4dc5569 100644 --- a/protocol/rpcsmartrouter/direct_ws_subscription_manager.go +++ b/protocol/rpcsmartrouter/direct_ws_subscription_manager.go @@ -58,6 +58,11 @@ type directActiveSubscription struct { // Client tracking - multiple clients can share one upstream subscription connectedClients map[string]*common.SafeChannelSender[*pairingtypes.RelayReply] + // clientDetached is closed for a client the moment it leaves this subscription by + // any path, so its disconnect watcher can stop waiting on a socket that may stay + // open for days after the client unsubscribed. + clientDetached map[string]chan struct{} + // First reply cached for deduplication firstReply *pairingtypes.RelayReply @@ -69,12 +74,59 @@ type directActiveSubscription struct { ctx context.Context cancel context.CancelFunc closeSubChan chan struct{} + closeSubOnce sync.Once messagesChan chan *rpcclient.JsonrpcMessage // Restoration state restoring atomic.Bool // Prevents concurrent restoration attempts } +// attachClientLocked registers a client on the subscription. The caller holds dwsm.lock, +// or owns the subscription exclusively as StartSubscription does before publishing it. +func (s *directActiveSubscription) attachClientLocked(clientKey, routerID string, sender *common.SafeChannelSender[*pairingtypes.RelayReply]) { + s.connectedClients[clientKey] = sender + s.clientRouterIDs[clientKey] = routerID + if s.clientDetached == nil { + s.clientDetached = make(map[string]chan struct{}) + } + s.clientDetached[clientKey] = make(chan struct{}) +} + +// releaseClientLocked takes a client off the subscription: it closes the client's reply +// channel so the ingress forwarder goroutine ranging over it ends, drops the client's +// bookkeeping, and wakes the client's disconnect watcher. It is the one place a client +// leaves through, whatever the reason, so no path can forget one of those steps again +// (MAG-3722). Idempotent. Returns the client's router id, "" when it was not attached. +func (s *directActiveSubscription) releaseClientLocked(clientKey string) (routerID string, wasAttached bool) { + sender, wasAttached := s.connectedClients[clientKey] + if wasAttached { + sender.Close() + delete(s.connectedClients, clientKey) + } + routerID = s.clientRouterIDs[clientKey] + delete(s.clientRouterIDs, clientKey) + if detached, ok := s.clientDetached[clientKey]; ok { + close(detached) + delete(s.clientDetached, clientKey) + } + return routerID, wasAttached +} + +// detachedSignal returns the channel closed when clientKey leaves this subscription. A nil +// channel never fires, which is the right answer for a client that is not attached. +func (s *directActiveSubscription) detachedSignal(clientKey string) <-chan struct{} { + return s.clientDetached[clientKey] +} + +// stopUpstreamListener ends listenForUpstreamMessages. Safe to call more than once. +func (s *directActiveSubscription) stopUpstreamListener() { + s.closeSubOnce.Do(func() { + if s.closeSubChan != nil { + close(s.closeSubChan) + } + }) +} + // pendingSubscriptionsBroadcastManager handles synchronization for concurrent subscription requests type pendingSubscriptionsBroadcastManager struct { broadcastChannelList []chan bool @@ -312,6 +364,17 @@ func (dwsm *DirectWSSubscriptionManager) cleanupStaleSubscriptions(ctx context.C } // performCleanup removes subscriptions with cancelled contexts and cleans up orphaned clients +// dropSubscriptionLocked releases every client on the subscription, forgets its ids and +// retires it, telling the node. Used by the periodic sweep. Caller holds dwsm.lock. +func (dwsm *DirectWSSubscriptionManager) dropSubscriptionLocked(hashedParams string, activeSub *directActiveSubscription) { + for clientKey := range activeSub.connectedClients { + activeSub.releaseClientLocked(clientKey) + dwsm.forgetSubscriptionForClientLocked(clientKey, hashedParams, false) + } + dwsm.idMapper.RemoveAllForUpstream(activeSub.upstreamID) + dwsm.teardownSubscriptionLocked(hashedParams, activeSub, true) +} + func (dwsm *DirectWSSubscriptionManager) performCleanup() { dwsm.lock.Lock() defer dwsm.lock.Unlock() @@ -328,9 +391,7 @@ func (dwsm *DirectWSSubscriptionManager) performCleanup() { utils.LavaFormatDebug("DirectWS: cleaning up subscription with cancelled context", utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), ) - activeSub.upstreamSubscription.Unsubscribe() - dwsm.idMapper.RemoveAllForUpstream(activeSub.upstreamID) - delete(dwsm.activeSubscriptions, hashedParams) + dwsm.dropSubscriptionLocked(hashedParams, activeSub) removedSubs++ continue default: @@ -346,14 +407,17 @@ func (dwsm *DirectWSSubscriptionManager) performCleanup() { utils.LavaFormatDebug("DirectWS: cleaning up subscription with no clients", utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), ) - activeSub.upstreamSubscription.Unsubscribe() - activeSub.cancel() - dwsm.idMapper.RemoveAllForUpstream(activeSub.upstreamID) - delete(dwsm.activeSubscriptions, hashedParams) + dwsm.dropSubscriptionLocked(hashedParams, activeSub) removedSubs++ } } + // Rate limiters for clients that never got a subscription registered (a failed + // subscribe, or a client that only unsubscribed) have no disconnect hook of their + // own; an entry idle for longer than its refill time is indistinguishable from a + // fresh one, so it can go. + dwsm.rateLimiter.SweepIdle(time.Now()) + // Clean up orphaned client entries (clients with no subscriptions) for clientKey, subs := range dwsm.connectedClients { // Check if subscriptions still exist in activeSubscriptions @@ -789,8 +853,13 @@ func (dwsm *DirectWSSubscriptionManager) StartSubscription( Data: firstReplyData, } - // Create subscription context - subCtx, cancel := context.WithCancel(ctx) + // The subscription is shared, so its lifetime belongs to the manager, not to the + // client that happened to create it: WithoutCancel keeps the request values (GUID) + // for log correlation but drops the creator's cancellation. Bound to the creator's + // context, the first subscriber closing its socket ended the upstream listener + // underneath every client that had joined, and the same rule already holds for + // the gRPC manager (MAG-3722). Teardown goes through teardownSubscriptionLocked. + subCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) // Increment subscription count on the connection (for pool auto-scaling) conn.IncrementSubscriptions() @@ -814,8 +883,7 @@ func (dwsm *DirectWSSubscriptionManager) StartSubscription( closeSubChan: make(chan struct{}), messagesChan: msgChan, // Use the channel from upstream subscription } - activeSub.connectedClients[clientKey] = safeChannelSender - activeSub.clientRouterIDs[clientKey] = routerID + activeSub.attachClientLocked(clientKey, routerID, safeChannelSender) dwsm.lock.Lock() dwsm.activeSubscriptions[hashedParams] = activeSub @@ -977,18 +1045,8 @@ func (dwsm *DirectWSSubscriptionManager) Unsubscribe( _, lastClient = dwsm.idMapper.RemoveMapping(routerSubID) - if sender, ok := activeSub.connectedClients[clientKey]; ok { - sender.Close() - delete(activeSub.connectedClients, clientKey) - } - delete(activeSub.clientRouterIDs, clientKey) - - if clientSubs, ok := dwsm.connectedClients[clientKey]; ok { - delete(clientSubs, hashedParams) - if len(clientSubs) == 0 { - delete(dwsm.connectedClients, clientKey) - } - } + activeSub.releaseClientLocked(clientKey) + dwsm.forgetSubscriptionForClientLocked(clientKey, hashedParams, false) if lastClient { activeSub.upstreamSubscription.Unsubscribe() @@ -1098,47 +1156,20 @@ func (dwsm *DirectWSSubscriptionManager) UnsubscribeAll( continue } - // Get this client's router ID before removing - clientRouterID := activeSub.clientRouterIDs[clientKey] - - // Close the client's channel - if sender, ok := activeSub.connectedClients[clientKey]; ok { - sender.Close() - delete(activeSub.connectedClients, clientKey) - } - - // Remove client's router ID from subscription - delete(activeSub.clientRouterIDs, clientKey) - - // Remove this client's router ID mapping + clientRouterID, _ := activeSub.releaseClientLocked(clientKey) if clientRouterID != "" { dwsm.idMapper.RemoveMapping(clientRouterID) } - // If no more clients, close upstream subscription + // Last client out tells the node too. This path used to stop only the local + // forwarding and leave the node publishing to a pooled connection (MAG-3722). if len(activeSub.connectedClients) == 0 { - activeSub.upstreamSubscription.Unsubscribe() - activeSub.cancel() - delete(dwsm.activeSubscriptions, hashedParams) - dwsm.totalSubscriptions.Add(-1) // Decrement global counter - // Notify pool to potentially scale down - if activeSub.upstreamConnection != nil { - activeSub.upstreamPool.NotifySubscriptionRemoved(activeSub.upstreamConnection) - } + dwsm.teardownSubscriptionLocked(hashedParams, activeSub, true) } } delete(dwsm.connectedClients, clientKey) - - // Clean up sticky session for this client - dwsm.stickyStore.Delete(clientKey) - - // Clean up rate limiter for this client - dwsm.rateLimiter.CleanupClient(clientKey) - - utils.LavaFormatTrace("DirectWS: cleared sticky session and rate limiter on unsubscribe all", - utils.LogAttr("clientKey", clientKey), - ) + dwsm.forgetClientLocked(clientKey) return nil } @@ -1282,8 +1313,13 @@ func (dwsm *DirectWSSubscriptionManager) checkForActiveSubscriptionAndConnect( dwsm.idMapper.RegisterMapping(clientRouterID, activeSub.upstreamID) // Add client to existing subscription with their unique router ID - activeSub.connectedClients[clientKey] = safeChannelSender - activeSub.clientRouterIDs[clientKey] = clientRouterID + activeSub.attachClientLocked(clientKey, clientRouterID, safeChannelSender) + // A joining client leaves the same way a creating one does, usually by closing its + // socket, so it needs the same watcher. Without one nothing ever released a joined + // client: its reply channel stayed open and parked the ingress forwarder goroutine, + // it kept counting as connected, and the upstream subscription outlived every real + // subscriber (MAG-3722). + go dwsm.handleClientDisconnect(ctx, clientKey, hashedParams) if dwsm.connectedClients[clientKey] == nil { dwsm.connectedClients[clientKey] = make(map[string]*common.SafeChannelSender[*pairingtypes.RelayReply]) } @@ -1618,14 +1654,28 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( go dwsm.listenForUpstreamMessages(activeSub.ctx, hashedParams, activeSub, newUpstreamSub) } -// handleClientDisconnect handles client WebSocket disconnection. -// It removes the client's router ID mapping and cleans up the subscription if this was the last client. +// handleClientDisconnect releases one client from one subscription when the client's +// connection ends, or returns early once the client has already left by another path. +// It is registered for every attached client, creators and joiners alike. func (dwsm *DirectWSSubscriptionManager) handleClientDisconnect( ctx context.Context, clientKey string, hashedParams string, ) { - <-ctx.Done() + dwsm.lock.RLock() + var detached <-chan struct{} + if activeSub, found := dwsm.activeSubscriptions[hashedParams]; found { + detached = activeSub.detachedSignal(clientKey) + } + dwsm.lock.RUnlock() + + select { + case <-ctx.Done(): + case <-detached: + // An explicit unsubscribe or a teardown already released the client. Nothing is + // left to do, and no reason to stay parked until the socket finally closes. + return + } utils.LavaFormatTrace("DirectWS: client disconnected", utils.LogAttr("clientKey", clientKey), @@ -1635,63 +1685,82 @@ func (dwsm *DirectWSSubscriptionManager) handleClientDisconnect( dwsm.lock.Lock() defer dwsm.lock.Unlock() - // Remove client from subscription activeSub, found := dwsm.activeSubscriptions[hashedParams] if !found { return } - // Get the client's router ID before removing - clientRouterID := activeSub.clientRouterIDs[clientKey] - - // Remove client from connected clients and router IDs - delete(activeSub.connectedClients, clientKey) - delete(activeSub.clientRouterIDs, clientKey) - - // Remove the client's router ID mapping from the ID mapper + clientRouterID, wasAttached := activeSub.releaseClientLocked(clientKey) + if !wasAttached { + return + } if clientRouterID != "" { dwsm.idMapper.RemoveMapping(clientRouterID) } + dwsm.forgetSubscriptionForClientLocked(clientKey, hashedParams, true) - if clientSubs, ok := dwsm.connectedClients[clientKey]; ok { - delete(clientSubs, hashedParams) - if len(clientSubs) == 0 { - delete(dwsm.connectedClients, clientKey) - // Clean up sticky session and rate limiter when client has no more subscriptions - dwsm.stickyStore.Delete(clientKey) - dwsm.rateLimiter.CleanupClient(clientKey) - utils.LavaFormatTrace("DirectWS: cleared sticky session and rate limiter on client disconnect", - utils.LogAttr("clientKey", clientKey), - ) - } - } - - // If no more clients, close the upstream subscription if len(activeSub.connectedClients) == 0 { utils.LavaFormatTrace("DirectWS: no more clients, closing upstream subscription", utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), ) - // Capture connection reference before async cleanup - upstreamConn := activeSub.upstreamConnection - upstreamPool := activeSub.upstreamPool - go func() { - // Tell the NODE first, while the connection is still pooled and live: - // NotifySubscriptionRemoved below may scale it down, and Unsubscribe - // sends nothing on the wire. Ordering is the whole fix. - dwsm.teardownUpstreamSubscription(activeSub) - activeSub.upstreamSubscription.Unsubscribe() - activeSub.cancel() - close(activeSub.closeSubChan) - // Moved inside the goroutine so it runs AFTER the teardown above. - if upstreamConn != nil { - upstreamPool.NotifySubscriptionRemoved(upstreamConn) - } - }() - delete(dwsm.activeSubscriptions, hashedParams) - dwsm.totalSubscriptions.Add(-1) // Decrement global counter + dwsm.teardownSubscriptionLocked(hashedParams, activeSub, true) } } +// forgetSubscriptionForClientLocked drops hashedParams from the client's subscription set. +// When that was the client's last subscription and the client itself is gone, the +// per-client state (sticky endpoint, rate limiters, id counter) goes with it. An explicit +// unsubscribe keeps the rate limiters, otherwise subscribe/unsubscribe cycles would reset +// the subscribe budget. Caller holds dwsm.lock. +func (dwsm *DirectWSSubscriptionManager) forgetSubscriptionForClientLocked(clientKey, hashedParams string, clientGone bool) { + clientSubs, ok := dwsm.connectedClients[clientKey] + if !ok { + return + } + delete(clientSubs, hashedParams) + if len(clientSubs) > 0 { + return + } + delete(dwsm.connectedClients, clientKey) + if !clientGone { + return + } + dwsm.forgetClientLocked(clientKey) +} + +// forgetClientLocked drops every per-client record once the client's connection is gone. +// Caller holds dwsm.lock. +func (dwsm *DirectWSSubscriptionManager) forgetClientLocked(clientKey string) { + dwsm.stickyStore.Delete(clientKey) + dwsm.rateLimiter.CleanupClient(clientKey) + dwsm.idMapper.RemoveClient(clientKey) + utils.LavaFormatTrace("DirectWS: cleared sticky session, rate limiter and id counter for a gone client", + utils.LogAttr("clientKey", clientKey), + ) +} + +// teardownSubscriptionLocked retires a subscription that has no clients left. The node is +// told first when tellNode is set, while the connection is still pooled and live, then the +// local subscription ends and the pool may scale the connection down. The network call runs +// off the lock. Caller holds dwsm.lock. +func (dwsm *DirectWSSubscriptionManager) teardownSubscriptionLocked(hashedParams string, activeSub *directActiveSubscription, tellNode bool) { + upstreamConn := activeSub.upstreamConnection + upstreamPool := activeSub.upstreamPool + go func() { + if tellNode { + dwsm.teardownUpstreamSubscription(activeSub) + } + activeSub.upstreamSubscription.Unsubscribe() + activeSub.cancel() + activeSub.stopUpstreamListener() + if upstreamConn != nil { + upstreamPool.NotifySubscriptionRemoved(upstreamConn) + } + }() + delete(dwsm.activeSubscriptions, hashedParams) + dwsm.totalSubscriptions.Add(-1) +} + // upstreamUnsubscribeTimeout bounds the teardown call below. Some gateways never // answer an unsubscribe; without a bound the disconnect path would block on one. const upstreamUnsubscribeTimeout = 10 * time.Second @@ -1753,15 +1822,10 @@ func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string return } - // Close all client channels - for clientKey, sender := range activeSub.connectedClients { - sender.Close() - if clientSubs, ok := dwsm.connectedClients[clientKey]; ok { - delete(clientSubs, hashedParams) - if len(clientSubs) == 0 { - delete(dwsm.connectedClients, clientKey) - } - } + // Release every client: closes their reply channels and wakes their watchers. + for clientKey := range activeSub.connectedClients { + activeSub.releaseClientLocked(clientKey) + dwsm.forgetSubscriptionForClientLocked(clientKey, hashedParams, false) } // Remove ID mappings diff --git a/protocol/rpcsmartrouter/direct_ws_subscription_release_test.go b/protocol/rpcsmartrouter/direct_ws_subscription_release_test.go new file mode 100644 index 000000000..44e205884 --- /dev/null +++ b/protocol/rpcsmartrouter/direct_ws_subscription_release_test.go @@ -0,0 +1,211 @@ +package rpcsmartrouter + +import ( + "context" + "encoding/json" + "runtime" + "strings" + "testing" + "time" + + "github.com/magma-Devs/smart-router/protocol/common" + pairingtypes "github.com/magma-Devs/smart-router/types/relay" + "github.com/stretchr/testify/require" +) + +// releaseHarness drives one DirectWSSubscriptionManager against the mock node with +// several clients, each on its own connection context, the way the ingress does. +type releaseHarness struct { + t *testing.T + srv *mockSubscriptionServer + manager *DirectWSSubscriptionManager +} + +func newReleaseHarness(t *testing.T) *releaseHarness { + t.Helper() + srv := newMockSubscriptionServer() + srv.messageInterval = 20 * time.Millisecond + t.Cleanup(srv.Close) + + manager := NewDirectWSSubscriptionManager( + getTestMetricsManager(), + "jsonrpc", "ETH", "jsonrpc", + []*common.NodeUrl{{Url: srv.URL()}}, + nil, nil, nil, + ) + return &releaseHarness{t: t, srv: srv, manager: manager} +} + +// subscribe opens a subscription for a client named uid on its own connection context. +func (h *releaseHarness) subscribe(uid string) (ctx context.Context, disconnect context.CancelFunc, replies <-chan *pairingtypes.RelayReply, clientKey string) { + h.t.Helper() + ctx, disconnect = context.WithCancel(context.Background()) + body, err := json.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_subscribe", + "params": []interface{}{"newHeads"}, + "id": 1, + }) + require.NoError(h.t, err) + msg := &mockWSProtocolMessageWithRawData{ + mockWSProtocolMessage: mockWSProtocolMessage{method: "eth_subscribe", params: []interface{}{"newHeads"}}, + rawData: body, + } + reply, replies, err := h.manager.StartSubscription(ctx, msg, "dapp", "10.0.0.1", uid, nil) + require.NoError(h.t, err) + require.NotNil(h.t, reply) + require.NotNil(h.t, replies, "every subscriber, joiner or creator, gets a reply channel") + return ctx, disconnect, replies, h.manager.CreateWebSocketConnectionUniqueKey("dapp", "10.0.0.1", uid) +} + +func (h *releaseHarness) upstreamCalls(method string) int { + n := 0 + for _, m := range h.srv.ObservedMethods() { + if m == method { + n++ + } + } + return n +} + +// attached reports whether clientKey is still counted on the (single) active subscription. +func (h *releaseHarness) attached(clientKey string) bool { + h.manager.lock.RLock() + defer h.manager.lock.RUnlock() + for _, sub := range h.manager.activeSubscriptions { + if _, ok := sub.connectedClients[clientKey]; ok { + return true + } + } + return false +} + +func (h *releaseHarness) activeSubscriptionCount() int { + h.manager.lock.RLock() + defer h.manager.lock.RUnlock() + return len(h.manager.activeSubscriptions) +} + +// channelClosed drains pushes until the channel closes, or reports false on timeout. +func channelClosed(replies <-chan *pairingtypes.RelayReply, within time.Duration) bool { + deadline := time.After(within) + for { + select { + case _, ok := <-replies: + if !ok { + return true + } + case <-deadline: + return false + } + } +} + +// goroutinesIn counts live goroutines whose stack mentions fn. +func goroutinesIn(fn string) int { + buf := make([]byte, 1<<20) + n := runtime.Stack(buf, true) + return strings.Count(string(buf[:n]), fn) +} + +// TestJoinedClientDisconnect_IsReleased is the regression for MAG-3722's largest ws leak. +// The second subscriber to a query joins the first one's upstream subscription, and +// before this fix nothing watched a joiner's connection: when it closed its socket the +// reply channel stayed open (parking the ingress forwarder goroutine for good), the +// joiner kept counting as connected, and the upstream subscription outlived every real +// subscriber. +func TestJoinedClientDisconnect_IsReleased(t *testing.T) { + h := newReleaseHarness(t) + + _, disconnectA, repliesA, keyA := h.subscribe("ws-a") + _, disconnectB, repliesB, keyB := h.subscribe("ws-b") + defer disconnectA() + defer disconnectB() + + require.Equal(t, 1, h.upstreamCalls("eth_subscribe"), "the second client must join, not resubscribe") + require.True(t, h.attached(keyA)) + require.True(t, h.attached(keyB)) + require.Equal(t, 2, h.manager.idMapper.ClientCount()) + + // The joiner closes its socket without unsubscribing. + disconnectB() + + require.Eventually(t, func() bool { return !h.attached(keyB) }, 5*time.Second, 20*time.Millisecond, + "a joined client must be released when its connection ends") + require.True(t, channelClosed(repliesB, 5*time.Second), + "the joiner's reply channel must close so the ingress forwarder goroutine can end") + require.Eventually(t, func() bool { return h.manager.idMapper.ClientCount() == 1 }, 5*time.Second, 20*time.Millisecond, + "the joiner's id counter must go with it") + require.True(t, h.attached(keyA), "the creator is unaffected") + require.Equal(t, 0, h.upstreamCalls("eth_unsubscribe"), "the subscription still has a subscriber") + + // The creator leaves too: last one out tells the node. + disconnectA() + require.True(t, channelClosed(repliesA, 5*time.Second), "the creator's reply channel must close on disconnect") + require.Eventually(t, func() bool { return h.upstreamCalls("eth_unsubscribe") == 1 }, 15*time.Second, 50*time.Millisecond, + "the last client leaving must tell the node to stop pushing") + require.Eventually(t, func() bool { return h.activeSubscriptionCount() == 0 }, 5*time.Second, 20*time.Millisecond) + require.Eventually(t, func() bool { return h.manager.idMapper.ClientCount() == 0 }, 5*time.Second, 20*time.Millisecond) + require.Equal(t, 0, h.manager.rateLimiter.ClientCount(), "no rate limiter survives its client") +} + +// TestCreatorDisconnect_ClosesItsReplyChannel covers the other half of the same leak: the +// creating client had a disconnect watcher, but it removed the client without closing +// the reply channel, so the ingress forwarder goroutine parked forever anyway. +func TestCreatorDisconnect_ClosesItsReplyChannel(t *testing.T) { + h := newReleaseHarness(t) + + _, disconnectA, repliesA, keyA := h.subscribe("ws-a") + _, disconnectB, repliesB, keyB := h.subscribe("ws-b") + defer disconnectB() + + disconnectA() + + require.True(t, channelClosed(repliesA, 5*time.Second), "the creator's reply channel must close on disconnect") + require.Eventually(t, func() bool { return !h.attached(keyA) }, 5*time.Second, 20*time.Millisecond) + require.True(t, h.attached(keyB), "the joiner keeps the subscription alive") + require.Equal(t, 1, h.activeSubscriptionCount()) + require.Equal(t, 0, h.upstreamCalls("eth_unsubscribe")) + + // The joiner still receives pushes after the creator left. + select { + case reply, ok := <-repliesB: + require.True(t, ok, "the joiner's channel must stay open") + require.NotNil(t, reply) + case <-time.After(5 * time.Second): + t.Fatal("the joiner must keep receiving pushes after the creator disconnects") + } +} + +// TestUnsubscribeAll_ReleasesWatchersAndTellsNode covers the path the ingress now takes +// when a connection ends. Two things were missing: an explicit release left the +// client's disconnect watcher parked until the socket closed, and the last client +// leaving through UnsubscribeAll never told the node. +func TestUnsubscribeAll_ReleasesWatchersAndTellsNode(t *testing.T) { + h := newReleaseHarness(t) + + ctxA, disconnectA, repliesA, keyA := h.subscribe("ws-a") + ctxB, disconnectB, repliesB, keyB := h.subscribe("ws-b") + defer disconnectA() + defer disconnectB() + + require.Eventually(t, func() bool { return goroutinesIn("handleClientDisconnect") == 2 }, 5*time.Second, 20*time.Millisecond, + "one disconnect watcher per attached client") + + require.NoError(t, h.manager.UnsubscribeAll(ctxB, "dapp", "10.0.0.1", "ws-b", nil)) + require.False(t, h.attached(keyB)) + require.True(t, channelClosed(repliesB, 5*time.Second)) + require.Eventually(t, func() bool { return goroutinesIn("handleClientDisconnect") == 1 }, 5*time.Second, 20*time.Millisecond, + "an explicit release must wake the client's watcher instead of leaving it parked until the socket closes") + require.True(t, h.attached(keyA)) + require.Equal(t, 0, h.upstreamCalls("eth_unsubscribe")) + + require.NoError(t, h.manager.UnsubscribeAll(ctxA, "dapp", "10.0.0.1", "ws-a", nil)) + require.True(t, channelClosed(repliesA, 5*time.Second)) + require.Eventually(t, func() bool { return h.upstreamCalls("eth_unsubscribe") == 1 }, 15*time.Second, 50*time.Millisecond, + "the last client leaving through UnsubscribeAll must tell the node to stop pushing") + require.Eventually(t, func() bool { return goroutinesIn("handleClientDisconnect") == 0 }, 5*time.Second, 20*time.Millisecond) + require.Equal(t, 0, h.activeSubscriptionCount()) + require.Equal(t, 0, h.manager.idMapper.ClientCount()) + require.Equal(t, 0, h.manager.rateLimiter.ClientCount()) +} diff --git a/protocol/rpcsmartrouter/subscription_id_mapper.go b/protocol/rpcsmartrouter/subscription_id_mapper.go index f3da0faa1..71b3225ad 100644 --- a/protocol/rpcsmartrouter/subscription_id_mapper.go +++ b/protocol/rpcsmartrouter/subscription_id_mapper.go @@ -85,6 +85,23 @@ func (m *SubscriptionIDMapper) GenerateRouterID(clientKey string) string { return fmt.Sprintf("rs_%s_%05d", clientHash, count) } +// RemoveClient forgets a client's ID counter. Call it once the client holds no +// subscriptions: the key carries the per-connection GUID, so nothing reuses it, +// and without this call the map kept one entry per connection that ever +// subscribed (MAG-3722). +func (m *SubscriptionIDMapper) RemoveClient(clientKey string) { + m.lock.Lock() + defer m.lock.Unlock() + delete(m.clientCounters, clientKey) +} + +// ClientCount reports how many clients currently hold an ID counter. +func (m *SubscriptionIDMapper) ClientCount() int { + m.lock.RLock() + defer m.lock.RUnlock() + return len(m.clientCounters) +} + // GenerateNumericRouterID creates a router subscription ID for chains that number their // subscriptions rather than naming them (Solana). The id is returned in the same canonical // decimal-string form as every other id here — every map in this type stays string-keyed — diff --git a/protocol/rpcsmartrouter/subscription_id_mapper_leak_test.go b/protocol/rpcsmartrouter/subscription_id_mapper_leak_test.go new file mode 100644 index 000000000..bc46be139 --- /dev/null +++ b/protocol/rpcsmartrouter/subscription_id_mapper_leak_test.go @@ -0,0 +1,29 @@ +package rpcsmartrouter + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestSubscriptionIDMapper_RemoveClientDropsCounter is the regression for MAG-3722: the +// per-client id counter was never removed, so the map kept one entry for every websocket +// connection that ever subscribed. +func TestSubscriptionIDMapper_RemoveClientDropsCounter(t *testing.T) { + mapper := NewSubscriptionIDMapper() + first := mapper.GenerateRouterID("client-a") + mapper.GenerateRouterID("client-b") + require.Equal(t, 2, mapper.ClientCount()) + + mapper.RemoveClient("client-a") + require.Equal(t, 1, mapper.ClientCount()) + + mapper.RemoveClient("client-a") // idempotent + require.Equal(t, 1, mapper.ClientCount()) + + // A client that comes back after release simply starts a fresh counter; every + // mapping it held was removed before its counter was, so the ids cannot collide + // with a live subscription. + require.Equal(t, first, mapper.GenerateRouterID("client-a")) + require.Equal(t, 2, mapper.ClientCount()) +} diff --git a/protocol/rpcsmartrouter/websocket_config.go b/protocol/rpcsmartrouter/websocket_config.go index 18a959746..283174b2b 100644 --- a/protocol/rpcsmartrouter/websocket_config.go +++ b/protocol/rpcsmartrouter/websocket_config.go @@ -83,14 +83,34 @@ type ClientRateLimiter struct { mu sync.Mutex // subscribeLimiters tracks subscription creation rate per client - subscribeLimiters map[string]*rate.Limiter + subscribeLimiters map[string]*clientLimiter // unsubscribeLimiters tracks unsubscription rate per client - unsubscribeLimiters map[string]*rate.Limiter + unsubscribeLimiters map[string]*clientLimiter subscribeRate rate.Limit // subscriptions per second unsubscribeRate rate.Limit // unsubscribes per second subscribeBurst int // max burst for subscriptions unsubscribeBurst int // max burst for unsubscribes + + // idleTTL is how long an untouched limiter is kept. It is the time a limiter needs + // to refill a full burst, so an entry older than that decides exactly as a fresh one + // would and can be dropped without changing any outcome (MAG-3722). + idleTTL time.Duration +} + +// clientLimiter pairs a limiter with the last time it was consulted, so SweepIdle can +// tell an abandoned entry from a live one. +type clientLimiter struct { + limiter *rate.Limiter + lastSeen time.Time +} + +// refillDuration is how long a limiter with the given rate takes to refill burst tokens. +func refillDuration(limit rate.Limit, burst int) time.Duration { + if limit <= 0 || burst <= 0 { + return time.Minute + } + return time.Duration(float64(burst) / float64(limit) * float64(time.Second)) } // NewClientRateLimiter creates a new rate limiter based on config @@ -99,38 +119,47 @@ func NewClientRateLimiter(config *WebsocketConfig) *ClientRateLimiter { subscribeRate := rate.Limit(float64(config.SubscriptionsPerMinutePerClient) / 60.0) unsubscribeRate := rate.Limit(float64(config.UnsubscribesPerMinutePerClient) / 60.0) + idleTTL := max( + refillDuration(subscribeRate, config.SubscriptionsPerMinutePerClient), + refillDuration(unsubscribeRate, config.UnsubscribesPerMinutePerClient), + time.Minute, + ) + return &ClientRateLimiter{ - subscribeLimiters: make(map[string]*rate.Limiter), - unsubscribeLimiters: make(map[string]*rate.Limiter), + subscribeLimiters: make(map[string]*clientLimiter), + unsubscribeLimiters: make(map[string]*clientLimiter), subscribeRate: subscribeRate, unsubscribeRate: unsubscribeRate, subscribeBurst: config.SubscriptionsPerMinutePerClient, // Allow burst up to full minute's worth unsubscribeBurst: config.UnsubscribesPerMinutePerClient, + idleTTL: idleTTL, } } // AllowSubscribe checks if the client is allowed to create a subscription func (crl *ClientRateLimiter) AllowSubscribe(clientKey string) bool { crl.mu.Lock() - limiter, exists := crl.subscribeLimiters[clientKey] + entry, exists := crl.subscribeLimiters[clientKey] if !exists { - limiter = rate.NewLimiter(crl.subscribeRate, crl.subscribeBurst) - crl.subscribeLimiters[clientKey] = limiter + entry = &clientLimiter{limiter: rate.NewLimiter(crl.subscribeRate, crl.subscribeBurst)} + crl.subscribeLimiters[clientKey] = entry } + entry.lastSeen = time.Now() crl.mu.Unlock() - return limiter.Allow() + return entry.limiter.Allow() } // AllowUnsubscribe checks if the client is allowed to unsubscribe func (crl *ClientRateLimiter) AllowUnsubscribe(clientKey string) bool { crl.mu.Lock() - limiter, exists := crl.unsubscribeLimiters[clientKey] + entry, exists := crl.unsubscribeLimiters[clientKey] if !exists { - limiter = rate.NewLimiter(crl.unsubscribeRate, crl.unsubscribeBurst) - crl.unsubscribeLimiters[clientKey] = limiter + entry = &clientLimiter{limiter: rate.NewLimiter(crl.unsubscribeRate, crl.unsubscribeBurst)} + crl.unsubscribeLimiters[clientKey] = entry } + entry.lastSeen = time.Now() crl.mu.Unlock() - return limiter.Allow() + return entry.limiter.Allow() } // CleanupClient removes rate limiters for a disconnected client @@ -141,6 +170,30 @@ func (crl *ClientRateLimiter) CleanupClient(clientKey string) { delete(crl.unsubscribeLimiters, clientKey) } +// SweepIdle drops every limiter not consulted since now minus idleTTL and reports how +// many went. It is the backstop for clients that never reached a disconnect hook. +func (crl *ClientRateLimiter) SweepIdle(now time.Time) int { + crl.mu.Lock() + defer crl.mu.Unlock() + removed := 0 + for _, limiters := range []map[string]*clientLimiter{crl.subscribeLimiters, crl.unsubscribeLimiters} { + for clientKey, entry := range limiters { + if now.Sub(entry.lastSeen) >= crl.idleTTL { + delete(limiters, clientKey) + removed++ + } + } + } + return removed +} + +// ClientCount reports how many clients currently hold a limiter of either kind. +func (crl *ClientRateLimiter) ClientCount() int { + crl.mu.Lock() + defer crl.mu.Unlock() + return len(crl.subscribeLimiters) + len(crl.unsubscribeLimiters) +} + // EnforcementMode represents how limit violations are handled type EnforcementMode string diff --git a/protocol/rpcsmartrouter/websocket_config_sweep_test.go b/protocol/rpcsmartrouter/websocket_config_sweep_test.go new file mode 100644 index 000000000..0cbbf1f6a --- /dev/null +++ b/protocol/rpcsmartrouter/websocket_config_sweep_test.go @@ -0,0 +1,36 @@ +package rpcsmartrouter + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestClientRateLimiter_SweepIdle is the regression for MAG-3722: a client whose subscribe +// failed, or that only ever unsubscribed, never reached CleanupClient, so its limiters +// stayed for the life of the process. An entry idle for its refill time decides exactly +// as a fresh one would, so dropping it changes no outcome. +func TestClientRateLimiter_SweepIdle(t *testing.T) { + config := DefaultWebsocketConfig() + config.SubscriptionsPerMinutePerClient = 10 + config.UnsubscribesPerMinutePerClient = 10 + limiter := NewClientRateLimiter(config) + require.Equal(t, time.Minute, limiter.idleTTL, "10 per minute with a burst of 10 refills in a minute") + + require.True(t, limiter.AllowSubscribe("client-a")) + require.True(t, limiter.AllowUnsubscribe("client-a")) + require.True(t, limiter.AllowSubscribe("client-b")) + require.Equal(t, 3, limiter.ClientCount()) + + now := time.Now() + require.Equal(t, 0, limiter.SweepIdle(now), "entries just consulted are live") + require.Equal(t, 3, limiter.ClientCount()) + + require.Equal(t, 3, limiter.SweepIdle(now.Add(limiter.idleTTL))) + require.Equal(t, 0, limiter.ClientCount()) + + // A swept client is admitted like a new one. + require.True(t, limiter.AllowSubscribe("client-a")) + require.Equal(t, 1, limiter.ClientCount()) +} From 9597848f0802da219447a0172a361b642b0f138d Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Wed, 16 Sep 2026 22:23:42 +0300 Subject: [PATCH 3/8] fix(rpcclient): drop a subscription's dispatch entry on unsubscribe The handler's subscription table only shrank when the connection closed. Upstream websocket connections are pooled and live for the life of the process, so every subscription ever torn down on one left its entry behind. Unsubscribe now asks the dispatch loop, the table's only owner, to forget the entry, on a goroutine of its own because dispatch may be busy delivering to another subscription whose consumer waits on a lock the caller holds. Jira ticket: MAG-3722 --- .../chainlib/chainproxy/rpcclient/client.go | 38 ++++- .../chainlib/chainproxy/rpcclient/handler.go | 13 ++ .../chainproxy/rpcclient/subscription.go | 1 + .../rpcclient/subscription_forget_test.go | 141 ++++++++++++++++++ 4 files changed, 185 insertions(+), 8 deletions(-) create mode 100644 protocol/chainlib/chainproxy/rpcclient/subscription_forget_test.go diff --git a/protocol/chainlib/chainproxy/rpcclient/client.go b/protocol/chainlib/chainproxy/rpcclient/client.go index fdbe1b368..8b36dc6de 100644 --- a/protocol/chainlib/chainproxy/rpcclient/client.go +++ b/protocol/chainlib/chainproxy/rpcclient/client.go @@ -104,14 +104,15 @@ type Client struct { // for dispatch close chan struct{} - closing chan struct{} // closed when client is quitting - didClose chan struct{} // closed when client quits - reconnected chan ServerCodec // where write/reconnect sends the new connection - readOp chan readOp // read messages - readErr chan error // errors from read - reqInit chan *requestOp // register response IDs, takes write lock - reqSent chan error // signals write completion, releases write lock - reqTimeout chan *requestOp // removes response IDs when call timeout expires + closing chan struct{} // closed when client is quitting + didClose chan struct{} // closed when client quits + reconnected chan ServerCodec // where write/reconnect sends the new connection + readOp chan readOp // read messages + readErr chan error // errors from read + reqInit chan *requestOp // register response IDs, takes write lock + reqSent chan error // signals write completion, releases write lock + reqTimeout chan *requestOp // removes response IDs when call timeout expires + subDone chan *ClientSubscription // drops a subscription's dispatch entry after Unsubscribe } type reconnectFunc func(ctx context.Context) (ServerCodec, error) @@ -214,6 +215,7 @@ func initClient(conn ServerCodec, idgen func() ID, services *serviceRegistry) *C reqInit: make(chan *requestOp), reqSent: make(chan error, 1), reqTimeout: make(chan *requestOp), + subDone: make(chan *ClientSubscription), } if !isHTTP { go c.dispatch(conn) @@ -724,10 +726,30 @@ func (c *Client) dispatch(codec ServerCodec) { case op := <-c.reqTimeout: conn.handler.removeRequestOp(op) + + case sub := <-c.subDone: + conn.handler.forgetClientSub(sub) } } } +// forgetSubscription asks the dispatch loop to drop the handler's entry for sub. Until +// now that entry lived until the connection closed, one per subscription ever torn +// down on a pooled connection (MAG-3722). It runs on its own goroutine because +// dispatch may be busy delivering to another subscription whose consumer is waiting +// on a lock the Unsubscribe caller holds. +func (c *Client) forgetSubscription(sub *ClientSubscription) { + if c == nil || c.isHTTP { + return + } + go func() { + select { + case c.subDone <- sub: + case <-c.closing: + } + }() +} + // drainRead drops read messages until an error occurs. func (c *Client) drainRead() { for { diff --git a/protocol/chainlib/chainproxy/rpcclient/handler.go b/protocol/chainlib/chainproxy/rpcclient/handler.go index 036c834cc..f239d9978 100755 --- a/protocol/chainlib/chainproxy/rpcclient/handler.go +++ b/protocol/chainlib/chainproxy/rpcclient/handler.go @@ -529,6 +529,19 @@ func (h *handler) runMethod(ctx context.Context, msg *JsonrpcMessage, callb *cal return msg.response(result) } +// forgetClientSub drops sub from the dispatch table once its forwarding loop has been +// told to quit. Runs on the dispatch goroutine, which is the only owner of clientSubs. +// A push for the id that still arrives is dropped by deliverSubscriptionPush as an +// unmatched notification, exactly as before the entry was removed. +func (h *handler) forgetClientSub(sub *ClientSubscription) { + if sub == nil { + return + } + if current, ok := h.clientSubs[sub.subid]; ok && current == sub { + delete(h.clientSubs, sub.subid) + } +} + // unsubscribe is the callback function for all *_unsubscribe calls. func (h *handler) unsubscribe(ctx context.Context, id ID) (bool, error) { h.subLock.Lock() diff --git a/protocol/chainlib/chainproxy/rpcclient/subscription.go b/protocol/chainlib/chainproxy/rpcclient/subscription.go index 3885ea90c..23455e9d4 100755 --- a/protocol/chainlib/chainproxy/rpcclient/subscription.go +++ b/protocol/chainlib/chainproxy/rpcclient/subscription.go @@ -274,6 +274,7 @@ func (sub *ClientSubscription) Unsubscribe() { // Unsubscribe already requested (or subscription already closed). } close(sub.err) + sub.client.forgetSubscription(sub) }) } diff --git a/protocol/chainlib/chainproxy/rpcclient/subscription_forget_test.go b/protocol/chainlib/chainproxy/rpcclient/subscription_forget_test.go new file mode 100644 index 000000000..f78865b2f --- /dev/null +++ b/protocol/chainlib/chainproxy/rpcclient/subscription_forget_test.go @@ -0,0 +1,141 @@ +package rpcclient + +import ( + "context" + "encoding/json" + "reflect" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestForgetClientSub_DropsOnlyThatSubscription pins the identity check: a later +// subscription registered under a reused id must not be removed by a stale unsubscribe. +func TestForgetClientSub_DropsOnlyThatSubscription(t *testing.T) { + h := newTestHandler() + ch := make(chan *JsonrpcMessage) + old := newClientSubscription(nil, "eth", reflect.ValueOf(ch)) + old.subid = "0xabc" + h.clientSubs[old.subid] = old + + h.forgetClientSub(old) + require.NotContains(t, h.clientSubs, "0xabc") + + replacement := newClientSubscription(nil, "eth", reflect.ValueOf(ch)) + replacement.subid = "0xabc" + h.clientSubs[replacement.subid] = replacement + h.forgetClientSub(old) // stale + require.Same(t, replacement, h.clientSubs["0xabc"], "a stale unsubscribe must not evict the live subscription") + + h.forgetClientSub(nil) // must not panic +} + +// scriptedCodec is an in-memory ServerCodec: the test feeds it frames to read and +// records what the client writes. +type scriptedCodec struct { + incoming chan []*JsonrpcMessage + closedCh chan interface{} + once sync.Once + + mu sync.Mutex + written []*JsonrpcMessage +} + +func newScriptedCodec() *scriptedCodec { + return &scriptedCodec{incoming: make(chan []*JsonrpcMessage, 16), closedCh: make(chan interface{})} +} + +func (c *scriptedCodec) peerInfo() PeerInfo { return PeerInfo{} } +func (c *scriptedCodec) readBatch() ([]*JsonrpcMessage, bool, error) { + select { + case msgs := <-c.incoming: + return msgs, false, nil + case <-c.closedCh: + return nil, false, ErrClientQuit + } +} +func (c *scriptedCodec) close() { c.once.Do(func() { close(c.closedCh) }) } +func (c *scriptedCodec) closed() <-chan interface{} { return c.closedCh } +func (c *scriptedCodec) remoteAddr() string { return "scripted" } +func (c *scriptedCodec) writeJSON(_ context.Context, v interface{}) error { + c.mu.Lock() + defer c.mu.Unlock() + if msg, ok := v.(*JsonrpcMessage); ok { + c.written = append(c.written, msg) + } + return nil +} + +func (c *scriptedCodec) writtenCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.written) +} + +func (c *scriptedCodec) lastWritten() *JsonrpcMessage { + c.mu.Lock() + defer c.mu.Unlock() + if len(c.written) == 0 { + return nil + } + return c.written[len(c.written)-1] +} + +// TestUnsubscribe_ForgetsDispatchEntry is the regression for MAG-3722's rpcclient leak: +// the dispatch table kept an entry for every subscription ever unsubscribed until the +// connection closed, and on a pooled upstream connection that is the life of the process. +// +// The entry is observed through behaviour rather than a hook. A push that names the +// subscription and carries a request id is claimed silently while the entry exists, and +// handed to the call path once it is gone, where the handler answers it. So the answer +// appearing on the wire is the entry disappearing. +func TestUnsubscribe_ForgetsDispatchEntry(t *testing.T) { + codec := newScriptedCodec() + client, err := newClient(context.Background(), func(context.Context) (ServerCodec, error) { return codec, nil }) + require.NoError(t, err) + defer client.Close() + + // Answer the subscribe request as the node would. + go func() { + require.Eventually(t, func() bool { return codec.writtenCount() == 1 }, 5*time.Second, 5*time.Millisecond) + req := codec.lastWritten() + codec.incoming <- []*JsonrpcMessage{{Version: Vsn, ID: req.ID, Result: json.RawMessage(`"0xabc"`)}} + }() + ch := make(chan *JsonrpcMessage, 8) + sub, _, err := client.Subscribe(context.Background(), nil, "eth_subscribe", ch, []interface{}{"newHeads"}) + require.NoError(t, err) + require.NotNil(t, sub) + + // A Substrate-shaped push: the handler routes it by the params envelope, and that + // route is the one that hands an unmatched id-bearing frame back to the call path. + // (An eth_subscription frame is claimed by its method name whether or not the + // table still has the entry, so it cannot show the entry going away.) + push := func() { + codec.incoming <- []*JsonrpcMessage{{ + Version: Vsn, + ID: json.RawMessage(`99`), + Method: "chain_newHead", + Params: json.RawMessage(`{"subscription":"0xabc","result":{"number":"0x1"}}`), + }} + } + + push() + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatal("a push must reach the subscriber while it is live") + } + require.Equal(t, 1, codec.writtenCount(), "a claimed push is not answered") + + sub.Unsubscribe() + + require.Eventually(t, func() bool { + push() + return codec.writtenCount() > 1 + }, 5*time.Second, 10*time.Millisecond, + "once the entry is forgotten an id-bearing push is handed to the call path and answered") + answer := codec.lastWritten() + require.NotNil(t, answer.Error, "the call path answers an unknown method with an error") +} From f2710c7165debdbcbb23c4e251b252775dc5b2d4 Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Wed, 16 Sep 2026 22:23:42 +0300 Subject: [PATCH 4/8] fix(relaycore): never block on the return-condition signal The state machine reads returnCondition at most once and may already have left on another branch, yet every validateReturnCondition goroutine sent into the buffer-1 channel unconditionally. A third stop decision inside one 15 ms burst parked its goroutine forever, pinning the whole state machine and the request it carries. The send is now non-blocking; only the first reason can matter. Jira ticket: MAG-3722 --- protocol/relaycore/return_condition_test.go | 38 +++++++++++++++++++ .../relaycore/unified_relay_state_machine.go | 15 +++++++- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 protocol/relaycore/return_condition_test.go diff --git a/protocol/relaycore/return_condition_test.go b/protocol/relaycore/return_condition_test.go new file mode 100644 index 000000000..7754372a3 --- /dev/null +++ b/protocol/relaycore/return_condition_test.go @@ -0,0 +1,38 @@ +package relaycore + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestSignalReturnCondition_NeverBlocks is the regression for MAG-3722: the main loop reads +// returnCondition at most once, so every validateReturnCondition goroutine past the second +// used to park forever on the buffer-1 channel, pinning the state machine and the request. +func TestSignalReturnCondition_NeverBlocks(t *testing.T) { + returnCondition := make(chan error, 1) + first := errors.New("first") + + done := make(chan struct{}) + go func() { + defer close(done) + signalReturnCondition(returnCondition, first) + signalReturnCondition(returnCondition, errors.New("second")) + signalReturnCondition(returnCondition, errors.New("third")) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("signalling a return condition nobody will read must not block") + } + + require.Equal(t, first, <-returnCondition, "the first reason is the one the loop acts on") + select { + case extra := <-returnCondition: + t.Fatalf("no further reason may be queued, got %v", extra) + default: + } +} diff --git a/protocol/relaycore/unified_relay_state_machine.go b/protocol/relaycore/unified_relay_state_machine.go index 6ae11d589..d968a16e2 100644 --- a/protocol/relaycore/unified_relay_state_machine.go +++ b/protocol/relaycore/unified_relay_state_machine.go @@ -304,6 +304,19 @@ func (sm *UnifiedRelayStateMachine) getResultsSummary() ResultsSummary { return sm.resultsChecker.GetResultsSummary() } +// signalReturnCondition hands the main loop a return reason without ever blocking. +// +// The loop reads returnCondition at most once and may already have left on another +// branch, so only the first signal can matter. A plain send from the third trigger in +// one burst parked its goroutine forever, pinning the whole state machine and the +// request it carries (MAG-3722). +func signalReturnCondition(returnCondition chan<- error, err error) { + select { + case returnCondition <- err: + default: + } +} + // buildDecisionInput assembles the DecisionInput for the policy engine. func (sm *UnifiedRelayStateMachine) buildDecisionInput(numberOfNodeErrors uint64, isTickerHedge bool) DecisionInput { latestState := sm.getLatestState() @@ -410,7 +423,7 @@ func (sm *UnifiedRelayStateMachine) GetRelayTaskChannel() (chan RelayStateSendIn utils.LavaFormatTrace("[StateMachine] validating return condition", utils.LogAttr("batch", sm.usedProviders.BatchNumber()), utils.LogAttr("GUID", sm.ctx)) if batchOnStart == sm.usedProviders.BatchNumber() && sm.usedProviders.CurrentlyUsed() == 0 { utils.LavaFormatTrace("[StateMachine] return condition triggered", utils.LogAttr("batch", sm.usedProviders.BatchNumber()), utils.LogAttr("err", err), utils.LogAttr("GUID", sm.ctx)) - returnCondition <- err + signalReturnCondition(returnCondition, err) } } From 4b1a4965913fee68ebced31098b84a04545f5db7 Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Wed, 16 Sep 2026 22:23:42 +0300 Subject: [PATCH 5/8] fix(metrics): prune stale epochs from the optimizer stake map The stake map kept one entry per provider per epoch and nothing removed them. Reads only ever ask for the epoch being scored, so the current and previous epoch are all that is kept now. Jira ticket: MAG-3722 --- .../metrics/consumer_optimizer_qos_client.go | 7 +++++ ...onsumer_optimizer_qos_client_stake_test.go | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 protocol/metrics/consumer_optimizer_qos_client_stake_test.go diff --git a/protocol/metrics/consumer_optimizer_qos_client.go b/protocol/metrics/consumer_optimizer_qos_client.go index 5cc1f1e14..e8764eb05 100644 --- a/protocol/metrics/consumer_optimizer_qos_client.go +++ b/protocol/metrics/consumer_optimizer_qos_client.go @@ -522,6 +522,13 @@ func (coqc *ConsumerOptimizerQoSClient) setProviderStake(chainId, providerAddres } epochMap[epoch] = stake + // Reads only ever ask for the epoch being scored, so anything older than the previous + // one is dead weight that used to accumulate for the life of the process (MAG-3722). + for storedEpoch := range epochMap { + if storedEpoch+1 < epoch { + delete(epochMap, storedEpoch) + } + } } func (coqc *ConsumerOptimizerQoSClient) UpdatePairingListStake(stakeMap map[string]int64, chainId string, epoch uint64) { diff --git a/protocol/metrics/consumer_optimizer_qos_client_stake_test.go b/protocol/metrics/consumer_optimizer_qos_client_stake_test.go new file mode 100644 index 000000000..9cd9120dc --- /dev/null +++ b/protocol/metrics/consumer_optimizer_qos_client_stake_test.go @@ -0,0 +1,27 @@ +package metrics + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestUpdatePairingListStake_PrunesOldEpochs is the regression for MAG-3722: the stake map +// kept one entry per provider per epoch for the life of the process. Reads only ever ask +// for the epoch being scored, so the current and previous epoch are all that is kept. +func TestUpdatePairingListStake_PrunesOldEpochs(t *testing.T) { + coqc := NewConsumerOptimizerQoSClient("addr", NoopUsageSink{}) + const chain, provider = "ETH1", "provider-a" + + for epoch := uint64(1); epoch <= 10; epoch++ { + coqc.UpdatePairingListStake(map[string]int64{provider: int64(epoch) * 100}, chain, epoch) + } + + coqc.lock.RLock() + defer coqc.lock.RUnlock() + epochs := coqc.chainIdToProviderToEpochToStake[chain][provider] + require.Len(t, epochs, 2, "only the current and previous epoch may remain") + require.Equal(t, int64(1000), coqc.getProviderChainStake(chain, provider, 10)) + require.Equal(t, int64(900), coqc.getProviderChainStake(chain, provider, 9)) + require.Equal(t, int64(0), coqc.getProviderChainStake(chain, provider, 8), "an epoch two behind is gone") +} From c6211dd5c6ce6d7ac4850a69262b0cb43e1573a2 Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Thu, 17 Sep 2026 11:46:20 +0300 Subject: [PATCH 6/8] fix(ws): release per-client state on every exit path, number router ids per mapper Review follow-ups on the client release. A client that unsubscribed everything explicitly and then closed its socket still leaked: the explicit unsubscribe emptied its subscription set, and the connection-close release returned early on the empty set before handing back the sticky endpoint and rate limiters. The early return now releases them. Unsubscribe's last-client branch and Close go through the same teardown and release helpers as every other path, so a subscription retires one way only, and Unsubscribe documents why it trusts the mapper's count. Router ids are numbered by one mapper-wide counter instead of one per client. The per-client counter map was the leak on both the ws and the gRPC manager, and it needed a release call on every exit path to stay bounded; a single counter needs nothing released and never reuses an id for the life of the process, so the reuse caveat on an explicit unsubscribe_all goes away too. The id format is unchanged; only the numbering is mapper-wide. Jira ticket: MAG-3722 --- .../direct_ws_subscription_manager.go | 34 ++-- .../direct_ws_subscription_release_test.go | 158 +++++++++++++----- .../rpcsmartrouter/subscription_id_mapper.go | 61 ++----- .../subscription_id_mapper_leak_test.go | 29 ---- .../subscription_id_mapper_test.go | 7 +- protocol/rpcsmartrouter/websocket_config.go | 4 + 6 files changed, 165 insertions(+), 128 deletions(-) delete mode 100644 protocol/rpcsmartrouter/subscription_id_mapper_leak_test.go diff --git a/protocol/rpcsmartrouter/direct_ws_subscription_manager.go b/protocol/rpcsmartrouter/direct_ws_subscription_manager.go index 6e4dc5569..88a00bcee 100644 --- a/protocol/rpcsmartrouter/direct_ws_subscription_manager.go +++ b/protocol/rpcsmartrouter/direct_ws_subscription_manager.go @@ -1048,14 +1048,12 @@ func (dwsm *DirectWSSubscriptionManager) Unsubscribe( activeSub.releaseClientLocked(clientKey) dwsm.forgetSubscriptionForClientLocked(clientKey, hashedParams, false) + // lastClient is the mapper's view after this client's mapping went, read under the + // lock: a client that joined during the unlocked upstream call above registered its + // mapping first, so it keeps the subscription. The node was already told, so the + // teardown skips the wire call. if lastClient { - activeSub.upstreamSubscription.Unsubscribe() - activeSub.cancel() - delete(dwsm.activeSubscriptions, hashedParams) - dwsm.totalSubscriptions.Add(-1) - if activeSub.upstreamConnection != nil { - activeSub.upstreamPool.NotifySubscriptionRemoved(activeSub.upstreamConnection) - } + dwsm.teardownSubscriptionLocked(hashedParams, activeSub, false) } return nodeResp, nil @@ -1146,7 +1144,12 @@ func (dwsm *DirectWSSubscriptionManager) UnsubscribeAll( clientSubs, ok := dwsm.connectedClients[clientKey] if !ok { - return nil // No subscriptions for this client + // Nothing left to unsubscribe, but the client can still own per-client state: a + // client that unsubscribed everything explicitly keeps its sticky endpoint and + // rate limiters for the rest of its connection, and this is where the connection + // ending hands them back. Returning early here left them behind (MAG-3722). + dwsm.forgetClientLocked(clientKey) + return nil } // Iterate over all subscriptions for this client @@ -1733,8 +1736,7 @@ func (dwsm *DirectWSSubscriptionManager) forgetSubscriptionForClientLocked(clien func (dwsm *DirectWSSubscriptionManager) forgetClientLocked(clientKey string) { dwsm.stickyStore.Delete(clientKey) dwsm.rateLimiter.CleanupClient(clientKey) - dwsm.idMapper.RemoveClient(clientKey) - utils.LavaFormatTrace("DirectWS: cleared sticky session, rate limiter and id counter for a gone client", + utils.LavaFormatTrace("DirectWS: cleared sticky session and rate limiter for a gone client", utils.LogAttr("clientKey", clientKey), ) } @@ -1858,15 +1860,19 @@ func (dwsm *DirectWSSubscriptionManager) Close() { dwsm.lock.Lock() defer dwsm.lock.Unlock() - // Close all subscriptions + // Release every client through the same path a disconnect takes, so reply channels + // close and watchers wake, then end each subscription. The pools closing below is + // what ends the node side. for _, sub := range dwsm.activeSubscriptions { + for clientKey := range sub.connectedClients { + sub.releaseClientLocked(clientKey) + } sub.upstreamSubscription.Unsubscribe() sub.cancel() - for _, sender := range sub.connectedClients { - sender.Close() - } + sub.stopUpstreamListener() } dwsm.activeSubscriptions = make(map[string]*directActiveSubscription) + dwsm.connectedClients = make(map[string]map[string]*common.SafeChannelSender[*pairingtypes.RelayReply]) // Close all pools for _, pool := range dwsm.upstreamPools { diff --git a/protocol/rpcsmartrouter/direct_ws_subscription_release_test.go b/protocol/rpcsmartrouter/direct_ws_subscription_release_test.go index 44e205884..db4cce6d7 100644 --- a/protocol/rpcsmartrouter/direct_ws_subscription_release_test.go +++ b/protocol/rpcsmartrouter/direct_ws_subscription_release_test.go @@ -36,10 +36,20 @@ func newReleaseHarness(t *testing.T) *releaseHarness { return &releaseHarness{t: t, srv: srv, manager: manager} } +// subscription is what one client got back from subscribe. +type subscription struct { + ctx context.Context + disconnect context.CancelFunc + replies <-chan *pairingtypes.RelayReply + clientKey string + routerID string + uid string +} + // subscribe opens a subscription for a client named uid on its own connection context. -func (h *releaseHarness) subscribe(uid string) (ctx context.Context, disconnect context.CancelFunc, replies <-chan *pairingtypes.RelayReply, clientKey string) { +func (h *releaseHarness) subscribe(uid string) subscription { h.t.Helper() - ctx, disconnect = context.WithCancel(context.Background()) + ctx, disconnect := context.WithCancel(context.Background()) body, err := json.Marshal(map[string]interface{}{ "jsonrpc": "2.0", "method": "eth_subscribe", @@ -55,7 +65,49 @@ func (h *releaseHarness) subscribe(uid string) (ctx context.Context, disconnect require.NoError(h.t, err) require.NotNil(h.t, reply) require.NotNil(h.t, replies, "every subscriber, joiner or creator, gets a reply channel") - return ctx, disconnect, replies, h.manager.CreateWebSocketConnectionUniqueKey("dapp", "10.0.0.1", uid) + var resp struct { + Result string `json:"result"` + } + require.NoError(h.t, json.Unmarshal(reply.Data, &resp)) + require.NotEmpty(h.t, resp.Result, "the subscribe reply carries the router id") + return subscription{ + ctx: ctx, + disconnect: disconnect, + replies: replies, + clientKey: h.manager.CreateWebSocketConnectionUniqueKey("dapp", "10.0.0.1", uid), + routerID: resp.Result, + uid: uid, + } +} + +// unsubscribe sends the client's explicit eth_unsubscribe for its router id. +func (h *releaseHarness) unsubscribe(sub subscription) { + h.t.Helper() + body, err := json.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_unsubscribe", + "params": []interface{}{sub.routerID}, + "id": 2, + }) + require.NoError(h.t, err) + msg := &mockWSProtocolMessageWithRawData{ + mockWSProtocolMessage: mockWSProtocolMessage{method: "eth_unsubscribe", params: []interface{}{sub.routerID}}, + rawData: body, + } + _, err = h.manager.Unsubscribe(sub.ctx, msg, "dapp", "10.0.0.1", sub.uid, nil) + require.NoError(h.t, err) +} + +// connectionClosed is what the ingress does when a client's socket ends. +func (h *releaseHarness) connectionClosed(sub subscription) { + h.t.Helper() + require.NoError(h.t, h.manager.UnsubscribeAll(sub.ctx, "dapp", "10.0.0.1", sub.uid, nil)) +} + +func (h *releaseHarness) connectedClientCount() int { + h.manager.lock.RLock() + defer h.manager.lock.RUnlock() + return len(h.manager.connectedClients) } func (h *releaseHarness) upstreamCalls(method string) int { @@ -117,35 +169,34 @@ func goroutinesIn(fn string) int { func TestJoinedClientDisconnect_IsReleased(t *testing.T) { h := newReleaseHarness(t) - _, disconnectA, repliesA, keyA := h.subscribe("ws-a") - _, disconnectB, repliesB, keyB := h.subscribe("ws-b") - defer disconnectA() - defer disconnectB() + a := h.subscribe("ws-a") + b := h.subscribe("ws-b") + defer a.disconnect() + defer b.disconnect() require.Equal(t, 1, h.upstreamCalls("eth_subscribe"), "the second client must join, not resubscribe") - require.True(t, h.attached(keyA)) - require.True(t, h.attached(keyB)) - require.Equal(t, 2, h.manager.idMapper.ClientCount()) + require.True(t, h.attached(a.clientKey)) + require.True(t, h.attached(b.clientKey)) + require.NotEqual(t, a.routerID, b.routerID, "each client gets its own id on the shared subscription") // The joiner closes its socket without unsubscribing. - disconnectB() + b.disconnect() - require.Eventually(t, func() bool { return !h.attached(keyB) }, 5*time.Second, 20*time.Millisecond, + require.Eventually(t, func() bool { return !h.attached(b.clientKey) }, 5*time.Second, 20*time.Millisecond, "a joined client must be released when its connection ends") - require.True(t, channelClosed(repliesB, 5*time.Second), + require.True(t, channelClosed(b.replies, 5*time.Second), "the joiner's reply channel must close so the ingress forwarder goroutine can end") - require.Eventually(t, func() bool { return h.manager.idMapper.ClientCount() == 1 }, 5*time.Second, 20*time.Millisecond, - "the joiner's id counter must go with it") - require.True(t, h.attached(keyA), "the creator is unaffected") + require.True(t, h.attached(a.clientKey), "the creator is unaffected") require.Equal(t, 0, h.upstreamCalls("eth_unsubscribe"), "the subscription still has a subscriber") // The creator leaves too: last one out tells the node. - disconnectA() - require.True(t, channelClosed(repliesA, 5*time.Second), "the creator's reply channel must close on disconnect") + a.disconnect() + require.True(t, channelClosed(a.replies, 5*time.Second), "the creator's reply channel must close on disconnect") require.Eventually(t, func() bool { return h.upstreamCalls("eth_unsubscribe") == 1 }, 15*time.Second, 50*time.Millisecond, "the last client leaving must tell the node to stop pushing") require.Eventually(t, func() bool { return h.activeSubscriptionCount() == 0 }, 5*time.Second, 20*time.Millisecond) - require.Eventually(t, func() bool { return h.manager.idMapper.ClientCount() == 0 }, 5*time.Second, 20*time.Millisecond) + require.Eventually(t, func() bool { return h.connectedClientCount() == 0 }, 5*time.Second, 20*time.Millisecond) + require.Equal(t, 0, h.manager.stickyStore.Len(), "no sticky endpoint survives its client") require.Equal(t, 0, h.manager.rateLimiter.ClientCount(), "no rate limiter survives its client") } @@ -155,21 +206,21 @@ func TestJoinedClientDisconnect_IsReleased(t *testing.T) { func TestCreatorDisconnect_ClosesItsReplyChannel(t *testing.T) { h := newReleaseHarness(t) - _, disconnectA, repliesA, keyA := h.subscribe("ws-a") - _, disconnectB, repliesB, keyB := h.subscribe("ws-b") - defer disconnectB() + a := h.subscribe("ws-a") + b := h.subscribe("ws-b") + defer b.disconnect() - disconnectA() + a.disconnect() - require.True(t, channelClosed(repliesA, 5*time.Second), "the creator's reply channel must close on disconnect") - require.Eventually(t, func() bool { return !h.attached(keyA) }, 5*time.Second, 20*time.Millisecond) - require.True(t, h.attached(keyB), "the joiner keeps the subscription alive") + require.True(t, channelClosed(a.replies, 5*time.Second), "the creator's reply channel must close on disconnect") + require.Eventually(t, func() bool { return !h.attached(a.clientKey) }, 5*time.Second, 20*time.Millisecond) + require.True(t, h.attached(b.clientKey), "the joiner keeps the subscription alive") require.Equal(t, 1, h.activeSubscriptionCount()) require.Equal(t, 0, h.upstreamCalls("eth_unsubscribe")) // The joiner still receives pushes after the creator left. select { - case reply, ok := <-repliesB: + case reply, ok := <-b.replies: require.True(t, ok, "the joiner's channel must stay open") require.NotNil(t, reply) case <-time.After(5 * time.Second): @@ -184,28 +235,59 @@ func TestCreatorDisconnect_ClosesItsReplyChannel(t *testing.T) { func TestUnsubscribeAll_ReleasesWatchersAndTellsNode(t *testing.T) { h := newReleaseHarness(t) - ctxA, disconnectA, repliesA, keyA := h.subscribe("ws-a") - ctxB, disconnectB, repliesB, keyB := h.subscribe("ws-b") - defer disconnectA() - defer disconnectB() + a := h.subscribe("ws-a") + b := h.subscribe("ws-b") + defer a.disconnect() + defer b.disconnect() require.Eventually(t, func() bool { return goroutinesIn("handleClientDisconnect") == 2 }, 5*time.Second, 20*time.Millisecond, "one disconnect watcher per attached client") - require.NoError(t, h.manager.UnsubscribeAll(ctxB, "dapp", "10.0.0.1", "ws-b", nil)) - require.False(t, h.attached(keyB)) - require.True(t, channelClosed(repliesB, 5*time.Second)) + h.connectionClosed(b) + require.False(t, h.attached(b.clientKey)) + require.True(t, channelClosed(b.replies, 5*time.Second)) require.Eventually(t, func() bool { return goroutinesIn("handleClientDisconnect") == 1 }, 5*time.Second, 20*time.Millisecond, "an explicit release must wake the client's watcher instead of leaving it parked until the socket closes") - require.True(t, h.attached(keyA)) + require.True(t, h.attached(a.clientKey)) require.Equal(t, 0, h.upstreamCalls("eth_unsubscribe")) - require.NoError(t, h.manager.UnsubscribeAll(ctxA, "dapp", "10.0.0.1", "ws-a", nil)) - require.True(t, channelClosed(repliesA, 5*time.Second)) + h.connectionClosed(a) + require.True(t, channelClosed(a.replies, 5*time.Second)) require.Eventually(t, func() bool { return h.upstreamCalls("eth_unsubscribe") == 1 }, 15*time.Second, 50*time.Millisecond, "the last client leaving through UnsubscribeAll must tell the node to stop pushing") require.Eventually(t, func() bool { return goroutinesIn("handleClientDisconnect") == 0 }, 5*time.Second, 20*time.Millisecond) require.Equal(t, 0, h.activeSubscriptionCount()) - require.Equal(t, 0, h.manager.idMapper.ClientCount()) + require.Equal(t, 0, h.connectedClientCount()) + require.Equal(t, 0, h.manager.stickyStore.Len()) require.Equal(t, 0, h.manager.rateLimiter.ClientCount()) } + +// TestUnsubscribeThenDisconnect_LeavesNothingBehind covers the commonest well-behaved +// client: subscribe, eth_unsubscribe, close the socket. The explicit unsubscribe empties +// the client's subscription set, and the connection-close release then found nothing to +// do and returned before handing back the client's sticky endpoint and rate limiters, +// so a well-behaved client leaked those per connection (MAG-3722, review finding). +func TestUnsubscribeThenDisconnect_LeavesNothingBehind(t *testing.T) { + h := newReleaseHarness(t) + + a := h.subscribe("ws-a") + defer a.disconnect() + require.Equal(t, 1, h.manager.stickyStore.Len(), "the creator pins the endpoint it subscribed on") + + h.unsubscribe(a) + require.True(t, channelClosed(a.replies, 5*time.Second), "an explicit unsubscribe closes the reply channel") + require.Eventually(t, func() bool { return h.activeSubscriptionCount() == 0 }, 5*time.Second, 20*time.Millisecond) + require.Eventually(t, func() bool { return goroutinesIn("handleClientDisconnect") == 0 }, 5*time.Second, 20*time.Millisecond, + "the disconnect watcher must not stay parked once the client unsubscribed") + require.Equal(t, 1, h.upstreamCalls("eth_unsubscribe")) + require.Equal(t, 0, h.connectedClientCount()) + require.Equal(t, 1, h.manager.stickyStore.Len(), "the sticky endpoint outlives an unsubscribe while the socket is open") + + // The socket closes. The ingress calls UnsubscribeAll for a client that has nothing + // left to unsubscribe, and that must still hand back its per-client state. + h.connectionClosed(a) + require.Equal(t, 0, h.manager.stickyStore.Len(), "closing the socket releases the sticky endpoint") + require.Equal(t, 0, h.manager.rateLimiter.ClientCount(), "closing the socket releases the rate limiters") + require.Equal(t, 0, h.connectedClientCount()) + require.Equal(t, 1, h.upstreamCalls("eth_unsubscribe"), "nothing is unsubscribed twice") +} diff --git a/protocol/rpcsmartrouter/subscription_id_mapper.go b/protocol/rpcsmartrouter/subscription_id_mapper.go index 71b3225ad..f91ec53af 100644 --- a/protocol/rpcsmartrouter/subscription_id_mapper.go +++ b/protocol/rpcsmartrouter/subscription_id_mapper.go @@ -18,7 +18,7 @@ import ( // while upstream nodes return their own IDs. This mapper maintains the translation. // // Key invariants: -// - Router IDs are unique per client (counter ensures this) +// - Router IDs are unique across the mapper (one counter numbers them all) // - Upstream IDs never exposed to clients // - Multiple router IDs can map to same upstream ID (subscription sharing) // - All operations are thread-safe @@ -30,15 +30,18 @@ type SubscriptionIDMapper struct { // One upstream subscription can serve multiple router subscriptions (dedup) upstreamToRouters map[string][]string - // clientCounters tracks per-client monotonic counters for ID generation - clientCounters map[string]*atomic.Uint64 - - // numericCounter backs GenerateNumericRouterID. Unlike clientCounters it is not - // per-client: a numeric id is handed to the client verbatim and is compared against - // upstream ids during unsubscribe, so it has to be unique across every client this - // mapper serves. That is the scope the lookups actually need — each manager builds its - // own mapper, so two managers do issue the same numbers, and nothing may key across - // them on this value. + // routerCounter numbers every id GenerateRouterID hands out. It is one counter for + // the whole mapper rather than one per client: a per-client counter needed a map + // entry per connection that ever subscribed, which nothing removed (MAG-3722), and + // an id only has to be unique, which a single counter guarantees for the life of the + // process without any per-client state to release. + routerCounter atomic.Uint64 + + // numericCounter backs GenerateNumericRouterID. A numeric id is handed to the client + // verbatim and is compared against upstream ids during unsubscribe, so it has to be + // unique across every client this mapper serves. That is the scope the lookups + // actually need — each manager builds its own mapper, so two managers do issue the + // same numbers, and nothing may key across them on this value. numericCounter atomic.Uint64 lock sync.RWMutex @@ -61,45 +64,15 @@ func NewSubscriptionIDMapper() *SubscriptionIDMapper { return &SubscriptionIDMapper{ routerToUpstream: make(map[string]string), upstreamToRouters: make(map[string][]string), - clientCounters: make(map[string]*atomic.Uint64), } } // GenerateRouterID creates a new router subscription ID for a client. -// Format: "rs_{clientHash6}_{counter5}" - e.g., "rs_a1b2c3_00042" -// The ID is unique per client and monotonically increasing. +// Format: "rs_{clientHash6}_{counter}" - e.g., "rs_a1b2c3_00042". The hash names the +// client for log correlation; the counter is mapper-wide, so an id is never reused +// for the life of the process, by any client. func (m *SubscriptionIDMapper) GenerateRouterID(clientKey string) string { - m.lock.Lock() - defer m.lock.Unlock() - - counter, exists := m.clientCounters[clientKey] - if !exists { - counter = &atomic.Uint64{} - m.clientCounters[clientKey] = counter - } - - // Format: rs_{first 6 chars of client hash}_{zero-padded counter} - clientHash := sha256Short(clientKey) - count := counter.Add(1) - - return fmt.Sprintf("rs_%s_%05d", clientHash, count) -} - -// RemoveClient forgets a client's ID counter. Call it once the client holds no -// subscriptions: the key carries the per-connection GUID, so nothing reuses it, -// and without this call the map kept one entry per connection that ever -// subscribed (MAG-3722). -func (m *SubscriptionIDMapper) RemoveClient(clientKey string) { - m.lock.Lock() - defer m.lock.Unlock() - delete(m.clientCounters, clientKey) -} - -// ClientCount reports how many clients currently hold an ID counter. -func (m *SubscriptionIDMapper) ClientCount() int { - m.lock.RLock() - defer m.lock.RUnlock() - return len(m.clientCounters) + return fmt.Sprintf("rs_%s_%05d", sha256Short(clientKey), m.routerCounter.Add(1)) } // GenerateNumericRouterID creates a router subscription ID for chains that number their diff --git a/protocol/rpcsmartrouter/subscription_id_mapper_leak_test.go b/protocol/rpcsmartrouter/subscription_id_mapper_leak_test.go deleted file mode 100644 index bc46be139..000000000 --- a/protocol/rpcsmartrouter/subscription_id_mapper_leak_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package rpcsmartrouter - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -// TestSubscriptionIDMapper_RemoveClientDropsCounter is the regression for MAG-3722: the -// per-client id counter was never removed, so the map kept one entry for every websocket -// connection that ever subscribed. -func TestSubscriptionIDMapper_RemoveClientDropsCounter(t *testing.T) { - mapper := NewSubscriptionIDMapper() - first := mapper.GenerateRouterID("client-a") - mapper.GenerateRouterID("client-b") - require.Equal(t, 2, mapper.ClientCount()) - - mapper.RemoveClient("client-a") - require.Equal(t, 1, mapper.ClientCount()) - - mapper.RemoveClient("client-a") // idempotent - require.Equal(t, 1, mapper.ClientCount()) - - // A client that comes back after release simply starts a fresh counter; every - // mapping it held was removed before its counter was, so the ids cannot collide - // with a live subscription. - require.Equal(t, first, mapper.GenerateRouterID("client-a")) - require.Equal(t, 2, mapper.ClientCount()) -} diff --git a/protocol/rpcsmartrouter/subscription_id_mapper_test.go b/protocol/rpcsmartrouter/subscription_id_mapper_test.go index 27aa13c50..45d82c5a4 100644 --- a/protocol/rpcsmartrouter/subscription_id_mapper_test.go +++ b/protocol/rpcsmartrouter/subscription_id_mapper_test.go @@ -42,11 +42,12 @@ func TestSubscriptionIDMapper_GenerateRouterID_DifferentClients(t *testing.T) { id2a := mapper.GenerateRouterID(client2) id2b := mapper.GenerateRouterID(client2) - // Each client has independent counter + // One counter numbers every id the mapper issues, so no two clients can ever be + // handed the same id and nothing per client has to be released (MAG-3722). assert.Contains(t, id1a, "_00001") assert.Contains(t, id1b, "_00002") - assert.Contains(t, id2a, "_00001") - assert.Contains(t, id2b, "_00002") + assert.Contains(t, id2a, "_00003") + assert.Contains(t, id2b, "_00004") // Different hash prefixes for different clients parts1 := strings.Split(id1a, "_") diff --git a/protocol/rpcsmartrouter/websocket_config.go b/protocol/rpcsmartrouter/websocket_config.go index 283174b2b..1b48f1fa7 100644 --- a/protocol/rpcsmartrouter/websocket_config.go +++ b/protocol/rpcsmartrouter/websocket_config.go @@ -119,6 +119,10 @@ func NewClientRateLimiter(config *WebsocketConfig) *ClientRateLimiter { subscribeRate := rate.Limit(float64(config.SubscriptionsPerMinutePerClient) / 60.0) unsubscribeRate := rate.Limit(float64(config.UnsubscribesPerMinutePerClient) / 60.0) + // Both limiters set the burst to the per-minute allowance and the rate to that + // allowance per second, so burst/rate is one minute for every configuration and + // this always evaluates to a minute today. The derivation stays so a future + // burst/rate split cannot silently shorten the sweep. idleTTL := max( refillDuration(subscribeRate, config.SubscriptionsPerMinutePerClient), refillDuration(unsubscribeRate, config.UnsubscribesPerMinutePerClient), From 1294b3dd7f3ddb2d30794f138563823e6c9c7c8d Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Thu, 17 Sep 2026 11:46:20 +0300 Subject: [PATCH 7/8] fix(provideroptimizer): keep the relay windows in a locked map, state the half-life change Review follow-ups on the relay-time window. The windows live in a plain mutex-guarded map rather than a ristretto cache: the set is bounded by the pairing list, nothing in it should be evicted, and ristretto's asynchronous Set let the relay right after a window's birth miss it and start a second window that dropped the samples in between. The window grows one sample at a time and wraps once full, so a provider that served a handful of relays costs a handful of entries. The comment now says what the change does to routing: with the unbounded history the median sat at half the uptime, so every provider on a pod up more than six hours decayed on the 3 h clamp; a busy provider now decays on the 1 h default, which is what the heuristic was written to do. A new test pins the values at three relay rates. Jira ticket: MAG-3722 --- .../provideroptimizer/provider_optimizer.go | 126 +++++++++++------- .../relay_time_window_test.go | 58 ++++++-- 2 files changed, 128 insertions(+), 56 deletions(-) diff --git a/protocol/provideroptimizer/provider_optimizer.go b/protocol/provideroptimizer/provider_optimizer.go index 6f569a4a1..bb069b2ad 100644 --- a/protocol/provideroptimizer/provider_optimizer.go +++ b/protocol/provideroptimizer/provider_optimizer.go @@ -50,7 +50,7 @@ type consumerOptimizerQoSClientInf interface { type ProviderOptimizer struct { strategy Strategy providersStorage cacheInf - providerRelayStats *ristretto.Cache[string, any] // used to decide on the half time of the decay + providerRelayStats *relayTimeWindows // per-provider relay times, used to decide on the half time of the decay averageBlockTime time.Duration wantedNumProvidersInConcurrency uint latestSyncData ConcurrentBlockStore @@ -882,50 +882,107 @@ func (po *ProviderOptimizer) updateDecayingWeightedAverage(providerData Provider // calculateHalfTime reads exactly one of them, the median, so the window only has // to be wide enough for that median to mean "the time it takes this provider to // serve half a window of relays". Unbounded, the slice grew by 24 bytes on every -// relay for the life of the process and its median drifted to half the uptime, -// which the MaxHalfTime clamp then discarded anyway (MAG-3722). +// relay for the life of the process, and its median drifted to half the uptime: +// after six hours every provider sat on the MaxHalfTime clamp, so scores decayed +// on a 3 h half-life regardless of traffic. With the window, a provider serving +// more than a window of relays per hour decays on DefaultHalfLifeTime (1 h) and +// only a sparse one stretches towards the clamp, which is what the heuristic was +// written to do. That is a change in production routing dynamics for busy +// providers, 3 h to 1 h, not only a memory fix (MAG-3722). const relayStatsWindowSize = 128 // relayTimeWindow holds a provider's most recent relay sample times in arrival order. -// It is mutated in place under the provider's stripe lock, so the pointer stored in -// providerRelayStats never needs a replacement Set once it is in the cache. +// It grows one sample at a time until it is full and then wraps, so a provider that +// has served a handful of relays costs a handful of entries. type relayTimeWindow struct { - times [relayStatsWindowSize]time.Time - next int // slot the next sample lands in - count int // samples held, at most len(times) + times []time.Time // at most relayStatsWindowSize entries + next int // slot the next sample overwrites once the window is full } // add records one sample, overwriting the oldest once the window is full. func (w *relayTimeWindow) add(sampleTime time.Time) { - w.times[w.next] = sampleTime - w.next = (w.next + 1) % len(w.times) - if w.count < len(w.times) { - w.count++ + if len(w.times) < relayStatsWindowSize { + w.times = append(w.times, sampleTime) + return } + w.times[w.next] = sampleTime + w.next = (w.next + 1) % relayStatsWindowSize } // median returns the middle sample by arrival order, which for monotonic sample // times is the median time. ok is false while the window is empty. func (w *relayTimeWindow) median() (median time.Time, ok bool) { - if w.count == 0 { + n := len(w.times) + if n == 0 { return time.Time{}, false } - oldest := (w.next - w.count + len(w.times)) % len(w.times) - return w.times[(oldest+(w.count-1)/2)%len(w.times)], true + oldest := 0 + if n == relayStatsWindowSize { + oldest = w.next // the slot about to be overwritten holds the oldest sample + } + return w.times[(oldest+(n-1)/2)%n], true } -// updateRelayTime adds a relay sample time to a provider's window. -func (po *ProviderOptimizer) updateRelayTime(providerAddress string, sampleTime time.Time) { - window := po.getRelayStatsWindow(providerAddress) - if window == nil { +// relayTimeWindows holds one relayTimeWindow per provider behind a mutex. A plain map +// rather than a ristretto cache: the set is bounded by the pairing list, nothing in it +// should ever be evicted, and ristretto's asynchronous Set let the relay right after a +// window's birth miss it and start a second window that dropped the samples between. +type relayTimeWindows struct { + mu sync.Mutex + windows map[string]*relayTimeWindow +} + +func newRelayTimeWindows() *relayTimeWindows { + return &relayTimeWindows{windows: make(map[string]*relayTimeWindow)} +} + +// record adds one sample to the provider's window, creating the window on first use. +func (s *relayTimeWindows) record(providerAddress string, sampleTime time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + window, ok := s.windows[providerAddress] + if !ok { window = &relayTimeWindow{} - window.add(sampleTime) - po.providerRelayStats.Set(providerAddress, window, 1) - return + s.windows[providerAddress] = window } window.add(sampleTime) } +// median returns the median sample time of the provider's window; ok is false when +// the provider has no samples. +func (s *relayTimeWindows) median(providerAddress string) (median time.Time, ok bool) { + s.mu.Lock() + defer s.mu.Unlock() + window, found := s.windows[providerAddress] + if !found { + return time.Time{}, false + } + return window.median() +} + +// size reports how many samples the provider's window holds. +func (s *relayTimeWindows) size(providerAddress string) int { + s.mu.Lock() + defer s.mu.Unlock() + window, found := s.windows[providerAddress] + if !found { + return 0 + } + return len(window.times) +} + +// Clear drops every window. +func (s *relayTimeWindows) Clear() { + s.mu.Lock() + defer s.mu.Unlock() + s.windows = make(map[string]*relayTimeWindow) +} + +// updateRelayTime adds a relay sample time to a provider's window. +func (po *ProviderOptimizer) updateRelayTime(providerAddress string, sampleTime time.Time) { + po.providerRelayStats.record(providerAddress, sampleTime) +} + // calculateHalfTime calculates a provider's half life time for a relay sampled in sampleTime func (po *ProviderOptimizer) calculateHalfTime(providerAddress string, sampleTime time.Time) time.Duration { halfTime := score.DefaultHalfLifeTime @@ -941,11 +998,7 @@ func (po *ProviderOptimizer) calculateHalfTime(providerAddress string, sampleTim // getRelayStatsTimeDiff returns the time passed since the median of the provider's recent relay times func (po *ProviderOptimizer) getRelayStatsTimeDiff(providerAddress string, sampleTime time.Time) time.Duration { - window := po.getRelayStatsWindow(providerAddress) - if window == nil { - return 0 - } - medianTime, ok := window.median() + medianTime, ok := po.providerRelayStats.median(providerAddress) if !ok { return 0 } @@ -960,28 +1013,11 @@ func (po *ProviderOptimizer) getRelayStatsTimeDiff(providerAddress string, sampl return time.Since(medianTime) } -// getRelayStatsWindow returns the provider's relay time window, or nil when none is cached yet. -func (po *ProviderOptimizer) getRelayStatsWindow(providerAddress string) *relayTimeWindow { - storedVal, found := po.providerRelayStats.Get(providerAddress) - if !found { - return nil - } - window, ok := storedVal.(*relayTimeWindow) - if !ok { - utils.LavaFormatFatal("invalid usage of optimizer relay stats cache", nil, utils.Attribute{Key: "storedVal", Value: storedVal}) - } - return window -} - func NewProviderOptimizer(strategy Strategy, averageBlockTIme time.Duration, wantedNumProvidersInConcurrency uint, consumerOptimizerQoSClient consumerOptimizerQoSClientInf, chainId string) *ProviderOptimizer { cache, err := ristretto.NewCache(&ristretto.Config[string, any]{NumCounters: CacheNumCounters, MaxCost: CacheMaxCost, BufferItems: 64, IgnoreInternalCost: true}) if err != nil { utils.LavaFormatFatal("failed setting up cache for queries", err) } - relayCache, err := ristretto.NewCache(&ristretto.Config[string, any]{NumCounters: CacheNumCounters, MaxCost: CacheMaxCost, BufferItems: 64, IgnoreInternalCost: true}) - if err != nil { - utils.LavaFormatFatal("failed setting up cache for queries", err) - } if strategy == StrategyPrivacy { // overwrite wantedNumProvidersInConcurrency = 1 @@ -1015,7 +1051,7 @@ func NewProviderOptimizer(strategy Strategy, averageBlockTIme time.Duration, wan strategy: strategy, providersStorage: cache, averageBlockTime: averageBlockTIme, - providerRelayStats: relayCache, + providerRelayStats: newRelayTimeWindows(), wantedNumProvidersInConcurrency: wantedNumProvidersInConcurrency, stakeCache: NewProviderStakeCache(), consumerOptimizerQoSClient: consumerOptimizerQoSClient, diff --git a/protocol/provideroptimizer/relay_time_window_test.go b/protocol/provideroptimizer/relay_time_window_test.go index 6e9585cf7..f004fedf9 100644 --- a/protocol/provideroptimizer/relay_time_window_test.go +++ b/protocol/provideroptimizer/relay_time_window_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/magma-Devs/smart-router/utils/score" "github.com/stretchr/testify/require" ) @@ -22,11 +23,20 @@ func TestRelayTimeWindow_BoundedAndMedian(t *testing.T) { require.True(t, ok) require.Equal(t, base, median, "one sample is its own median") + // Half full: the median is the middle of what has arrived so far. + for i := 1; i < relayStatsWindowSize/2; i++ { + w.add(base.Add(time.Duration(i) * time.Second)) + } + require.Len(t, w.times, relayStatsWindowSize/2, "the window grows one sample at a time") + median, ok = w.median() + require.True(t, ok) + require.Equal(t, base.Add(time.Duration((relayStatsWindowSize/2-1)/2)*time.Second), median) + const total = 10 * relayStatsWindowSize - for i := 1; i < total; i++ { + for i := relayStatsWindowSize / 2; i < total; i++ { w.add(base.Add(time.Duration(i) * time.Second)) } - require.Equal(t, relayStatsWindowSize, w.count, "the window must stop growing at its size") + require.Len(t, w.times, relayStatsWindowSize, "the window must stop growing at its size") // The window holds samples [total-size, total). Its median is the one at offset // (size-1)/2 from the oldest retained sample. @@ -51,11 +61,8 @@ func TestUpdateRelayTime_DoesNotGrowWithRelays(t *testing.T) { for i := 0; i < relays; i++ { po.updateRelayTime(addr, base.Add(time.Duration(i)*time.Second)) } - po.providerRelayStats.Wait() // ristretto Set is asynchronous - window := po.getRelayStatsWindow(addr) - require.NotNil(t, window, "the provider's window must be cached") - require.Equal(t, relayStatsWindowSize, window.count, + require.Equal(t, relayStatsWindowSize, po.providerRelayStats.size(addr), "after %d relays the window must hold exactly its size, not one entry per relay", relays) // The median of the last relayStatsWindowSize one-second samples sits @@ -65,12 +72,41 @@ func TestUpdateRelayTime_DoesNotGrowWithRelays(t *testing.T) { wantAge := time.Duration(relayStatsWindowSize/2) * time.Second require.Equal(t, wantAge, po.getRelayStatsTimeDiff(addr, newest)) - // A busy provider therefore keeps the default half-life; only a provider slower - // than a window per DefaultHalfLifeTime stretches it. - require.Equal(t, po.calculateHalfTime(addr, newest), po.calculateHalfTime("never-seen", newest), + // A busy provider therefore decays on the default half-life. Before the window, + // the median sat at half the uptime, so every provider on a pod up more than six + // hours decayed on the 3 h clamp instead; this pins the intended behaviour. + require.Equal(t, score.DefaultHalfLifeTime, po.calculateHalfTime(addr, newest), "a provider serving relays every second must decay on the default half-life") po.ResetState() - po.providerRelayStats.Wait() - require.Nil(t, po.getRelayStatsWindow(addr), "ResetState must drop the window") + require.Equal(t, 0, po.providerRelayStats.size(addr), "ResetState must drop the window") +} + +// TestCalculateHalfTime_StretchesOnlyForSparseProviders pins the heuristic the window +// serves: the half-life is the time it takes a provider to serve half a window of +// relays, floored at the default and capped at the clamp. +func TestCalculateHalfTime_StretchesOnlyForSparseProviders(t *testing.T) { + po := setupProviderOptimizer(1) + base := time.Date(2026, 9, 16, 0, 0, 0, 0, time.UTC) + halfWindow := time.Duration(relayStatsWindowSize / 2) + + for _, tc := range []struct { + name string + spacing time.Duration + want time.Duration + }{ + {"one relay a second stays on the default", time.Second, score.DefaultHalfLifeTime}, + {"one relay every two minutes stretches to the window's half", 2 * time.Minute, halfWindow * 2 * time.Minute}, + {"one relay every five minutes hits the clamp", 5 * time.Minute, score.MaxHalfTime}, + } { + t.Run(tc.name, func(t *testing.T) { + addr := "provider-" + tc.name + var newest time.Time + for i := 0; i < relayStatsWindowSize; i++ { + newest = base.Add(time.Duration(i) * tc.spacing) + po.updateRelayTime(addr, newest) + } + require.Equal(t, tc.want, po.calculateHalfTime(addr, newest)) + }) + } } From 849177576a009831edf026068f44a70b48ef2f09 Mon Sep 17 00:00:00 2001 From: Nimrod Teich Date: Thu, 17 Sep 2026 11:46:20 +0300 Subject: [PATCH 8/8] docs(rpcclient): describe the id-bearing case in forgetClientSub A late push without a request id is dropped as before; one with a request id now reaches the call path and is answered with an error upstream, where the stale entry used to swallow it. Jira ticket: MAG-3722 --- protocol/chainlib/chainproxy/rpcclient/handler.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/protocol/chainlib/chainproxy/rpcclient/handler.go b/protocol/chainlib/chainproxy/rpcclient/handler.go index f239d9978..f89fa539a 100755 --- a/protocol/chainlib/chainproxy/rpcclient/handler.go +++ b/protocol/chainlib/chainproxy/rpcclient/handler.go @@ -531,8 +531,10 @@ func (h *handler) runMethod(ctx context.Context, msg *JsonrpcMessage, callb *cal // forgetClientSub drops sub from the dispatch table once its forwarding loop has been // told to quit. Runs on the dispatch goroutine, which is the only owner of clientSubs. -// A push for the id that still arrives is dropped by deliverSubscriptionPush as an -// unmatched notification, exactly as before the entry was removed. +// A push for the id that still arrives then reaches deliverSubscriptionPush as an +// unmatched frame: without a request id it is dropped, the fate it met while the entry +// existed; with one it goes to the call path, which answers the node with a JSON-RPC +// error where the entry used to swallow it. func (h *handler) forgetClientSub(sub *ClientSubscription) { if sub == nil { return