Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 70 additions & 18 deletions protocol/rpcsmartrouter/direct_ws_subscription_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,11 @@ type DirectWSSubscriptionManager struct {
// Pending subscriptions for deduplication
pendingSubscriptions map[string]*pendingSubscriptionsBroadcastManager

// beforeRestoreCommit, when set, runs in handleUpstreamDisconnect after the node has
// answered the re-subscribe and before the result is committed under dwsm.lock. Nil
// outside tests; tests use it to hold a restore at exactly that point.
beforeRestoreCommit func()

// Upstream connection pools keyed by endpoint URL
upstreamPools map[string]*UpstreamWSPool

Expand Down Expand Up @@ -1481,7 +1486,7 @@ func (dwsm *DirectWSSubscriptionManager) listenForUpstreamMessages(
reconnectInFlight := false
defer func() {
if !reconnectInFlight {
dwsm.cleanupSubscription(hashedParams)
dwsm.cleanupSubscription(hashedParams, activeSub)
}
}()

Expand Down Expand Up @@ -1579,9 +1584,12 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect(
}
defer activeSub.restoring.Store(false)

dwsm.lock.RLock()
connectedClients := len(activeSub.clientRouterIDs)
dwsm.lock.RUnlock()
utils.LavaFormatInfo("DirectWS: handling upstream disconnect",
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
utils.LogAttr("connectedClients", len(activeSub.clientRouterIDs)),
utils.LogAttr("connectedClients", connectedClients),
)

// Try to reconnect the WebSocket connection
Expand All @@ -1590,7 +1598,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect(
utils.LavaFormatError("DirectWS: reconnection failed, cleaning up subscription", err,
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
)
dwsm.cleanupSubscription(hashedParams)
dwsm.cleanupSubscription(hashedParams, activeSub)
return
}

Expand All @@ -1605,7 +1613,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect(
utils.LavaFormatError("DirectWS: failed to get connection after reconnect", err,
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
)
dwsm.cleanupSubscription(hashedParams)
dwsm.cleanupSubscription(hashedParams, activeSub)
return
}

Expand All @@ -1615,7 +1623,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect(
utils.LavaFormatError("DirectWS: failed to restore subscription", err,
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
)
dwsm.cleanupSubscription(hashedParams)
dwsm.cleanupSubscription(hashedParams, activeSub)
return
}

Expand All @@ -1625,8 +1633,29 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect(
// Increment subscription count on the NEW connection (matching initial creation behavior)
conn.IncrementSubscriptions()

if dwsm.beforeRestoreCommit != nil {
dwsm.beforeRestoreCommit()
}

// Update the ID mapping, message channel, and connection bookkeeping
dwsm.lock.Lock()
if dwsm.activeSubscriptions[hashedParams] != activeSub {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Medium · exists on main, same race family as this PR. An explicit eth_unsubscribe that overlaps the restore commit still leaves the restored subscription live on the node. So "Nothing is left to leak" doesn't hold on this path:

  1. Unsubscribe reads conn and upstreamID under the lock (:1002-1003), unlocks, and tells the node to drop the old id.
  2. Meanwhile the restore commits. The entry is still mapped, so this check passes, and the id, the connection and the client's mapping move to the new subscription.
  3. Unsubscribe re-locks. RemoveMapping finds the mapping, so this is the last client, and it tears down with tellNode=false (:1063). Nobody tells the node about the restored id.

This is the natural ordering in production.

  • The pool hands back the same connection, so the unsubscribe queues on the same rpcclient behind the restore's Subscribe.
  • A real node answers the stale id with a JSON-RPC error, and CallContext returns that as a message with err == nil.

Probe: hold the restore at the test hook, stall the node's writes, start Unsubscribe, then release. It prints methods=[eth_subscribe eth_subscribe eth_unsubscribe] nodeHasNew=true total=0, 3/3 here, on main and with #508.

Fix (pushing):

  • In Unsubscribe's second critical section, tell the node when a restore committed since the read (activeSub.upstreamID != upstreamID || activeSub.upstreamConnection != conn).
  • Add TestUnsubscribeDuringRestore_TellsTheNodeTheRestoredID.

// The subscription was torn down while the node was answering the re-subscribe:
// its last client left, or it was dropped, and the teardown released everything
// the old upstream subscription held. Nothing owns the one just created, so it
// ends here — locally, on the node, and in the connection's count — instead of
// living on as an upstream subscription nobody reads and a count that stops the
// pool from ever scaling the connection down (MAG-2547).
dwsm.lock.Unlock()
utils.LavaFormatInfo("DirectWS: subscription was torn down while being restored, discarding the restored upstream subscription",
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
utils.LogAttr("newUpstreamID", newUpstreamID),
)
newUpstreamSub.Unsubscribe()
dwsm.teardownUpstreamSubscription(conn, newUpstreamID, activeSub.subscribeMethod)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Low · exists on main. The MAG-3527 closing comment noted this, but no ticket was filed.

  • Cause: teardownUpstreamSubscription derives the method with getUnsubscribeMethod and always sends the id as a string. That is right only for eth_subscribe/subscribe with string ids.
  • Result on Solana: this discard sends unsubscribe, a method the node doesn't have, so the discarded subscription stays live there. With the Solana mock, the node saw [accountSubscribe accountSubscribe unsubscribe unsubscribe].
  • Why not here: the fix means storing the resolved unsubscribe method and the id encoding at subscribe time. That is a design call, so I'm not changing it in this PR; it is worth its own ticket.

activeSub.upstreamPool.NotifySubscriptionRemoved(conn)
return
}
oldUpstreamID := activeSub.upstreamID
oldConnection := activeSub.upstreamConnection // Keep reference to old connection for cleanup

Expand Down Expand Up @@ -1755,13 +1784,22 @@ func (dwsm *DirectWSSubscriptionManager) forgetClientLocked(clientKey string) {
// local subscription ends and the pool may scale the connection down. The network call runs
// off the lock. Caller holds dwsm.lock.
func (dwsm *DirectWSSubscriptionManager) teardownSubscriptionLocked(hashedParams string, activeSub *directActiveSubscription, tellNode bool) {
// Everything the goroutine needs is read here, under the lock. A restore that is
// mid-flight rewrites upstreamSubscription, upstreamID and upstreamConnection under
// this same lock once the node answers, so reading them off the lock raced that
// write and could tear down whichever half of the restore it happened to see.
upstreamConn := activeSub.upstreamConnection
upstreamPool := activeSub.upstreamPool
upstreamSub := activeSub.upstreamSubscription
upstreamID := activeSub.upstreamID
subscribeMethod := activeSub.subscribeMethod
go func() {
if tellNode {
dwsm.teardownUpstreamSubscription(activeSub)
dwsm.teardownUpstreamSubscription(upstreamConn, upstreamID, subscribeMethod)
}
if upstreamSub != nil {
upstreamSub.Unsubscribe()
}
activeSub.upstreamSubscription.Unsubscribe()
activeSub.cancel()
activeSub.stopUpstreamListener()
if upstreamConn != nil {
Expand Down Expand Up @@ -1795,43 +1833,55 @@ const upstreamUnsubscribeTimeout = 10 * time.Second
//
// Best-effort by design: it also runs where the connection may already be gone
// (upstream disconnect, shutdown), so a failure is traced rather than warned.
// Must be called WITHOUT dwsm.lock held — it makes a network call.
func (dwsm *DirectWSSubscriptionManager) teardownUpstreamSubscription(activeSub *directActiveSubscription) {
if activeSub == nil || activeSub.upstreamConnection == nil {
// Must be called WITHOUT dwsm.lock held — it makes a network call. It takes the
// connection and id as values rather than reading them off the subscription, because
// a restore in flight rewrites those fields under dwsm.lock while this runs off it
// (MAG-2547): the caller snapshots them while it still holds the lock.
func (dwsm *DirectWSSubscriptionManager) teardownUpstreamSubscription(upstreamConn *UpstreamWSConnection, upstreamID, subscribeMethod string) {
if upstreamConn == nil {
return
}
client := activeSub.upstreamConnection.GetClient()
client := upstreamConn.GetClient()
if client == nil {
return
}

var params interface{}
if dwsm.apiInterface == "tendermintrpc" {
params = map[string]interface{}{"query": activeSub.upstreamID}
params = map[string]interface{}{"query": upstreamID}
} else {
params = []interface{}{activeSub.upstreamID}
params = []interface{}{upstreamID}
}

ctx, cancel := context.WithTimeout(context.Background(), upstreamUnsubscribeTimeout)
defer cancel()
if _, err := client.CallContext(ctx, json.RawMessage("1"),
getUnsubscribeMethod(activeSub.subscribeMethod), params, true, false); err != nil {
getUnsubscribeMethod(subscribeMethod), params, true, false); err != nil {
utils.LavaFormatTrace("DirectWS: upstream unsubscribe on disconnect failed",
utils.LogAttr("error", err),
utils.LogAttr("upstreamID", activeSub.upstreamID),
utils.LogAttr("upstreamID", upstreamID),
)
}
}

// cleanupSubscription cleans up a subscription
func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string) {
// cleanupSubscription cleans up a subscription, but only the one its caller owns. The map
// is keyed by params alone, so once owner has been torn down a new client can register a
// fresh subscription under the same key; a late caller (a listener ending, a restore
// failing) that cleaned up by key would then destroy that newer subscription.
func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string, owner *directActiveSubscription) {
dwsm.lock.Lock()
defer dwsm.lock.Unlock()

activeSub, found := dwsm.activeSubscriptions[hashedParams]
if !found {
return
}
if activeSub != owner {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Low · test gap. This guard fixes a real main bug, but no test pins it: with the guard removed, all 73 WS subscription tests on this branch still pass, the new one included.

The main bug it prevents:

  • A torn-down subscription's listener exits only after the teardown's node call returns, which can take up to upstreamUnsubscribeTimeout (10 s).
  • Its deferred cleanup by key then destroys any successor registered under the same params in the meantime: it closes the successor client's channel, unsubscribes it on the node, and decrements the counter to 0.

Tests I'm adding:

  • TestListenerEndingLate_LeavesASuccessorAlone: fails on main, and here without the guard.
  • TestRestoreAfterTeardown_LeavesASuccessorAlone: fails on main.

utils.LavaFormatTrace("DirectWS: subscription under this key is not the one being cleaned up, leaving it",
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
)
return
}

// Release every client: closes their reply channels and wakes their watchers.
for clientKey := range activeSub.connectedClients {
Expand All @@ -1852,8 +1902,10 @@ func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string
// pool notify goes after it so the connection is still live when it is used.
upstreamConn := activeSub.upstreamConnection
upstreamPool := activeSub.upstreamPool
upstreamID := activeSub.upstreamID
subscribeMethod := activeSub.subscribeMethod
go func() {
dwsm.teardownUpstreamSubscription(activeSub)
dwsm.teardownUpstreamSubscription(upstreamConn, upstreamID, subscribeMethod)
if upstreamConn != nil {
upstreamPool.NotifySubscriptionRemoved(upstreamConn)
}
Expand Down
29 changes: 29 additions & 0 deletions protocol/rpcsmartrouter/direct_ws_subscription_manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ type mockSubscriptionServer struct {
// subscriptionSeq makes successive ids distinct, so a test can tell a restored
// subscription from the one it replaced.
subscriptionSeq int

// conns holds every server-side socket still open, so a test can drop them all and
// watch the router restore its subscriptions.
conns []*websocket.Conn
}

// safeWriteMessage serializes WS writes: handleWS (response) and the spawned sendSubscriptionMessages
Expand Down Expand Up @@ -90,6 +94,9 @@ func (ms *mockSubscriptionServer) handleWS(w http.ResponseWriter, r *http.Reques
return
}
defer conn.Close()
ms.lock.Lock()
ms.conns = append(ms.conns, conn)
ms.lock.Unlock()

for {
_, message, err := conn.ReadMessage()
Expand Down Expand Up @@ -277,6 +284,28 @@ func (ms *mockSubscriptionServer) sendSubscriptionMessages(conn *websocket.Conn,
}
}

// dropConnections closes every server-side socket, which is what an upstream node
// restarting looks like from the router: every subscription on those sockets errors and
// the router re-subscribes over a fresh connection.
func (ms *mockSubscriptionServer) dropConnections() {
ms.lock.Lock()
conns := ms.conns
ms.conns = nil
ms.lock.Unlock()
for _, conn := range conns {
conn.Close()
}
}

// liveSubscriptions is how many subscriptions the node still believes exist: created and
// not yet unsubscribed. A router that leaks an upstream subscription leaves this above zero
// after every client has gone.
func (ms *mockSubscriptionServer) liveSubscriptions() int {
ms.lock.RLock()
defer ms.lock.RUnlock()
return len(ms.subscriptions)
}

func (ms *mockSubscriptionServer) Close() {
ms.lock.Lock()
defer ms.lock.Unlock()
Expand Down
21 changes: 21 additions & 0 deletions protocol/rpcsmartrouter/direct_ws_subscription_release_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,27 @@ func (h *releaseHarness) attached(clientKey string) bool {
return false
}

// pooledSubscriptionCount sums what every pooled upstream connection believes it carries.
// The pool scales a connection down only when this reaches zero, so a count left behind
// keeps a connection open for the life of the process.
func (h *releaseHarness) pooledSubscriptionCount() int32 {
h.manager.lock.RLock()
pools := make([]*UpstreamWSPool, 0, len(h.manager.upstreamPools))
for _, pool := range h.manager.upstreamPools {
pools = append(pools, pool)
}
h.manager.lock.RUnlock()
var n int32
for _, pool := range pools {
pool.lock.RLock()
for _, conn := range pool.connections {
n += conn.SubscriptionCount()
}
pool.lock.RUnlock()
}
return n
}

func (h *releaseHarness) activeSubscriptionCount() int {
h.manager.lock.RLock()
defer h.manager.lock.RUnlock()
Expand Down
63 changes: 63 additions & 0 deletions protocol/rpcsmartrouter/direct_ws_subscription_restore_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package rpcsmartrouter

import (
"sync"
"testing"
"time"

"github.com/stretchr/testify/require"
)

// TestRestoreAfterTeardown_LeavesNoUpstreamSubscription is MAG-2547: an upstream socket
// drops, the router re-subscribes, and the subscription's last client disconnects while
// that restore is still waiting on the node. The teardown runs first and releases the
// old upstream subscription; the restore must then notice its subscription is gone and
// end the one it just created — on the node, locally, and in the connection's count —
// rather than committing it onto an entry nothing holds any more. Before the fix the
// restored upstream subscription stayed live on the node with nobody reading it, the
// connection kept counting it so the pool could never scale it down, and the teardown
// goroutine read the subscription's upstream fields off the lock while the restore
// rewrote them under it.
func TestRestoreAfterTeardown_LeavesNoUpstreamSubscription(t *testing.T) {
h := newReleaseHarness(t)

reached := make(chan struct{})
release := make(chan struct{})
var reachedOnce sync.Once
h.manager.beforeRestoreCommit = func() {
reachedOnce.Do(func() { close(reached) })
<-release
}

sub := h.subscribe("ws-1")
require.Equal(t, 1, h.srv.liveSubscriptions())

// The upstream socket drops. The router re-subscribes on a fresh connection and is
// held just before it commits the result.
h.srv.dropConnections()
select {
case <-reached:
case <-time.After(10 * time.Second):
t.Fatal("the restore never re-subscribed on the node")
}
require.Equal(t, 2, h.upstreamCalls("eth_subscribe"), "the restore subscribed again on the node")

// The last client leaves while the restore is held, so the subscription is torn down
// underneath it.
sub.disconnect()
require.Eventually(t, func() bool { return h.activeSubscriptionCount() == 0 },
5*time.Second, 10*time.Millisecond, "the last client leaving tears the subscription down")

close(release)

require.Eventually(t, func() bool { return h.srv.liveSubscriptions() == 0 },
5*time.Second, 10*time.Millisecond,
"the restored upstream subscription was left running on the node with nobody reading it")
require.Eventually(t, func() bool { return h.pooledSubscriptionCount() == 0 },
5*time.Second, 10*time.Millisecond,
"a pooled connection still counts a subscription, so the pool can never scale it down")
require.Equal(t, int64(0), h.manager.totalSubscriptions.Load())
require.Equal(t, 0, h.activeSubscriptionCount())
require.Eventually(t, func() bool { return goroutinesIn("listenForUpstreamMessages") == 0 },
5*time.Second, 10*time.Millisecond, "no upstream listener may outlive the subscription")
}