Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 79 additions & 23 deletions protocol/rpcsmartrouter/direct_ws_subscription_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,11 @@ type DirectWSSubscriptionManager struct {
// Pending subscriptions for deduplication
pendingSubscriptions map[string]*pendingSubscriptionsBroadcastManager

// beforeRestoreCommit, when set, runs in handleUpstreamDisconnect after the node has
// answered the re-subscribe and before the result is committed under dwsm.lock. Nil
// outside tests; tests use it to hold a restore at exactly that point.
beforeRestoreCommit func()

// Upstream connection pools keyed by endpoint URL
upstreamPools map[string]*UpstreamWSPool

Expand Down Expand Up @@ -1053,9 +1058,12 @@ func (dwsm *DirectWSSubscriptionManager) Unsubscribe(
// lastClient is the mapper's view after this client's mapping went, read under the
// lock: a client that joined during the unlocked upstream call above registered its
// mapping first, so it keeps the subscription. The node was already told, so the
// teardown skips the wire call.
// teardown skips the wire call — unless a restore committed while that call was in
// flight: the node was told the id read before it, and the restored subscription,
// with its new id and connection, was never told (MAG-2547).
if lastClient {
dwsm.teardownSubscriptionLocked(hashedParams, activeSub, false)
restoredSinceRead := activeSub.upstreamID != upstreamID || activeSub.upstreamConnection != conn
dwsm.teardownSubscriptionLocked(hashedParams, activeSub, restoredSinceRead)
}

return nodeResp, nil
Expand Down Expand Up @@ -1498,7 +1506,7 @@ func (dwsm *DirectWSSubscriptionManager) listenForUpstreamMessages(
reconnectInFlight := false
defer func() {
if !reconnectInFlight {
dwsm.cleanupSubscription(hashedParams)
dwsm.cleanupSubscription(hashedParams, activeSub)
}
}()

Expand Down Expand Up @@ -1596,9 +1604,12 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect(
}
defer activeSub.restoring.Store(false)

dwsm.lock.RLock()
connectedClients := len(activeSub.clientRouterIDs)
dwsm.lock.RUnlock()
utils.LavaFormatInfo("DirectWS: handling upstream disconnect",
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
utils.LogAttr("connectedClients", len(activeSub.clientRouterIDs)),
utils.LogAttr("connectedClients", connectedClients),
)

// Try to reconnect the WebSocket connection
Expand All @@ -1607,7 +1618,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect(
utils.LavaFormatError("DirectWS: reconnection failed, cleaning up subscription", err,
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
)
dwsm.cleanupSubscription(hashedParams)
dwsm.cleanupSubscription(hashedParams, activeSub)
return
}

Expand All @@ -1622,7 +1633,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect(
utils.LavaFormatError("DirectWS: failed to get connection after reconnect", err,
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
)
dwsm.cleanupSubscription(hashedParams)
dwsm.cleanupSubscription(hashedParams, activeSub)
return
}

Expand All @@ -1632,7 +1643,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect(
utils.LavaFormatError("DirectWS: failed to restore subscription", err,
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
)
dwsm.cleanupSubscription(hashedParams)
dwsm.cleanupSubscription(hashedParams, activeSub)
return
}

Expand All @@ -1642,8 +1653,29 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect(
// Increment subscription count on the NEW connection (matching initial creation behavior)
conn.IncrementSubscriptions()

if dwsm.beforeRestoreCommit != nil {
dwsm.beforeRestoreCommit()
}

// Update the ID mapping, message channel, and connection bookkeeping
dwsm.lock.Lock()
if dwsm.activeSubscriptions[hashedParams] != activeSub {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Medium · exists on main, same race family as this PR. An explicit eth_unsubscribe that overlaps the restore commit still leaves the restored subscription live on the node. So "Nothing is left to leak" doesn't hold on this path:

  1. Unsubscribe reads conn and upstreamID under the lock (:1002-1003), unlocks, and tells the node to drop the old id.
  2. Meanwhile the restore commits. The entry is still mapped, so this check passes, and the id, the connection and the client's mapping move to the new subscription.
  3. Unsubscribe re-locks. RemoveMapping finds the mapping, so this is the last client, and it tears down with tellNode=false (:1063). Nobody tells the node about the restored id.

This is the natural ordering in production.

  • The pool hands back the same connection, so the unsubscribe queues on the same rpcclient behind the restore's Subscribe.
  • A real node answers the stale id with a JSON-RPC error, and CallContext returns that as a message with err == nil.

Probe: hold the restore at the test hook, stall the node's writes, start Unsubscribe, then release. It prints methods=[eth_subscribe eth_subscribe eth_unsubscribe] nodeHasNew=true total=0, 3/3 here, on main and with #508.

Fix (pushing):

  • In Unsubscribe's second critical section, tell the node when a restore committed since the read (activeSub.upstreamID != upstreamID || activeSub.upstreamConnection != conn).
  • Add TestUnsubscribeDuringRestore_TellsTheNodeTheRestoredID.

// The subscription was torn down while the node was answering the re-subscribe:
// its last client left, or it was dropped, and the teardown released everything
// the old upstream subscription held. Nothing owns the one just created, so it
// ends here — locally, on the node, and in the connection's count — instead of
// living on as an upstream subscription nobody reads and a count that stops the
// pool from ever scaling the connection down (MAG-2547).
dwsm.lock.Unlock()
utils.LavaFormatInfo("DirectWS: subscription was torn down while being restored, discarding the restored upstream subscription",
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
utils.LogAttr("newUpstreamID", newUpstreamID),
)
newUpstreamSub.Unsubscribe()
dwsm.teardownUpstreamSubscription(conn, newUpstreamID, activeSub.subscribeMethod)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Low · exists on main. The MAG-3527 closing comment noted this, but no ticket was filed.

  • Cause: teardownUpstreamSubscription derives the method with getUnsubscribeMethod and always sends the id as a string. That is right only for eth_subscribe/subscribe with string ids.
  • Result on Solana: this discard sends unsubscribe, a method the node doesn't have, so the discarded subscription stays live there. With the Solana mock, the node saw [accountSubscribe accountSubscribe unsubscribe unsubscribe].
  • Why not here: the fix means storing the resolved unsubscribe method and the id encoding at subscribe time. That is a design call, so I'm not changing it in this PR; it is worth its own ticket.

activeSub.upstreamPool.NotifySubscriptionRemoved(conn)
return
}
oldUpstreamID := activeSub.upstreamID
oldConnection := activeSub.upstreamConnection // Keep reference to old connection for cleanup

Expand All @@ -1659,9 +1691,10 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect(
}
dwsm.lock.Unlock()

// Notify the pool that the OLD connection no longer has this subscription
// This decrements the subscription count on the stale connection
if oldConnection != nil && oldConnection != conn {
// Notify the pool that the OLD connection no longer has this subscription. This
// decrements the subscription count on the stale connection — also when the pool handed
// back the same connection object, which it incremented above for the restored one.
if oldConnection != nil {
activeSub.upstreamPool.NotifySubscriptionRemoved(oldConnection)
}

Expand Down Expand Up @@ -1772,13 +1805,22 @@ func (dwsm *DirectWSSubscriptionManager) forgetClientLocked(clientKey string) {
// local subscription ends and the pool may scale the connection down. The network call runs
// off the lock. Caller holds dwsm.lock.
func (dwsm *DirectWSSubscriptionManager) teardownSubscriptionLocked(hashedParams string, activeSub *directActiveSubscription, tellNode bool) {
// Everything the goroutine needs is read here, under the lock. A restore that is
// mid-flight rewrites upstreamSubscription, upstreamID and upstreamConnection under
// this same lock once the node answers, so reading them off the lock raced that
// write and could tear down whichever half of the restore it happened to see.
upstreamConn := activeSub.upstreamConnection
upstreamPool := activeSub.upstreamPool
upstreamSub := activeSub.upstreamSubscription
upstreamID := activeSub.upstreamID
subscribeMethod := activeSub.subscribeMethod
go func() {
if tellNode {
dwsm.teardownUpstreamSubscription(activeSub)
dwsm.teardownUpstreamSubscription(upstreamConn, upstreamID, subscribeMethod)
}
if upstreamSub != nil {
upstreamSub.Unsubscribe()
}
activeSub.upstreamSubscription.Unsubscribe()
activeSub.cancel()
activeSub.stopUpstreamListener()
if upstreamConn != nil {
Expand Down Expand Up @@ -1812,43 +1854,55 @@ const upstreamUnsubscribeTimeout = 10 * time.Second
//
// Best-effort by design: it also runs where the connection may already be gone
// (upstream disconnect, shutdown), so a failure is traced rather than warned.
// Must be called WITHOUT dwsm.lock held — it makes a network call.
func (dwsm *DirectWSSubscriptionManager) teardownUpstreamSubscription(activeSub *directActiveSubscription) {
if activeSub == nil || activeSub.upstreamConnection == nil {
// Must be called WITHOUT dwsm.lock held — it makes a network call. It takes the
// connection and id as values rather than reading them off the subscription, because
// a restore in flight rewrites those fields under dwsm.lock while this runs off it
// (MAG-2547): the caller snapshots them while it still holds the lock.
func (dwsm *DirectWSSubscriptionManager) teardownUpstreamSubscription(upstreamConn *UpstreamWSConnection, upstreamID, subscribeMethod string) {
if upstreamConn == nil {
return
}
client := activeSub.upstreamConnection.GetClient()
client := upstreamConn.GetClient()
if client == nil {
return
}

var params interface{}
if dwsm.apiInterface == "tendermintrpc" {
params = map[string]interface{}{"query": activeSub.upstreamID}
params = map[string]interface{}{"query": upstreamID}
} else {
params = []interface{}{activeSub.upstreamID}
params = []interface{}{upstreamID}
}

ctx, cancel := context.WithTimeout(context.Background(), upstreamUnsubscribeTimeout)
defer cancel()
if _, err := client.CallContext(ctx, json.RawMessage("1"),
getUnsubscribeMethod(activeSub.subscribeMethod), params, true, false); err != nil {
getUnsubscribeMethod(subscribeMethod), params, true, false); err != nil {
utils.LavaFormatTrace("DirectWS: upstream unsubscribe on disconnect failed",
utils.LogAttr("error", err),
utils.LogAttr("upstreamID", activeSub.upstreamID),
utils.LogAttr("upstreamID", upstreamID),
)
}
}

// cleanupSubscription cleans up a subscription
func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string) {
// cleanupSubscription cleans up a subscription, but only the one its caller owns. The map
// is keyed by params alone, so once owner has been torn down a new client can register a
// fresh subscription under the same key; a late caller (a listener ending, a restore
// failing) that cleaned up by key would then destroy that newer subscription.
func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string, owner *directActiveSubscription) {
dwsm.lock.Lock()
defer dwsm.lock.Unlock()

activeSub, found := dwsm.activeSubscriptions[hashedParams]
if !found {
return
}
if activeSub != owner {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Low · test gap. This guard fixes a real main bug, but no test pins it: with the guard removed, all 73 WS subscription tests on this branch still pass, the new one included.

The main bug it prevents:

  • A torn-down subscription's listener exits only after the teardown's node call returns, which can take up to upstreamUnsubscribeTimeout (10 s).
  • Its deferred cleanup by key then destroys any successor registered under the same params in the meantime: it closes the successor client's channel, unsubscribes it on the node, and decrements the counter to 0.

Tests I'm adding:

  • TestListenerEndingLate_LeavesASuccessorAlone: fails on main, and here without the guard.
  • TestRestoreAfterTeardown_LeavesASuccessorAlone: fails on main.

utils.LavaFormatTrace("DirectWS: subscription under this key is not the one being cleaned up, leaving it",
utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)),
)
return
}

// Release every client: closes their reply channels and wakes their watchers.
for clientKey := range activeSub.connectedClients {
Expand All @@ -1869,8 +1923,10 @@ func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string
// pool notify goes after it so the connection is still live when it is used.
upstreamConn := activeSub.upstreamConnection
upstreamPool := activeSub.upstreamPool
upstreamID := activeSub.upstreamID
subscribeMethod := activeSub.subscribeMethod
go func() {
dwsm.teardownUpstreamSubscription(activeSub)
dwsm.teardownUpstreamSubscription(upstreamConn, upstreamID, subscribeMethod)
if upstreamConn != nil {
upstreamPool.NotifySubscriptionRemoved(upstreamConn)
}
Expand Down
29 changes: 29 additions & 0 deletions protocol/rpcsmartrouter/direct_ws_subscription_manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ type mockSubscriptionServer struct {
// subscriptionSeq makes successive ids distinct, so a test can tell a restored
// subscription from the one it replaced.
subscriptionSeq int

// conns holds every server-side socket still open, so a test can drop them all and
// watch the router restore its subscriptions.
conns []*websocket.Conn
}

// safeWriteMessage serializes WS writes: handleWS (response) and the spawned sendSubscriptionMessages
Expand Down Expand Up @@ -90,6 +94,9 @@ func (ms *mockSubscriptionServer) handleWS(w http.ResponseWriter, r *http.Reques
return
}
defer conn.Close()
ms.lock.Lock()
ms.conns = append(ms.conns, conn)
ms.lock.Unlock()

for {
_, message, err := conn.ReadMessage()
Expand Down Expand Up @@ -277,6 +284,28 @@ func (ms *mockSubscriptionServer) sendSubscriptionMessages(conn *websocket.Conn,
}
}

// dropConnections closes every server-side socket, which is what an upstream node
// restarting looks like from the router: every subscription on those sockets errors and
// the router re-subscribes over a fresh connection.
func (ms *mockSubscriptionServer) dropConnections() {
ms.lock.Lock()
conns := ms.conns
ms.conns = nil
ms.lock.Unlock()
for _, conn := range conns {
conn.Close()
}
}

// liveSubscriptions is how many subscriptions the node still believes exist: created and
// not yet unsubscribed. A router that leaks an upstream subscription leaves this above zero
// after every client has gone.
func (ms *mockSubscriptionServer) liveSubscriptions() int {
ms.lock.RLock()
defer ms.lock.RUnlock()
return len(ms.subscriptions)
}

func (ms *mockSubscriptionServer) Close() {
ms.lock.Lock()
defer ms.lock.Unlock()
Expand Down
21 changes: 21 additions & 0 deletions protocol/rpcsmartrouter/direct_ws_subscription_release_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,27 @@ func (h *releaseHarness) attached(clientKey string) bool {
return false
}

// pooledSubscriptionCount sums what every pooled upstream connection believes it carries.
// The pool scales a connection down only when this reaches zero, so a count left behind
// keeps a connection open for the life of the process.
func (h *releaseHarness) pooledSubscriptionCount() int32 {
h.manager.lock.RLock()
pools := make([]*UpstreamWSPool, 0, len(h.manager.upstreamPools))
for _, pool := range h.manager.upstreamPools {
pools = append(pools, pool)
}
h.manager.lock.RUnlock()
var n int32
for _, pool := range pools {
pool.lock.RLock()
for _, conn := range pool.connections {
n += conn.SubscriptionCount()
}
pool.lock.RUnlock()
}
return n
}

func (h *releaseHarness) activeSubscriptionCount() int {
h.manager.lock.RLock()
defer h.manager.lock.RUnlock()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package rpcsmartrouter

import (
"testing"
"time"

"github.com/stretchr/testify/require"
)

// TestRestoreOnTheSameConnection_KeepsOneCount: the pool hands the restore the same
// connection object it lost the socket on. The restore counts the subscription on it again,
// so it must also uncount the old one, or the connection counts one subscription forever and
// the pool can never scale it down.
func TestRestoreOnTheSameConnection_KeepsOneCount(t *testing.T) {
h := newReleaseHarness(t)
sub := h.subscribe("ws-1")
t.Cleanup(sub.disconnect)
h.manager.lock.RLock()
var activeSub *directActiveSubscription
for _, s := range h.manager.activeSubscriptions {
activeSub = s
}
oldID := activeSub.upstreamID
h.manager.lock.RUnlock()
require.Equal(t, int32(1), h.pooledSubscriptionCount())

h.srv.dropConnections()
require.Eventually(t, func() bool {
h.manager.lock.RLock()
defer h.manager.lock.RUnlock()
return activeSub.upstreamID != oldID
}, 10*time.Second, 10*time.Millisecond, "the restore commits")
// The restore counted the subscription on the connection before it committed; the old
// connection is uncounted just after.
require.Eventually(t, func() bool { return h.pooledSubscriptionCount() == 1 },
5*time.Second, 10*time.Millisecond, "one live subscription, one count")

sub.disconnect()
require.Eventually(t, func() bool { return h.activeSubscriptionCount() == 0 }, 5*time.Second, 10*time.Millisecond)
require.Eventually(t, func() bool { return h.pooledSubscriptionCount() == 0 }, 5*time.Second, 10*time.Millisecond,
"a pooled connection still counts a subscription, so the pool can never scale it down")
}
Loading