Repository navigation
fix(rpcsmartrouter): let a WebSocket teardown win over a restore in flight (MAG-2547) #509
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
bd6cbbb
87b1a8d
d09e446
56d842e
35f0e1a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -155,6 +155,11 @@ type DirectWSSubscriptionManager struct { | |
| // Pending subscriptions for deduplication | ||
| pendingSubscriptions map[string]*pendingSubscriptionsBroadcastManager | ||
|
|
||
| // beforeRestoreCommit, when set, runs in handleUpstreamDisconnect after the node has | ||
| // answered the re-subscribe and before the result is committed under dwsm.lock. Nil | ||
| // outside tests; tests use it to hold a restore at exactly that point. | ||
| beforeRestoreCommit func() | ||
|
|
||
| // Upstream connection pools keyed by endpoint URL | ||
| upstreamPools map[string]*UpstreamWSPool | ||
|
|
||
|
|
@@ -1053,9 +1058,12 @@ func (dwsm *DirectWSSubscriptionManager) Unsubscribe( | |
| // lastClient is the mapper's view after this client's mapping went, read under the | ||
| // lock: a client that joined during the unlocked upstream call above registered its | ||
| // mapping first, so it keeps the subscription. The node was already told, so the | ||
| // teardown skips the wire call. | ||
| // teardown skips the wire call — unless a restore committed while that call was in | ||
| // flight: the node was told the id read before it, and the restored subscription, | ||
| // with its new id and connection, was never told (MAG-2547). | ||
| if lastClient { | ||
| dwsm.teardownSubscriptionLocked(hashedParams, activeSub, false) | ||
| restoredSinceRead := activeSub.upstreamID != upstreamID || activeSub.upstreamConnection != conn | ||
| dwsm.teardownSubscriptionLocked(hashedParams, activeSub, restoredSinceRead) | ||
| } | ||
|
|
||
| return nodeResp, nil | ||
|
|
@@ -1498,7 +1506,7 @@ func (dwsm *DirectWSSubscriptionManager) listenForUpstreamMessages( | |
| reconnectInFlight := false | ||
| defer func() { | ||
| if !reconnectInFlight { | ||
| dwsm.cleanupSubscription(hashedParams) | ||
| dwsm.cleanupSubscription(hashedParams, activeSub) | ||
| } | ||
| }() | ||
|
|
||
|
|
@@ -1596,9 +1604,12 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( | |
| } | ||
| defer activeSub.restoring.Store(false) | ||
|
|
||
| dwsm.lock.RLock() | ||
| connectedClients := len(activeSub.clientRouterIDs) | ||
| dwsm.lock.RUnlock() | ||
| utils.LavaFormatInfo("DirectWS: handling upstream disconnect", | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| utils.LogAttr("connectedClients", len(activeSub.clientRouterIDs)), | ||
| utils.LogAttr("connectedClients", connectedClients), | ||
| ) | ||
|
|
||
| // Try to reconnect the WebSocket connection | ||
|
|
@@ -1607,7 +1618,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( | |
| utils.LavaFormatError("DirectWS: reconnection failed, cleaning up subscription", err, | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| ) | ||
| dwsm.cleanupSubscription(hashedParams) | ||
| dwsm.cleanupSubscription(hashedParams, activeSub) | ||
| return | ||
| } | ||
|
|
||
|
|
@@ -1622,7 +1633,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( | |
| utils.LavaFormatError("DirectWS: failed to get connection after reconnect", err, | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| ) | ||
| dwsm.cleanupSubscription(hashedParams) | ||
| dwsm.cleanupSubscription(hashedParams, activeSub) | ||
| return | ||
| } | ||
|
|
||
|
|
@@ -1632,7 +1643,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( | |
| utils.LavaFormatError("DirectWS: failed to restore subscription", err, | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| ) | ||
| dwsm.cleanupSubscription(hashedParams) | ||
| dwsm.cleanupSubscription(hashedParams, activeSub) | ||
| return | ||
| } | ||
|
|
||
|
|
@@ -1642,8 +1653,29 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( | |
| // Increment subscription count on the NEW connection (matching initial creation behavior) | ||
| conn.IncrementSubscriptions() | ||
|
|
||
| if dwsm.beforeRestoreCommit != nil { | ||
| dwsm.beforeRestoreCommit() | ||
| } | ||
|
|
||
| // Update the ID mapping, message channel, and connection bookkeeping | ||
| dwsm.lock.Lock() | ||
| if dwsm.activeSubscriptions[hashedParams] != activeSub { | ||
| // The subscription was torn down while the node was answering the re-subscribe: | ||
| // its last client left, or it was dropped, and the teardown released everything | ||
| // the old upstream subscription held. Nothing owns the one just created, so it | ||
| // ends here — locally, on the node, and in the connection's count — instead of | ||
| // living on as an upstream subscription nobody reads and a count that stops the | ||
| // pool from ever scaling the connection down (MAG-2547). | ||
| dwsm.lock.Unlock() | ||
| utils.LavaFormatInfo("DirectWS: subscription was torn down while being restored, discarding the restored upstream subscription", | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| utils.LogAttr("newUpstreamID", newUpstreamID), | ||
| ) | ||
| newUpstreamSub.Unsubscribe() | ||
| dwsm.teardownUpstreamSubscription(conn, newUpstreamID, activeSub.subscribeMethod) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Low · exists on main. The MAG-3527 closing comment noted this, but no ticket was filed.
|
||
| activeSub.upstreamPool.NotifySubscriptionRemoved(conn) | ||
| return | ||
| } | ||
| oldUpstreamID := activeSub.upstreamID | ||
| oldConnection := activeSub.upstreamConnection // Keep reference to old connection for cleanup | ||
|
|
||
|
|
@@ -1659,9 +1691,10 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( | |
| } | ||
| dwsm.lock.Unlock() | ||
|
|
||
| // Notify the pool that the OLD connection no longer has this subscription | ||
| // This decrements the subscription count on the stale connection | ||
| if oldConnection != nil && oldConnection != conn { | ||
| // Notify the pool that the OLD connection no longer has this subscription. This | ||
| // decrements the subscription count on the stale connection — also when the pool handed | ||
| // back the same connection object, which it incremented above for the restored one. | ||
| if oldConnection != nil { | ||
| activeSub.upstreamPool.NotifySubscriptionRemoved(oldConnection) | ||
| } | ||
|
|
||
|
|
@@ -1772,13 +1805,22 @@ func (dwsm *DirectWSSubscriptionManager) forgetClientLocked(clientKey string) { | |
| // local subscription ends and the pool may scale the connection down. The network call runs | ||
| // off the lock. Caller holds dwsm.lock. | ||
| func (dwsm *DirectWSSubscriptionManager) teardownSubscriptionLocked(hashedParams string, activeSub *directActiveSubscription, tellNode bool) { | ||
| // Everything the goroutine needs is read here, under the lock. A restore that is | ||
| // mid-flight rewrites upstreamSubscription, upstreamID and upstreamConnection under | ||
| // this same lock once the node answers, so reading them off the lock raced that | ||
| // write and could tear down whichever half of the restore it happened to see. | ||
| upstreamConn := activeSub.upstreamConnection | ||
| upstreamPool := activeSub.upstreamPool | ||
| upstreamSub := activeSub.upstreamSubscription | ||
| upstreamID := activeSub.upstreamID | ||
| subscribeMethod := activeSub.subscribeMethod | ||
| go func() { | ||
| if tellNode { | ||
| dwsm.teardownUpstreamSubscription(activeSub) | ||
| dwsm.teardownUpstreamSubscription(upstreamConn, upstreamID, subscribeMethod) | ||
| } | ||
| if upstreamSub != nil { | ||
| upstreamSub.Unsubscribe() | ||
| } | ||
| activeSub.upstreamSubscription.Unsubscribe() | ||
| activeSub.cancel() | ||
| activeSub.stopUpstreamListener() | ||
| if upstreamConn != nil { | ||
|
|
@@ -1812,43 +1854,55 @@ const upstreamUnsubscribeTimeout = 10 * time.Second | |
| // | ||
| // Best-effort by design: it also runs where the connection may already be gone | ||
| // (upstream disconnect, shutdown), so a failure is traced rather than warned. | ||
| // Must be called WITHOUT dwsm.lock held — it makes a network call. | ||
| func (dwsm *DirectWSSubscriptionManager) teardownUpstreamSubscription(activeSub *directActiveSubscription) { | ||
| if activeSub == nil || activeSub.upstreamConnection == nil { | ||
| // Must be called WITHOUT dwsm.lock held — it makes a network call. It takes the | ||
| // connection and id as values rather than reading them off the subscription, because | ||
| // a restore in flight rewrites those fields under dwsm.lock while this runs off it | ||
| // (MAG-2547): the caller snapshots them while it still holds the lock. | ||
| func (dwsm *DirectWSSubscriptionManager) teardownUpstreamSubscription(upstreamConn *UpstreamWSConnection, upstreamID, subscribeMethod string) { | ||
| if upstreamConn == nil { | ||
| return | ||
| } | ||
| client := activeSub.upstreamConnection.GetClient() | ||
| client := upstreamConn.GetClient() | ||
| if client == nil { | ||
| return | ||
| } | ||
|
|
||
| var params interface{} | ||
| if dwsm.apiInterface == "tendermintrpc" { | ||
| params = map[string]interface{}{"query": activeSub.upstreamID} | ||
| params = map[string]interface{}{"query": upstreamID} | ||
| } else { | ||
| params = []interface{}{activeSub.upstreamID} | ||
| params = []interface{}{upstreamID} | ||
| } | ||
|
|
||
| ctx, cancel := context.WithTimeout(context.Background(), upstreamUnsubscribeTimeout) | ||
| defer cancel() | ||
| if _, err := client.CallContext(ctx, json.RawMessage("1"), | ||
| getUnsubscribeMethod(activeSub.subscribeMethod), params, true, false); err != nil { | ||
| getUnsubscribeMethod(subscribeMethod), params, true, false); err != nil { | ||
| utils.LavaFormatTrace("DirectWS: upstream unsubscribe on disconnect failed", | ||
| utils.LogAttr("error", err), | ||
| utils.LogAttr("upstreamID", activeSub.upstreamID), | ||
| utils.LogAttr("upstreamID", upstreamID), | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| // cleanupSubscription cleans up a subscription | ||
| func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string) { | ||
| // cleanupSubscription cleans up a subscription, but only the one its caller owns. The map | ||
| // is keyed by params alone, so once owner has been torn down a new client can register a | ||
| // fresh subscription under the same key; a late caller (a listener ending, a restore | ||
| // failing) that cleaned up by key would then destroy that newer subscription. | ||
| func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string, owner *directActiveSubscription) { | ||
| dwsm.lock.Lock() | ||
| defer dwsm.lock.Unlock() | ||
|
|
||
| activeSub, found := dwsm.activeSubscriptions[hashedParams] | ||
| if !found { | ||
| return | ||
| } | ||
| if activeSub != owner { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Low · test gap. This guard fixes a real main bug, but no test pins it: with the guard removed, all 73 WS subscription tests on this branch still pass, the new one included. The main bug it prevents:
Tests I'm adding:
|
||
| utils.LavaFormatTrace("DirectWS: subscription under this key is not the one being cleaned up, leaving it", | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| ) | ||
| return | ||
| } | ||
|
|
||
| // Release every client: closes their reply channels and wakes their watchers. | ||
| for clientKey := range activeSub.connectedClients { | ||
|
|
@@ -1869,8 +1923,10 @@ func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string | |
| // pool notify goes after it so the connection is still live when it is used. | ||
| upstreamConn := activeSub.upstreamConnection | ||
| upstreamPool := activeSub.upstreamPool | ||
| upstreamID := activeSub.upstreamID | ||
| subscribeMethod := activeSub.subscribeMethod | ||
| go func() { | ||
| dwsm.teardownUpstreamSubscription(activeSub) | ||
| dwsm.teardownUpstreamSubscription(upstreamConn, upstreamID, subscribeMethod) | ||
| if upstreamConn != nil { | ||
| upstreamPool.NotifySubscriptionRemoved(upstreamConn) | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package rpcsmartrouter | ||
|
|
||
| import ( | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // TestRestoreOnTheSameConnection_KeepsOneCount: the pool hands the restore the same | ||
| // connection object it lost the socket on. The restore counts the subscription on it again, | ||
| // so it must also uncount the old one, or the connection counts one subscription forever and | ||
| // the pool can never scale it down. | ||
| func TestRestoreOnTheSameConnection_KeepsOneCount(t *testing.T) { | ||
| h := newReleaseHarness(t) | ||
| sub := h.subscribe("ws-1") | ||
| t.Cleanup(sub.disconnect) | ||
| h.manager.lock.RLock() | ||
| var activeSub *directActiveSubscription | ||
| for _, s := range h.manager.activeSubscriptions { | ||
| activeSub = s | ||
| } | ||
| oldID := activeSub.upstreamID | ||
| h.manager.lock.RUnlock() | ||
| require.Equal(t, int32(1), h.pooledSubscriptionCount()) | ||
|
|
||
| h.srv.dropConnections() | ||
| require.Eventually(t, func() bool { | ||
| h.manager.lock.RLock() | ||
| defer h.manager.lock.RUnlock() | ||
| return activeSub.upstreamID != oldID | ||
| }, 10*time.Second, 10*time.Millisecond, "the restore commits") | ||
| // The restore counted the subscription on the connection before it committed; the old | ||
| // connection is uncounted just after. | ||
| require.Eventually(t, func() bool { return h.pooledSubscriptionCount() == 1 }, | ||
| 5*time.Second, 10*time.Millisecond, "one live subscription, one count") | ||
|
|
||
| sub.disconnect() | ||
| require.Eventually(t, func() bool { return h.activeSubscriptionCount() == 0 }, 5*time.Second, 10*time.Millisecond) | ||
| require.Eventually(t, func() bool { return h.pooledSubscriptionCount() == 0 }, 5*time.Second, 10*time.Millisecond, | ||
| "a pooled connection still counts a subscription, so the pool can never scale it down") | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Medium · exists on main, same race family as this PR. An explicit
eth_unsubscribethat overlaps the restore commit still leaves the restored subscription live on the node. So "Nothing is left to leak" doesn't hold on this path:UnsubscribereadsconnandupstreamIDunder the lock (:1002-1003), unlocks, and tells the node to drop the old id.Unsubscribere-locks.RemoveMappingfinds the mapping, so this is the last client, and it tears down withtellNode=false(:1063). Nobody tells the node about the restored id.This is the natural ordering in production.
Subscribe.CallContextreturns that as a message witherr == nil.Probe: hold the restore at the test hook, stall the node's writes, start
Unsubscribe, then release. It printsmethods=[eth_subscribe eth_subscribe eth_unsubscribe] nodeHasNew=true total=0, 3/3 here, on main and with #508.Fix (pushing):
Unsubscribe's second critical section, tell the node when a restore committed since the read (activeSub.upstreamID != upstreamID || activeSub.upstreamConnection != conn).TestUnsubscribeDuringRestore_TellsTheNodeTheRestoredID.