Repository navigation
fix(rpcsmartrouter): let a WebSocket teardown win over a restore in flight (MAG-2547) #509
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
bd6cbbb
87b1a8d
d09e446
56d842e
35f0e1a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -155,6 +155,11 @@ type DirectWSSubscriptionManager struct { | |
| // Pending subscriptions for deduplication | ||
| pendingSubscriptions map[string]*pendingSubscriptionsBroadcastManager | ||
|
|
||
| // beforeRestoreCommit, when set, runs in handleUpstreamDisconnect after the node has | ||
| // answered the re-subscribe and before the result is committed under dwsm.lock. Nil | ||
| // outside tests; tests use it to hold a restore at exactly that point. | ||
| beforeRestoreCommit func() | ||
|
|
||
| // Upstream connection pools keyed by endpoint URL | ||
| upstreamPools map[string]*UpstreamWSPool | ||
|
|
||
|
|
@@ -1481,7 +1486,7 @@ func (dwsm *DirectWSSubscriptionManager) listenForUpstreamMessages( | |
| reconnectInFlight := false | ||
| defer func() { | ||
| if !reconnectInFlight { | ||
| dwsm.cleanupSubscription(hashedParams) | ||
| dwsm.cleanupSubscription(hashedParams, activeSub) | ||
| } | ||
| }() | ||
|
|
||
|
|
@@ -1579,9 +1584,12 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( | |
| } | ||
| defer activeSub.restoring.Store(false) | ||
|
|
||
| dwsm.lock.RLock() | ||
| connectedClients := len(activeSub.clientRouterIDs) | ||
| dwsm.lock.RUnlock() | ||
| utils.LavaFormatInfo("DirectWS: handling upstream disconnect", | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| utils.LogAttr("connectedClients", len(activeSub.clientRouterIDs)), | ||
| utils.LogAttr("connectedClients", connectedClients), | ||
| ) | ||
|
|
||
| // Try to reconnect the WebSocket connection | ||
|
|
@@ -1590,7 +1598,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( | |
| utils.LavaFormatError("DirectWS: reconnection failed, cleaning up subscription", err, | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| ) | ||
| dwsm.cleanupSubscription(hashedParams) | ||
| dwsm.cleanupSubscription(hashedParams, activeSub) | ||
| return | ||
| } | ||
|
|
||
|
|
@@ -1605,7 +1613,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( | |
| utils.LavaFormatError("DirectWS: failed to get connection after reconnect", err, | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| ) | ||
| dwsm.cleanupSubscription(hashedParams) | ||
| dwsm.cleanupSubscription(hashedParams, activeSub) | ||
| return | ||
| } | ||
|
|
||
|
|
@@ -1615,7 +1623,7 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( | |
| utils.LavaFormatError("DirectWS: failed to restore subscription", err, | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| ) | ||
| dwsm.cleanupSubscription(hashedParams) | ||
| dwsm.cleanupSubscription(hashedParams, activeSub) | ||
| return | ||
| } | ||
|
|
||
|
|
@@ -1625,8 +1633,29 @@ func (dwsm *DirectWSSubscriptionManager) handleUpstreamDisconnect( | |
| // Increment subscription count on the NEW connection (matching initial creation behavior) | ||
| conn.IncrementSubscriptions() | ||
|
|
||
| if dwsm.beforeRestoreCommit != nil { | ||
| dwsm.beforeRestoreCommit() | ||
| } | ||
|
|
||
| // Update the ID mapping, message channel, and connection bookkeeping | ||
| dwsm.lock.Lock() | ||
| if dwsm.activeSubscriptions[hashedParams] != activeSub { | ||
| // The subscription was torn down while the node was answering the re-subscribe: | ||
| // its last client left, or it was dropped, and the teardown released everything | ||
| // the old upstream subscription held. Nothing owns the one just created, so it | ||
| // ends here — locally, on the node, and in the connection's count — instead of | ||
| // living on as an upstream subscription nobody reads and a count that stops the | ||
| // pool from ever scaling the connection down (MAG-2547). | ||
| dwsm.lock.Unlock() | ||
| utils.LavaFormatInfo("DirectWS: subscription was torn down while being restored, discarding the restored upstream subscription", | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| utils.LogAttr("newUpstreamID", newUpstreamID), | ||
| ) | ||
| newUpstreamSub.Unsubscribe() | ||
| dwsm.teardownUpstreamSubscription(conn, newUpstreamID, activeSub.subscribeMethod) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Low · exists on main. The MAG-3527 closing comment noted this, but no ticket was filed.
|
||
| activeSub.upstreamPool.NotifySubscriptionRemoved(conn) | ||
| return | ||
| } | ||
| oldUpstreamID := activeSub.upstreamID | ||
| oldConnection := activeSub.upstreamConnection // Keep reference to old connection for cleanup | ||
|
|
||
|
|
@@ -1755,13 +1784,22 @@ func (dwsm *DirectWSSubscriptionManager) forgetClientLocked(clientKey string) { | |
| // local subscription ends and the pool may scale the connection down. The network call runs | ||
| // off the lock. Caller holds dwsm.lock. | ||
| func (dwsm *DirectWSSubscriptionManager) teardownSubscriptionLocked(hashedParams string, activeSub *directActiveSubscription, tellNode bool) { | ||
| // Everything the goroutine needs is read here, under the lock. A restore that is | ||
| // mid-flight rewrites upstreamSubscription, upstreamID and upstreamConnection under | ||
| // this same lock once the node answers, so reading them off the lock raced that | ||
| // write and could tear down whichever half of the restore it happened to see. | ||
| upstreamConn := activeSub.upstreamConnection | ||
| upstreamPool := activeSub.upstreamPool | ||
| upstreamSub := activeSub.upstreamSubscription | ||
| upstreamID := activeSub.upstreamID | ||
| subscribeMethod := activeSub.subscribeMethod | ||
| go func() { | ||
| if tellNode { | ||
| dwsm.teardownUpstreamSubscription(activeSub) | ||
| dwsm.teardownUpstreamSubscription(upstreamConn, upstreamID, subscribeMethod) | ||
| } | ||
| if upstreamSub != nil { | ||
| upstreamSub.Unsubscribe() | ||
| } | ||
| activeSub.upstreamSubscription.Unsubscribe() | ||
| activeSub.cancel() | ||
| activeSub.stopUpstreamListener() | ||
| if upstreamConn != nil { | ||
|
|
@@ -1795,43 +1833,55 @@ const upstreamUnsubscribeTimeout = 10 * time.Second | |
| // | ||
| // Best-effort by design: it also runs where the connection may already be gone | ||
| // (upstream disconnect, shutdown), so a failure is traced rather than warned. | ||
| // Must be called WITHOUT dwsm.lock held — it makes a network call. | ||
| func (dwsm *DirectWSSubscriptionManager) teardownUpstreamSubscription(activeSub *directActiveSubscription) { | ||
| if activeSub == nil || activeSub.upstreamConnection == nil { | ||
| // Must be called WITHOUT dwsm.lock held — it makes a network call. It takes the | ||
| // connection and id as values rather than reading them off the subscription, because | ||
| // a restore in flight rewrites those fields under dwsm.lock while this runs off it | ||
| // (MAG-2547): the caller snapshots them while it still holds the lock. | ||
| func (dwsm *DirectWSSubscriptionManager) teardownUpstreamSubscription(upstreamConn *UpstreamWSConnection, upstreamID, subscribeMethod string) { | ||
| if upstreamConn == nil { | ||
| return | ||
| } | ||
| client := activeSub.upstreamConnection.GetClient() | ||
| client := upstreamConn.GetClient() | ||
| if client == nil { | ||
| return | ||
| } | ||
|
|
||
| var params interface{} | ||
| if dwsm.apiInterface == "tendermintrpc" { | ||
| params = map[string]interface{}{"query": activeSub.upstreamID} | ||
| params = map[string]interface{}{"query": upstreamID} | ||
| } else { | ||
| params = []interface{}{activeSub.upstreamID} | ||
| params = []interface{}{upstreamID} | ||
| } | ||
|
|
||
| ctx, cancel := context.WithTimeout(context.Background(), upstreamUnsubscribeTimeout) | ||
| defer cancel() | ||
| if _, err := client.CallContext(ctx, json.RawMessage("1"), | ||
| getUnsubscribeMethod(activeSub.subscribeMethod), params, true, false); err != nil { | ||
| getUnsubscribeMethod(subscribeMethod), params, true, false); err != nil { | ||
| utils.LavaFormatTrace("DirectWS: upstream unsubscribe on disconnect failed", | ||
| utils.LogAttr("error", err), | ||
| utils.LogAttr("upstreamID", activeSub.upstreamID), | ||
| utils.LogAttr("upstreamID", upstreamID), | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| // cleanupSubscription cleans up a subscription | ||
| func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string) { | ||
| // cleanupSubscription cleans up a subscription, but only the one its caller owns. The map | ||
| // is keyed by params alone, so once owner has been torn down a new client can register a | ||
| // fresh subscription under the same key; a late caller (a listener ending, a restore | ||
| // failing) that cleaned up by key would then destroy that newer subscription. | ||
| func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string, owner *directActiveSubscription) { | ||
| dwsm.lock.Lock() | ||
| defer dwsm.lock.Unlock() | ||
|
|
||
| activeSub, found := dwsm.activeSubscriptions[hashedParams] | ||
| if !found { | ||
| return | ||
| } | ||
| if activeSub != owner { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Low · test gap. This guard fixes a real main bug, but no test pins it: with the guard removed, all 73 WS subscription tests on this branch still pass, the new one included. The main bug it prevents:
Tests I'm adding:
|
||
| utils.LavaFormatTrace("DirectWS: subscription under this key is not the one being cleaned up, leaving it", | ||
| utils.LogAttr("hashedParams", utils.ToHexString(hashedParams)), | ||
| ) | ||
| return | ||
| } | ||
|
|
||
| // Release every client: closes their reply channels and wakes their watchers. | ||
| for clientKey := range activeSub.connectedClients { | ||
|
|
@@ -1852,8 +1902,10 @@ func (dwsm *DirectWSSubscriptionManager) cleanupSubscription(hashedParams string | |
| // pool notify goes after it so the connection is still live when it is used. | ||
| upstreamConn := activeSub.upstreamConnection | ||
| upstreamPool := activeSub.upstreamPool | ||
| upstreamID := activeSub.upstreamID | ||
| subscribeMethod := activeSub.subscribeMethod | ||
| go func() { | ||
| dwsm.teardownUpstreamSubscription(activeSub) | ||
| dwsm.teardownUpstreamSubscription(upstreamConn, upstreamID, subscribeMethod) | ||
| if upstreamConn != nil { | ||
| upstreamPool.NotifySubscriptionRemoved(upstreamConn) | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| package rpcsmartrouter | ||
|
|
||
| import ( | ||
| "sync" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // TestRestoreAfterTeardown_LeavesNoUpstreamSubscription is MAG-2547: an upstream socket | ||
| // drops, the router re-subscribes, and the subscription's last client disconnects while | ||
| // that restore is still waiting on the node. The teardown runs first and releases the | ||
| // old upstream subscription; the restore must then notice its subscription is gone and | ||
| // end the one it just created — on the node, locally, and in the connection's count — | ||
| // rather than committing it onto an entry nothing holds any more. Before the fix the | ||
| // restored upstream subscription stayed live on the node with nobody reading it, the | ||
| // connection kept counting it so the pool could never scale it down, and the teardown | ||
| // goroutine read the subscription's upstream fields off the lock while the restore | ||
| // rewrote them under it. | ||
| func TestRestoreAfterTeardown_LeavesNoUpstreamSubscription(t *testing.T) { | ||
| h := newReleaseHarness(t) | ||
|
|
||
| reached := make(chan struct{}) | ||
| release := make(chan struct{}) | ||
| var reachedOnce sync.Once | ||
| h.manager.beforeRestoreCommit = func() { | ||
| reachedOnce.Do(func() { close(reached) }) | ||
| <-release | ||
| } | ||
|
|
||
| sub := h.subscribe("ws-1") | ||
| require.Equal(t, 1, h.srv.liveSubscriptions()) | ||
|
|
||
| // The upstream socket drops. The router re-subscribes on a fresh connection and is | ||
| // held just before it commits the result. | ||
| h.srv.dropConnections() | ||
| select { | ||
| case <-reached: | ||
| case <-time.After(10 * time.Second): | ||
| t.Fatal("the restore never re-subscribed on the node") | ||
| } | ||
| require.Equal(t, 2, h.upstreamCalls("eth_subscribe"), "the restore subscribed again on the node") | ||
|
|
||
| // The last client leaves while the restore is held, so the subscription is torn down | ||
| // underneath it. | ||
| sub.disconnect() | ||
| require.Eventually(t, func() bool { return h.activeSubscriptionCount() == 0 }, | ||
| 5*time.Second, 10*time.Millisecond, "the last client leaving tears the subscription down") | ||
|
|
||
| close(release) | ||
|
|
||
| require.Eventually(t, func() bool { return h.srv.liveSubscriptions() == 0 }, | ||
| 5*time.Second, 10*time.Millisecond, | ||
| "the restored upstream subscription was left running on the node with nobody reading it") | ||
| require.Eventually(t, func() bool { return h.pooledSubscriptionCount() == 0 }, | ||
| 5*time.Second, 10*time.Millisecond, | ||
| "a pooled connection still counts a subscription, so the pool can never scale it down") | ||
| require.Equal(t, int64(0), h.manager.totalSubscriptions.Load()) | ||
| require.Equal(t, 0, h.activeSubscriptionCount()) | ||
| require.Eventually(t, func() bool { return goroutinesIn("listenForUpstreamMessages") == 0 }, | ||
| 5*time.Second, 10*time.Millisecond, "no upstream listener may outlive the subscription") | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Medium · exists on main, same race family as this PR. An explicit
eth_unsubscribethat overlaps the restore commit still leaves the restored subscription live on the node. So "Nothing is left to leak" doesn't hold on this path:UnsubscribereadsconnandupstreamIDunder the lock (:1002-1003), unlocks, and tells the node to drop the old id.Unsubscribere-locks.RemoveMappingfinds the mapping, so this is the last client, and it tears down withtellNode=false(:1063). Nobody tells the node about the restored id.This is the natural ordering in production.
Subscribe.CallContextreturns that as a message witherr == nil.Probe: hold the restore at the test hook, stall the node's writes, start
Unsubscribe, then release. It printsmethods=[eth_subscribe eth_subscribe eth_unsubscribe] nodeHasNew=true total=0, 3/3 here, on main and with #508.Fix (pushing):
Unsubscribe's second critical section, tell the node when a restore committed since the read (activeSub.upstreamID != upstreamID || activeSub.upstreamConnection != conn).TestUnsubscribeDuringRestore_TellsTheNodeTheRestoredID.