Skip to content
Merged
38 changes: 30 additions & 8 deletions protocol/chainlib/chainproxy/rpcclient/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,14 +104,15 @@ type Client struct {

// for dispatch
close chan struct{}
closing chan struct{} // closed when client is quitting
didClose chan struct{} // closed when client quits
reconnected chan ServerCodec // where write/reconnect sends the new connection
readOp chan readOp // read messages
readErr chan error // errors from read
reqInit chan *requestOp // register response IDs, takes write lock
reqSent chan error // signals write completion, releases write lock
reqTimeout chan *requestOp // removes response IDs when call timeout expires
closing chan struct{} // closed when client is quitting
didClose chan struct{} // closed when client quits
reconnected chan ServerCodec // where write/reconnect sends the new connection
readOp chan readOp // read messages
readErr chan error // errors from read
reqInit chan *requestOp // register response IDs, takes write lock
reqSent chan error // signals write completion, releases write lock
reqTimeout chan *requestOp // removes response IDs when call timeout expires
subDone chan *ClientSubscription // drops a subscription's dispatch entry after Unsubscribe
}

type reconnectFunc func(ctx context.Context) (ServerCodec, error)
Expand Down Expand Up @@ -214,6 +215,7 @@ func initClient(conn ServerCodec, idgen func() ID, services *serviceRegistry) *C
reqInit: make(chan *requestOp),
reqSent: make(chan error, 1),
reqTimeout: make(chan *requestOp),
subDone: make(chan *ClientSubscription),
}
if !isHTTP {
go c.dispatch(conn)
Expand Down Expand Up @@ -724,10 +726,30 @@ func (c *Client) dispatch(codec ServerCodec) {

case op := <-c.reqTimeout:
conn.handler.removeRequestOp(op)

case sub := <-c.subDone:
conn.handler.forgetClientSub(sub)
}
}
}

// forgetSubscription asks the dispatch loop to drop the handler's entry for sub. Until
// now that entry lived until the connection closed, one per subscription ever torn
// down on a pooled connection (MAG-3722). It runs on its own goroutine because
// dispatch may be busy delivering to another subscription whose consumer is waiting
// on a lock the Unsubscribe caller holds.
func (c *Client) forgetSubscription(sub *ClientSubscription) {
if c == nil || c.isHTTP {
return
}
go func() {
select {
case c.subDone <- sub:
case <-c.closing:
}
}()
}

// drainRead drops read messages until an error occurs.
func (c *Client) drainRead() {
for {
Expand Down
15 changes: 15 additions & 0 deletions protocol/chainlib/chainproxy/rpcclient/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -529,6 +529,21 @@ func (h *handler) runMethod(ctx context.Context, msg *JsonrpcMessage, callb *cal
return msg.response(result)
}

// forgetClientSub drops sub from the dispatch table once its forwarding loop has been
// told to quit. Runs on the dispatch goroutine, which is the only owner of clientSubs.
// A push for the id that still arrives then reaches deliverSubscriptionPush as an
// unmatched frame: without a request id it is dropped, the fate it met while the entry
// existed; with one it goes to the call path, which answers the node with a JSON-RPC
// error where the entry used to swallow it.
func (h *handler) forgetClientSub(sub *ClientSubscription) {
if sub == nil {
return
}
if current, ok := h.clientSubs[sub.subid]; ok && current == sub {
delete(h.clientSubs, sub.subid)
}
}

// unsubscribe is the callback function for all *_unsubscribe calls.
func (h *handler) unsubscribe(ctx context.Context, id ID) (bool, error) {
h.subLock.Lock()
Expand Down
1 change: 1 addition & 0 deletions protocol/chainlib/chainproxy/rpcclient/subscription.go
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,7 @@ func (sub *ClientSubscription) Unsubscribe() {
// Unsubscribe already requested (or subscription already closed).
}
close(sub.err)
sub.client.forgetSubscription(sub)
})
}

Expand Down
141 changes: 141 additions & 0 deletions protocol/chainlib/chainproxy/rpcclient/subscription_forget_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package rpcclient

import (
"context"
"encoding/json"
"reflect"
"sync"
"testing"
"time"

"github.com/stretchr/testify/require"
)

// TestForgetClientSub_DropsOnlyThatSubscription pins the identity check: a later
// subscription registered under a reused id must not be removed by a stale unsubscribe.
func TestForgetClientSub_DropsOnlyThatSubscription(t *testing.T) {
h := newTestHandler()
ch := make(chan *JsonrpcMessage)
old := newClientSubscription(nil, "eth", reflect.ValueOf(ch))
old.subid = "0xabc"
h.clientSubs[old.subid] = old

h.forgetClientSub(old)
require.NotContains(t, h.clientSubs, "0xabc")

replacement := newClientSubscription(nil, "eth", reflect.ValueOf(ch))
replacement.subid = "0xabc"
h.clientSubs[replacement.subid] = replacement
h.forgetClientSub(old) // stale
require.Same(t, replacement, h.clientSubs["0xabc"], "a stale unsubscribe must not evict the live subscription")

h.forgetClientSub(nil) // must not panic
}

// scriptedCodec is an in-memory ServerCodec: the test feeds it frames to read and
// records what the client writes.
type scriptedCodec struct {
incoming chan []*JsonrpcMessage
closedCh chan interface{}
once sync.Once

mu sync.Mutex
written []*JsonrpcMessage
}

func newScriptedCodec() *scriptedCodec {
return &scriptedCodec{incoming: make(chan []*JsonrpcMessage, 16), closedCh: make(chan interface{})}
}

func (c *scriptedCodec) peerInfo() PeerInfo { return PeerInfo{} }
func (c *scriptedCodec) readBatch() ([]*JsonrpcMessage, bool, error) {
select {
case msgs := <-c.incoming:
return msgs, false, nil
case <-c.closedCh:
return nil, false, ErrClientQuit
}
}
func (c *scriptedCodec) close() { c.once.Do(func() { close(c.closedCh) }) }
func (c *scriptedCodec) closed() <-chan interface{} { return c.closedCh }
func (c *scriptedCodec) remoteAddr() string { return "scripted" }
func (c *scriptedCodec) writeJSON(_ context.Context, v interface{}) error {
c.mu.Lock()
defer c.mu.Unlock()
if msg, ok := v.(*JsonrpcMessage); ok {
c.written = append(c.written, msg)
}
return nil
}

func (c *scriptedCodec) writtenCount() int {
c.mu.Lock()
defer c.mu.Unlock()
return len(c.written)
}

func (c *scriptedCodec) lastWritten() *JsonrpcMessage {
c.mu.Lock()
defer c.mu.Unlock()
if len(c.written) == 0 {
return nil
}
return c.written[len(c.written)-1]
}

// TestUnsubscribe_ForgetsDispatchEntry is the regression for MAG-3722's rpcclient leak:
// the dispatch table kept an entry for every subscription ever unsubscribed until the
// connection closed, and on a pooled upstream connection that is the life of the process.
//
// The entry is observed through behaviour rather than a hook. A push that names the
// subscription and carries a request id is claimed silently while the entry exists, and
// handed to the call path once it is gone, where the handler answers it. So the answer
// appearing on the wire is the entry disappearing.
func TestUnsubscribe_ForgetsDispatchEntry(t *testing.T) {
codec := newScriptedCodec()
client, err := newClient(context.Background(), func(context.Context) (ServerCodec, error) { return codec, nil })
require.NoError(t, err)
defer client.Close()

// Answer the subscribe request as the node would.
go func() {
require.Eventually(t, func() bool { return codec.writtenCount() == 1 }, 5*time.Second, 5*time.Millisecond)
req := codec.lastWritten()
codec.incoming <- []*JsonrpcMessage{{Version: Vsn, ID: req.ID, Result: json.RawMessage(`"0xabc"`)}}
}()
ch := make(chan *JsonrpcMessage, 8)
sub, _, err := client.Subscribe(context.Background(), nil, "eth_subscribe", ch, []interface{}{"newHeads"})
require.NoError(t, err)
require.NotNil(t, sub)

// A Substrate-shaped push: the handler routes it by the params envelope, and that
// route is the one that hands an unmatched id-bearing frame back to the call path.
// (An eth_subscription frame is claimed by its method name whether or not the
// table still has the entry, so it cannot show the entry going away.)
push := func() {
codec.incoming <- []*JsonrpcMessage{{
Version: Vsn,
ID: json.RawMessage(`99`),
Method: "chain_newHead",
Params: json.RawMessage(`{"subscription":"0xabc","result":{"number":"0x1"}}`),
}}
}

push()
select {
case <-ch:
case <-time.After(5 * time.Second):
t.Fatal("a push must reach the subscriber while it is live")
}
require.Equal(t, 1, codec.writtenCount(), "a claimed push is not answered")

sub.Unsubscribe()

require.Eventually(t, func() bool {
push()
return codec.writtenCount() > 1
}, 5*time.Second, 10*time.Millisecond,
"once the entry is forgotten an id-bearing push is handed to the call path and answered")
answer := codec.lastWritten()
require.NotNil(t, answer.Error, "the call path answers an unknown method with an error")
}
133 changes: 84 additions & 49 deletions protocol/chainlib/consumer_websocket_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,75 @@ func (cwm *ConsumerWebsocketManager) handleRateLimitReached(inpData []byte) ([]b
return bytesRateLimitError, nil
}

// webSocketMsgWithType is one frame queued for the connection's writer goroutine.
type webSocketMsgWithType struct {
messageType int
msg []byte
}

// websocketFrameWriter is the slice of the connection the writer goroutine touches.
// *websocket.Conn satisfies it; tests hand in a fake whose writes fail on demand.
type websocketFrameWriter interface {
WriteMessage(messageType int, data []byte) error
SetReadDeadline(t time.Time) error
}

// runWebsocketWriter is the connection's single writer. Single-writer discipline
// guarantees WriteMessage is never called concurrently, which the underlying
// gorilla/fasthttp websocket library does not allow.
//
// It serves regular frames and the server-shutdown close frame, and closes writerDone
// on its way out. A failed write ends it, and that used to leave every later
// enqueueWebsocketFrame caller blocked forever: the read loop's own error path sends a
// frame before it can return, so the handler never returned, and fasthttp kept the
// 128 KiB read buffer, the handler goroutine and the per-IP limiter slot for the life
// of the process (MAG-3722). Now a failed write also sets an immediate read deadline so
// the read loop wakes and the handler tears the connection down.
func runWebsocketWriter(ctx, webSocketCtx context.Context, conn websocketFrameWriter, frames <-chan webSocketMsgWithType, writerDone chan<- struct{}) {
defer close(writerDone)
for {
select {
case <-ctx.Done():
// Server-wide shutdown — send a CloseGoingAway (1001) frame so the client can
// distinguish an intentional shutdown from a crash, then unblock the read loop by
// setting an immediate read deadline rather than Close()-ing the conn here.
//
// Close() from this goroutine raced gofiber's handler cleanup: it unblocks ReadMessage,
// the handler returns, and gofiber recycles the Conn (fasthttp pooling) WHILE this
// goroutine is still inside Close() — a use-after-recycle data race. SetReadDeadline
// returns immediately (the read loop breaks on the resulting error) and lets the handler,
// the conn's sole owner, do the single Close on return.
_ = conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseGoingAway, "server shutting down"))
_ = conn.SetReadDeadline(time.Now())
return
case <-webSocketCtx.Done():
utils.LavaFormatTrace("websocket's context cancelled", utils.LogAttr("GUID", webSocketCtx))
return
case msg, ok := <-frames:
if !ok {
return
}
if err := conn.WriteMessage(msg.messageType, msg.msg); err != nil {
utils.LavaFormatTrace("error writing msg to the websocket", utils.LogAttr("err", err))
_ = conn.SetReadDeadline(time.Now())
return
}
}
}
}

// enqueueWebsocketFrame hands a frame to the writer goroutine, but gives up as soon as
// ctx (server shutdown) or webSocketCtx (per-connection cleanup) cancels or the writer
// is gone: a plain send would park the caller for good.
func enqueueWebsocketFrame(ctx, webSocketCtx context.Context, writerDone <-chan struct{}, frames chan<- webSocketMsgWithType, msg webSocketMsgWithType) {
select {
case frames <- msg:
case <-ctx.Done():
case <-webSocketCtx.Done():
case <-writerDone:
}
}

func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) {
// adding metrics for how many active connections we have.
cwm.rpcConsumerLogs.SetWebSocketConnectionActive(cwm.chainId, cwm.apiInterface, true)
Expand All @@ -116,12 +185,9 @@ func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) {
err error
)

type webSocketMsgWithType struct {
messageType int
msg []byte
}

websocketConnWriteChan := make(chan webSocketMsgWithType)
// Closed by the writer goroutine on its way out.
writerDone := make(chan struct{})

websocketConn := cwm.websocketConn
logger := cwm.rpcConsumerLogs
Expand All @@ -132,57 +198,26 @@ func (cwm *ConsumerWebsocketManager) ListenToMessages(ctx context.Context) {
webSocketCtx = utils.WithUniqueIdentifier(webSocketCtx, guid)
utils.LavaFormatDebug("consumer websocket manager started", utils.LogAttr("GUID", webSocketCtx))
defer func() {
// The connection is over. Release every subscription this client still holds
// right here, synchronously: reply channels close and the forwarder goroutines
// below end now, not whenever a cancelled context happens to be noticed. dappID
// and the remote address are per-connection, so this key matches every
// subscribe the loop registered.
if cwm.wsSubscriptionManager != nil {
dappID, _ := websocketConn.Locals(ProjectIDHeader).(string)
if err := cwm.wsSubscriptionManager.UnsubscribeAll(webSocketCtx, dappID, websocketConn.RemoteAddr().String(), cwm.WebsocketConnectionUID, nil); err != nil {
utils.LavaFormatDebug("error releasing subscriptions on websocket close", utils.LogAttr("err", err), utils.LogAttr("GUID", webSocketCtx))
}
}
cancelWebSocketCtx() // In case there's a problem make sure to cancel the connection
utils.LavaFormatDebug("consumer websocket manager stopped", utils.LogAttr("GUID", webSocketCtx))
}()

// sendWS enqueues a frame for the writer goroutine, but unblocks if either
// ctx (server shutdown) or webSocketCtx (per-conn cleanup) cancels first —
// the writer exits on those signals, so a plain send would deadlock the
// caller and leave the connection goroutine stuck (preventing wsWG.Done()
// from firing during graceful shutdown).
sendWS := func(msg webSocketMsgWithType) {
select {
case websocketConnWriteChan <- msg:
case <-ctx.Done():
case <-webSocketCtx.Done():
}
enqueueWebsocketFrame(ctx, webSocketCtx, writerDone, websocketConnWriteChan, msg)
}

// Single writer goroutine: serves both regular messages and the server-shutdown
// close frame. Single-writer discipline guarantees we never call WriteMessage
// concurrently — the underlying gorilla/fasthttp websocket library is not safe
// for concurrent writes.
go func() {
for {
select {
case <-ctx.Done():
// Server-wide shutdown — send a CloseGoingAway (1001) frame so the client can
// distinguish an intentional shutdown from a crash, then unblock the read loop by
// setting an immediate read deadline rather than Close()-ing the conn here.
//
// Close() from this goroutine raced gofiber's handler cleanup: it unblocks ReadMessage,
// the handler returns, and gofiber recycles the Conn (fasthttp pooling) WHILE this
// goroutine is still inside Close() — a use-after-recycle data race. SetReadDeadline
// returns immediately (the read loop breaks on the resulting error) and lets the handler,
// the conn's sole owner, do the single Close on return.
_ = cwm.websocketConn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseGoingAway, "server shutting down"))
_ = cwm.websocketConn.SetReadDeadline(time.Now())
return
case <-webSocketCtx.Done():
utils.LavaFormatTrace("websocket's context cancelled", utils.LogAttr("GUID", webSocketCtx))
return
case msg, ok := <-websocketConnWriteChan:
if !ok {
return
}
if err := cwm.websocketConn.WriteMessage(msg.messageType, msg.msg); err != nil {
utils.LavaFormatTrace("error writing msg to the websocket")
return
}
}
}
}()
go runWebsocketWriter(ctx, webSocketCtx, cwm.websocketConn, websocketConnWriteChan, writerDone)

// set up a routine to check for rate limits or idle time
idleFor := atomic.Int64{}
Expand Down
Loading
Loading