-
Notifications
You must be signed in to change notification settings - Fork 239
feat(rpc): gate WSS requests and bound active subscriptions #3883
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ | |
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "errors" | ||
| "io" | ||
| "net/http" | ||
|
|
@@ -20,13 +21,25 @@ | |
| maxConns = 2048 // TODO: an arbitrary default number, should be revisited after monitoring | ||
| ) | ||
|
|
||
| var serverBusyResponse = func() []byte { | ||
| b, err := json.Marshal(&response{ | ||
| Version: "2.0", | ||
| Error: &Error{Code: InternalError, Message: ErrServerBusy.Error()}, | ||
| }) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| return b | ||
| }() | ||
|
|
||
| type Websocket struct { | ||
| rpc *Server | ||
| logger log.StructuredLogger | ||
| connParams *WebsocketConnParams | ||
| listener NewRequestListener | ||
| shutdown <-chan struct{} | ||
| requestTimeout time.Duration | ||
| gate *Gate | ||
|
|
||
| // Add connection tracking | ||
| connSem *semaphore.Weighted | ||
|
|
@@ -62,6 +75,11 @@ | |
| return ws | ||
| } | ||
|
|
||
| func (ws *Websocket) WithGate(g *Gate) *Websocket { | ||
| ws.gate = g | ||
| return ws | ||
| } | ||
|
|
||
| // WithListener registers a NewRequestListener | ||
| func (ws *Websocket) WithListener(listener NewRequestListener) *Websocket { | ||
| ws.listener = listener | ||
|
|
@@ -116,7 +134,7 @@ | |
| break | ||
| } | ||
| ws.listener.OnNewRequest("any") | ||
| if err = ws.rpc.HandleReadWriter(wsc.ctx, ws.requestTimeout, wsc); err != nil { | ||
| if err = ws.handleMessage(wsc); err != nil { | ||
| break | ||
| } | ||
| // From websocket docs: "Read to EOF otherwise connection will hang." | ||
|
|
@@ -146,6 +164,27 @@ | |
| } | ||
| } | ||
|
|
||
| func (ws *Websocket) handleMessage(wsc *websocketConn) error { | ||
| if ws.gate != nil { | ||
| acquireCtx := wsc.ctx | ||
| if ws.requestTimeout > 0 { | ||
| var cancel context.CancelFunc | ||
| acquireCtx, cancel = context.WithTimeout(acquireCtx, ws.requestTimeout) | ||
| defer cancel() | ||
| } | ||
| if err := ws.gate.Acquire(acquireCtx); err != nil { | ||
| if errors.Is(err, context.Canceled) { | ||
| return err | ||
| } | ||
| _, writeErr := wsc.Write(serverBusyResponse) | ||
| return writeErr | ||
| } | ||
|
Comment on lines
+175
to
+181
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. important — rejections here are silent, and Two things:
Also note the metrics side: |
||
| defer ws.gate.Release() | ||
| } | ||
|
|
||
| return ws.rpc.HandleReadWriter(wsc.ctx, ws.requestTimeout, wsc) | ||
| } | ||
|
Comment on lines
+167
to
+186
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. important — a websocket request can now consume
The HTTP path doesn't have this: Suggest building the deadline once here and threading it into func (ws *Websocket) handleMessage(wsc *websocketConn) error {
ctx := wsc.ctx
if ws.requestTimeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, ws.requestTimeout)
defer cancel()
}
if ws.gate != nil {
if err := ws.gate.Acquire(ctx); err != nil { ... }
defer ws.gate.Release()
}
return ws.rpc.HandleReadWriter(ctx, 0, wsc)
}Careful with that shape though: |
||
|
|
||
| type WebsocketConnParams struct { | ||
| // Maximum message size allowed. | ||
| ReadLimit int64 | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -296,3 +296,59 @@ func TestWebsocketConnectionLimit(t *testing.T) { | |||||||
| require.Equal(t, http.StatusSwitchingProtocols, resp4.StatusCode) | ||||||||
| require.NoError(t, conn4.Close(websocket.StatusNormalClosure, "")) | ||||||||
| } | ||||||||
|
|
||||||||
| func TestWebsocketGateRejectsWhenBusy(t *testing.T) { | ||||||||
| started := make(chan struct{}) | ||||||||
| release := make(chan struct{}) | ||||||||
| block := jsonrpc.Method{ | ||||||||
| Name: "test_block", | ||||||||
| Handler: func(ctx context.Context) (int, *jsonrpc.Error) { | ||||||||
| close(started) | ||||||||
| <-release | ||||||||
| return 0, nil | ||||||||
| }, | ||||||||
| } | ||||||||
| echo := jsonrpc.Method{ | ||||||||
| Name: "test_echo", | ||||||||
| Params: []jsonrpc.Parameter{{Name: "msg"}}, | ||||||||
| Handler: func(msg string) (string, *jsonrpc.Error) { return msg, nil }, | ||||||||
| } | ||||||||
|
|
||||||||
| rpc := jsonrpc.NewServer(1, log.NewNopZapLogger()) | ||||||||
| require.NoError(t, rpc.RegisterMethods(block, echo)) | ||||||||
| gate := jsonrpc.NewGate(1, 0) | ||||||||
| ws := jsonrpc.NewWebsocket(rpc, nil, log.NewNopZapLogger()).WithGate(gate) | ||||||||
| srv := httptest.NewServer(ws) | ||||||||
| t.Cleanup(srv.Close) | ||||||||
|
|
||||||||
| connA, respA, err := websocket.Dial(t.Context(), srv.URL, nil) //nolint:bodyclose // lib closes it | ||||||||
| require.NoError(t, err) | ||||||||
| require.Equal(t, http.StatusSwitchingProtocols, respA.StatusCode) | ||||||||
| defer connA.Close(websocket.StatusNormalClosure, "") | ||||||||
| require.NoError(t, connA.Write(t.Context(), websocket.MessageText, | ||||||||
| []byte(`{"jsonrpc":"2.0","method":"test_block","params":[],"id":1}`))) | ||||||||
| <-started | ||||||||
|
|
||||||||
| connB, respB, err := websocket.Dial(t.Context(), srv.URL, nil) //nolint:bodyclose // lib closes it | ||||||||
| require.NoError(t, err) | ||||||||
| require.Equal(t, http.StatusSwitchingProtocols, respB.StatusCode) | ||||||||
| defer connB.Close(websocket.StatusNormalClosure, "") | ||||||||
| require.NoError(t, connB.Write(t.Context(), websocket.MessageText, | ||||||||
| []byte(`{"jsonrpc":"2.0","method":"test_echo","params":["hi"],"id":2}`))) | ||||||||
| _, got, err := connB.Read(t.Context()) | ||||||||
| require.NoError(t, err) | ||||||||
| assert.Equal(t, | ||||||||
| `{"jsonrpc":"2.0","error":{"code":-32603,"message":"server busy"},"id":null}`, | ||||||||
| string(got)) | ||||||||
|
|
||||||||
| close(release) | ||||||||
| _, _, err = connA.Read(t.Context()) | ||||||||
| require.NoError(t, err) | ||||||||
| require.Eventually(t, func() bool { return gate.Running() == 0 }, time.Second, 5*time.Millisecond) | ||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit —
func (g *Gate) Release() {
<-g.sem // Running() drops to 0 here
g.decreaseActiveReq() // activeRequests drops to 0 only here
}
It's a nanosecond-wide window and there's a websocket round trip after it, so it will almost never fire in CI, but it's a real ordering gap rather than a theoretical one. Cheap fix — assert both counters are drained:
Suggested change
|
||||||||
|
|
||||||||
| require.NoError(t, connB.Write(t.Context(), websocket.MessageText, | ||||||||
| []byte(`{"jsonrpc":"2.0","method":"test_echo","params":["hi"],"id":3}`))) | ||||||||
| _, got, err = connB.Read(t.Context()) | ||||||||
| require.NoError(t, err) | ||||||||
| assert.Equal(t, `{"jsonrpc":"2.0","result":"hi","id":3}`, string(got)) | ||||||||
| } | ||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -543,6 +543,13 @@ | |
| "/rpc" + pathV09: jsonrpcServerV09, | ||
| "/rpc" + pathV08: jsonrpcServerV08, | ||
| } | ||
| var rpcGate *jsonrpc.Gate | ||
| if cfg.RPCMaxConcurrentRequests > 0 { | ||
| rpcGate = jsonrpc.NewGate(cfg.RPCMaxConcurrentRequests, uint64(cfg.RPCMaxRequestQueue)) | ||
| if cfg.Metrics { | ||
| makeHTTPGateMetrics(rpcGate) | ||
| } | ||
| } | ||
|
Comment on lines
+546
to
+552
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit — hoisting the gate here so HTTP and WS share it is the right call, but it now runs unconditionally: with |
||
| if cfg.HTTP { | ||
| readinessHandlers := NewReadinessHandlers(chain, synchronizer, cfg.ReadinessBlockTolerance) | ||
| httpHandlers := map[string]http.HandlerFunc{ | ||
|
|
@@ -561,8 +568,7 @@ | |
| cfg.Metrics, | ||
| cfg.RPCCorsEnable, | ||
| cfg.RPCRequestTimeout, | ||
| cfg.RPCMaxConcurrentRequests, | ||
| cfg.RPCMaxRequestQueue, | ||
| rpcGate, | ||
| ), | ||
| ) | ||
| } | ||
|
|
@@ -577,6 +583,7 @@ | |
| cfg.Metrics, | ||
| cfg.RPCCorsEnable, | ||
| cfg.RPCRequestTimeout, | ||
| rpcGate, | ||
| ), | ||
| ) | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,7 @@ import ( | |
| "github.com/NethermindEth/juno/utils/log" | ||
| "github.com/NethermindEth/juno/vm" | ||
| "golang.org/x/sync/errgroup" | ||
| "golang.org/x/sync/semaphore" | ||
| ) | ||
|
|
||
| const ( | ||
|
|
@@ -44,9 +45,13 @@ type Handler struct { | |
| func New(bcReader blockchain.Reader, syncReader sync.Reader, virtualMachine vm.VM, version string, | ||
| logger log.Logger, network *networks.Network, | ||
| ) *Handler { | ||
| handlerv8 := rpcv8.New(bcReader, syncReader, virtualMachine, logger) | ||
| handlerv9 := rpcv9.New(bcReader, syncReader, virtualMachine, logger) | ||
| handlerv10 := rpcv10.New(bcReader, syncReader, virtualMachine, logger) | ||
| subscriptionLimiter := semaphore.NewWeighted(rpccore.DefaultMaxSubscriptions) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. important — the limit is process-wide, not per-connection The PR description says "Bound the number of active subscriptions per connection", but this is a single Combined with Suggest tracking a per-connection counter (keyed on the |
||
| handlerv8 := rpcv8.New(bcReader, syncReader, virtualMachine, logger). | ||
| WithSubscriptionLimiter(subscriptionLimiter) | ||
| handlerv9 := rpcv9.New(bcReader, syncReader, virtualMachine, logger). | ||
| WithSubscriptionLimiter(subscriptionLimiter) | ||
| handlerv10 := rpcv10.New(bcReader, syncReader, virtualMachine, logger). | ||
| WithSubscriptionLimiter(subscriptionLimiter) | ||
|
|
||
| return &Handler{ | ||
| rpcv8Handler: handlerv8, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,8 @@ const ( | |
| MaxBlocksBack = 1024 | ||
| EntrypointNotFoundFelt string = "0x454e545259504f494e545f4e4f545f464f554e44" | ||
| ErrEPSNotFound = "Entry point EntryPointSelector(%s) not found in contract." | ||
|
|
||
| DefaultMaxSubscriptions int64 = 2048 | ||
|
Comment on lines
+22
to
+23
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit — hardcoded, unlike the sibling request gate
Also
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Valid
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In fact, if we make them configurable, it should be a pair of flags: 1 for connections and 1 for subscriptions. |
||
| ) | ||
|
|
||
| //go:generate mockgen -destination=../mocks/mock_gateway_handler.go -package=mocks github.com/NethermindEth/juno/rpc/rpccore Gateway | ||
|
|
@@ -95,4 +97,5 @@ var ( | |
|
|
||
| // These errors can be only be returned by Juno-specific methods. | ||
| ErrSubscriptionNotFound = &jsonrpc.Error{Code: 100, Message: "Subscription not found"} | ||
| ErrTooManySubscriptions = &jsonrpc.Error{Code: 101, Message: "Too many subscriptions"} | ||
| ) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -54,6 +54,9 @@ func (h *Handler) subscribe( | |
| wsConn jsonrpc.Conn, | ||
| subscriber subscriber, | ||
| ) (SubscriptionID, *jsonrpc.Error) { | ||
| if h.subscriptionLimiter != nil && !h.subscriptionLimiter.TryAcquire(1) { | ||
| return "", rpccore.ErrTooManySubscriptions | ||
| } | ||
|
Comment on lines
+57
to
+59
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit — the limit is checked after the expensive work, and the pair is triplicated The acquire/release pairing itself is correct: there is no early return between Two smaller things:
|
||
| id := h.idgen() | ||
| //nolint:gosec // G118: cancel called in unsubscribe() | ||
| subscriptionCtx, subscriptionCtxCancel := context.WithCancel(wsConn.Context()) | ||
|
|
@@ -75,6 +78,9 @@ func (h *Handler) subscribe( | |
| ) | ||
|
|
||
| sub.wg.Go(func() { | ||
| if h.subscriptionLimiter != nil { | ||
| defer h.subscriptionLimiter.Release(1) | ||
| } | ||
| defer func() { | ||
| h.unsubscribe(sub, id) | ||
| unsubscribeFeedSubscription(reorgSub) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -32,6 +32,7 @@ import ( | |
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| "go.uber.org/mock/gomock" | ||
| "golang.org/x/sync/semaphore" | ||
| ) | ||
|
|
||
| // mustNewChain builds a pre_confirmed ChainReader from statically valid test | ||
|
|
@@ -1217,6 +1218,49 @@ func TestSubscribeNewHeads(t *testing.T) { | |
| assertNextHead(t, conn, subID, &adaptedHeader3) | ||
| } | ||
|
|
||
| func TestSubscribeNewHeadsRespectsLimit(t *testing.T) { | ||
| logger := log.NewNopZapLogger() | ||
| client := feeder.NewTestClient(t, &networks.Sepolia) | ||
| block1, commitments1, stateUpdate1 := GetTestBlockWithCommitments(t, client, 56377) | ||
| adaptedHeader := AdaptBlockHeader(block1.Header, commitments1, stateUpdate1.StateDiff) | ||
|
|
||
| mockCtrl := gomock.NewController(t) | ||
| t.Cleanup(mockCtrl.Finish) | ||
| mockChain := mocks.NewMockReader(mockCtrl) | ||
|
|
||
| handler := New(mockChain, nil, nil, logger). | ||
| WithSubscriptionLimiter(semaphore.NewWeighted(1)) | ||
|
|
||
| mockChain.EXPECT().Height().Return(block1.Number, nil).Times(3) | ||
| mockChain.EXPECT().BlockHeaderByNumber(block1.Number).Return(block1.Header, nil).Times(2) | ||
| mockChain.EXPECT().BlockCommitmentsByNumber(block1.Number).Return(commitments1, nil).Times(2) | ||
| mockChain.EXPECT().StateUpdateByNumber(block1.Number).Return(stateUpdate1, nil).Times(2) | ||
|
|
||
| blockIDLatest := BlockIDLatest() | ||
|
|
||
| subID1, conn1 := createTestNewHeadsWebsocket(t, handler, (*SubscriptionBlockID)(&blockIDLatest)) | ||
| assertNextHead(t, conn1, subID1, &adaptedHeader) | ||
|
|
||
| serverConn, clientConn := net.Pipe() | ||
| t.Cleanup(func() { | ||
| require.NoError(t, serverConn.Close()) | ||
| require.NoError(t, clientConn.Close()) | ||
| }) | ||
| rejConn := &fakeConn{Conn: clientConn, w: serverConn} | ||
| rejCtx := context.WithValue(t.Context(), jsonrpc.ConnKey{}, rejConn) | ||
| id, rpcErr := handler.SubscribeNewHeads(rejCtx, nil) | ||
| assert.Zero(t, id) | ||
| assert.Equal(t, rpccore.ErrTooManySubscriptions, rpcErr) | ||
|
|
||
| unsubCtx := context.WithValue(t.Context(), jsonrpc.ConnKey{}, conn1) | ||
| ok, rpcErr := handler.Unsubscribe(unsubCtx, string(subID1)) | ||
| require.Nil(t, rpcErr) | ||
| require.True(t, ok) | ||
|
|
||
| subID3, conn3 := createTestNewHeadsWebsocket(t, handler, (*SubscriptionBlockID)(&blockIDLatest)) | ||
| assertNextHead(t, conn3, subID3, &adaptedHeader) | ||
| } | ||
|
Comment on lines
+1221
to
+1262
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit — good test, two gaps Solid: it exercises reject-at-limit and slot-reclaimed-after-unsubscribe, and it correctly relies on
Also |
||
|
|
||
| func TestSubscribeNewHeadsHistorical(t *testing.T) { | ||
| logger := log.NewNopZapLogger() | ||
| client := feeder.NewTestClient(t, &networks.Sepolia) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
important — the busy response is always
"id":null, so pipelining clients can't correlate itBecause the gate is checked before the frame is parsed (
HandleReadWriteris what readswsc.r), this canned response carries no request id. Websocket clients routinely have several requests in flight on one connection and match responses byid; anid:nullerror is unattributable, so a client typically either drops it or fails the wrong request. The HTTP path doesn't have this problem — it answers with a 503 at the transport level, which is unambiguous.Related: a client that sent a notification (no
id) now receives an unsolicited response frame, which is a JSON-RPC violation.Options: read the frame into a bounded buffer and decode just the
idbefore gating (cost: one extra copy bounded byReadLimit), or close the connection with a websocket close code instead of a fake response. If the current behaviour is a deliberate trade-off, please note it in a comment onserverBusyResponse.Separately,
InternalError(-32603) is a poor fit semantically — nothing failed internally, the server is shedding load. A dedicated code (orErrServerBusymapped to one) would let clients implement backoff instead of treating it as a bug.