Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions protocol/lavasession/direct_rpc_connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -449,9 +449,14 @@ type GRPCDirectRPCConnection struct {

// resolver resolves and caches method descriptors for this connection: the
// single-flight registry, the whole-service fill and the reflection-timeout
// budget (MAG-2860) live there, shared with the subscription stream pool. It
// borrows pooled clients through borrowConn. Set at construction like
// descriptorsCache, which it shares with the warm-up sweep.
// budget (MAG-2860) live there. It borrows pooled clients through borrowConn. Set
// at construction like descriptorsCache, which it shares with the warm-up sweep.
//
// It is this connection's own, NOT shared with the subscription stream pool: that
// pool owns a second GRPCDescriptorResolver over a second cache, because a resolver
// can only run reflection on a channel its owner holds and the two hold different
// channels (MAG-4397). The rules are shared; the instance and the cache are not, so
// a cold service is resolved once on the unary path and once on the stream path.
resolver *GRPCDescriptorResolver

// Reflection snapshot state, guarded by snapshotMu and usable at its zero value.
Expand Down
37 changes: 16 additions & 21 deletions protocol/lavasession/grpc_descriptor_resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,24 @@ type descriptorResolution struct {
}

// GRPCDescriptorResolver resolves a node's gRPC method descriptors through its
// configured descriptor source (reflection, file, or hybrid) and caches them for
// the life of one upstream connection. It is the one resolver behind both the unary
// GRPCDirectRPCConnection and the subscription stream pool's connection, so the
// MAG-2860 rules hold on every path: a cold service is resolved once however many
// callers ask for it (claimServiceResolution), one lookup caches the whole service
// (cacheServiceMethods), and the lookup runs on grpc-config's reflection-timeout
// rooted at the connection's lifetime rather than at the caller's deadline
// (lookupService). The stream pool used to carry a copy of the pre-MAG-2860 shape:
// one reflection client per call, one method cached, the caller's context as the
// only bound, so N cold subscriptions opened N reflection streams against a slow
// node with no deadline of their own (MAG-4278 #18).
// configured descriptor source (reflection, file, or hybrid) and caches them for the
// life of its owner. The MAG-2860 rules live here, so every owner gets them: a cold
// service is resolved once however many callers ask for it (claimServiceResolution),
// one lookup caches the whole service (cacheServiceMethods), and the lookup runs on
// grpc-config's reflection-timeout rooted at the owner's lifetime rather than at the
// caller's deadline (lookupService). The stream path used to carry a copy of the
// pre-MAG-2860 shape: one reflection client per call, one method cached, the caller's
// context as the only bound, so N cold subscriptions opened N reflection streams
// against a slow node with no deadline of their own (MAG-4278 #18).
//
// There are TWO instances per endpoint, not one, and they do not share a cache: the
// unary GRPCDirectRPCConnection owns one, and the subscription stream pool owns the
// other for all of its channels (MAG-4397). They are separate because their borrow
// sources are — a resolver can only run reflection on a channel its owner holds.
//
// borrow hands out a client to run reflection on and the way to give it back: the
// unary connection borrows from its pooled connector, the stream connection lends
// its single channel.
// unary connection borrows from its pooled connector, the stream pool lends any
// healthy channel of its endpoint.
type GRPCDescriptorResolver struct {
grpcConfig *common.GrpcConfig
url string
Expand Down Expand Up @@ -75,14 +78,6 @@ func NewGRPCDescriptorResolver(
}
}

// Cached returns the descriptor cached under service.method, or nil.
func (r *GRPCDescriptorResolver) Cached(fullMethodName string) *desc.MethodDescriptor {
if methodDesc, found, _ := r.cache.Load(fullMethodName); found {
return methodDesc
}
return nil
}

// MethodDescriptor retrieves the method descriptor from cache, or waits on a
// lookup against the node's configured descriptor source (reflection, file, or
// hybrid).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,7 @@ func newManagerAgainstUpstreams(t *testing.T, addrs ...string) *DirectGRPCSubscr
require.NoError(t, err)
conn, err := pool.GetConnectionForStream(ctx)
require.NoError(t, err)
conn.descriptorsCache.Store("grpc.reflection.v1.ServerReflection.ServerReflectionInfo", streamingMethodDescriptor(t))
conn.descriptors.cache.Store("grpc.reflection.v1.ServerReflection.ServerReflectionInfo", streamingMethodDescriptor(t))
}
return manager
}
Expand Down Expand Up @@ -668,7 +668,7 @@ func dialRestoreListener(t *testing.T, manager *DirectGRPCSubscriptionManager) *

// The fake upstream serves any method, so the feed reuses the streaming descriptor the
// harness has: a server-streaming method over the reflection request and response types.
connectionForUpstream(t, manager).descriptorsCache.Store("test.v1.Feed.Subscribe", streamingMethodDescriptor(t))
connectionForUpstream(t, manager).descriptors.cache.Store("test.v1.Feed.Subscribe", streamingMethodDescriptor(t))

endpoint := &lavasession.RPCEndpoint{NetworkAddress: "127.0.0.1:0", ChainID: "SUI", ApiInterface: spectypes.APIInterfaceGrpc}
sender := &restoreRelaySender{rpcss: &RPCSmartRouterServer{chainParser: parser, listenEndpoint: endpoint}, manager: manager}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ func TestGRPCStartSubscription_ReleasesClientStateOnFailure(t *testing.T) {

// Same method, described as unary — createNewSubscription refuses it after it has
// connected and pinned the client to the endpoint.
connectionForUpstream(t, manager).descriptorsCache.Store(
connectionForUpstream(t, manager).descriptors.cache.Store(
"grpc.reflection.v1.ServerReflection.ServerReflectionInfo", unaryMethodDescriptor(t))

message := newGrpcSubscriptionMessage(t)
Expand Down Expand Up @@ -350,7 +350,7 @@ func TestGRPCSubscription_RecordsUpstreamForRelayEnd(t *testing.T) {
// Described as unary, the method is refused after the manager reached the endpoint.
refusing := newManagerAgainstUpstream(t, upstream.addr)
defer refusing.Stop()
connectionForUpstream(t, refusing).descriptorsCache.Store("grpc.reflection.v1.ServerReflection.ServerReflectionInfo", unaryMethodDescriptor(t))
connectionForUpstream(t, refusing).descriptors.cache.Store("grpc.reflection.v1.ServerReflection.ServerReflectionInfo", unaryMethodDescriptor(t))
failed := chainlib.WithSubscriptionUpstream(context.Background())
_, _, err = refusing.StartSubscription(failed, newGrpcSubscriptionMessage(t), "dapp", "3.3.3.3", "conn-3", nil)
require.Error(t, err)
Expand Down Expand Up @@ -524,7 +524,7 @@ func startFakeStreamingUpstream(t *testing.T) *fakeStreamingUpstream {
}

// newManagerAgainstUpstream points a manager at the fake upstream and seeds the method
// descriptor into the connection cache, so GetMethodDescriptor never calls reflection.
// descriptor into the endpoint's shared cache, so GetMethodDescriptor never calls reflection.
func newManagerAgainstUpstream(t *testing.T, addr string) *DirectGRPCSubscriptionManager {
t.Helper()

Expand All @@ -540,7 +540,7 @@ func newManagerAgainstUpstream(t *testing.T, addr string) *DirectGRPCSubscriptio
require.NoError(t, err)
conn, err := pool.GetConnectionForStream(ctx)
require.NoError(t, err)
conn.descriptorsCache.Store("grpc.reflection.v1.ServerReflection.ServerReflectionInfo", streamingMethodDescriptor(t))
conn.descriptors.cache.Store("grpc.reflection.v1.ServerReflection.ServerReflectionInfo", streamingMethodDescriptor(t))

return manager
}
Expand Down
Loading