Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 112 additions & 16 deletions protocol/rpcsmartrouter/failed_relay_reply_e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@ import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"

Expand Down Expand Up @@ -38,6 +40,11 @@ type failedReplyInterface struct {
body string
// jsonRpcShape: the router's own error is a JSON-RPC error object; otherwise {"error": "..."}.
jsonRpcShape bool
// headers are request headers sent on top of the ones every request here sends.
headers map[string]string
// upstreamTimeout is the node-url timeout of every upstream, which lengthens each attempt's
// own budget past the request's (NodeUrl.LowerContextTimeoutWithDuration).
upstreamTimeout time.Duration
}

func failedReplyInterfaces() []failedReplyInterface {
Expand Down Expand Up @@ -80,28 +87,36 @@ type failedReplyClientRead struct {
contentType string
guid string
nodeError string // lava-identified-node-error
header http.Header
}

// startFailedReplyRouter serves iface through a real listener and router whose only provider is
// upstreamURL, and returns a function that sends the interface's request and reads the reply.
func startFailedReplyRouter(t *testing.T, iface failedReplyInterface, upstreamURL string) func() failedReplyClientRead {
// failedReplyRelayTimeout is the attempt window every request here asks for in lava-relay-timeout.
const failedReplyRelayTimeout = 300 * time.Millisecond

// startFailedReplyRouter serves iface through a real listener and router whose providers are
// upstreamURLs, named upstream-1, upstream-2, … in that order, and returns a function that sends
// the interface's request and reads the reply.
func startFailedReplyRouter(t *testing.T, iface failedReplyInterface, upstreamURLs ...string) func() failedReplyClientRead {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)

parser := iface.parser(t)
nodeURL := common.NodeUrl{Url: upstreamURL}
connection, err := lavasession.NewDirectRPCConnection(ctx, nodeURL, 5, iface.apiInterface)
require.NoError(t, err)
t.Cleanup(func() { _ = connection.Close() })
provider := lavasession.NewConsumerSessionWithProvider("upstream", []*lavasession.Endpoint{{
NetworkAddress: upstreamURL, Enabled: true, DirectConnections: []lavasession.DirectRPCConnection{connection},
}}, 100000, 1, 1)
provider.StaticProvider = true
providers := map[uint64]*lavasession.ConsumerSessionsWithProvider{}
for i, upstreamURL := range upstreamURLs {
connection, err := lavasession.NewDirectRPCConnection(ctx, common.NodeUrl{Url: upstreamURL, Timeout: iface.upstreamTimeout}, 5, iface.apiInterface)
require.NoError(t, err)
t.Cleanup(func() { _ = connection.Close() })
provider := lavasession.NewConsumerSessionWithProvider(fmt.Sprintf("upstream-%d", i+1), []*lavasession.Endpoint{{
NetworkAddress: upstreamURL, Enabled: true, DirectConnections: []lavasession.DirectRPCConnection{connection},
}}, 100000, 1, 1)
provider.StaticProvider = true
providers[uint64(i)] = provider
}
sessionManager, rpcEndpoint := createTestSessionManager(iface.chainID, iface.apiInterface)
rpcEndpoint.NetworkAddress = "127.0.0.1:0"
rpcEndpoint.HealthCheckPath = common.DEFAULT_HEALTH_PATH
require.NoError(t, sessionManager.UpdateAllProviders(1, map[uint64]*lavasession.ConsumerSessionsWithProvider{0: provider}, nil))
require.NoError(t, sessionManager.UpdateAllProviders(1, providers, nil))
logs, err := metrics.NewRPCConsumerLogs(nil, nil, nil)
require.NoError(t, err)
server := &RPCSmartRouterServer{
Expand All @@ -128,7 +143,10 @@ func startFailedReplyRouter(t *testing.T, iface failedReplyInterface, upstreamUR
request, err := http.NewRequestWithContext(ctx, iface.method, "http://"+address+iface.path, body)
require.NoError(t, err)
request.Header.Set("Content-Type", "application/json")
request.Header.Set(common.RELAY_TIMEOUT_HEADER_NAME, "300ms")
request.Header.Set(common.RELAY_TIMEOUT_HEADER_NAME, failedReplyRelayTimeout.String())
for name, value := range iface.headers {
request.Header.Set(name, value)
}
response, err := http.DefaultClient.Do(request)
require.NoError(t, err)
defer response.Body.Close()
Expand All @@ -140,6 +158,7 @@ func startFailedReplyRouter(t *testing.T, iface failedReplyInterface, upstreamUR
contentType: response.Header.Get("Content-Type"),
guid: response.Header.Get(common.GUID_HEADER_NAME),
nodeError: response.Header.Get(common.LAVA_IDENTIFIED_NODE_ERROR_HEADER),
header: response.Header,
}
}
}
Expand Down Expand Up @@ -168,11 +187,15 @@ func refusingUpstream(t *testing.T) string {
return "http://" + address
}

// hangingUpstream accepts every request and never answers.
func hangingUpstream(t *testing.T) string {
// hangingUpstream accepts every request and never answers. arrivals, when not nil, counts the
// requests it received.
func hangingUpstream(t *testing.T, arrivals *atomic.Int32) string {
t.Helper()
release := make(chan struct{})
upstream := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
if arrivals != nil {
arrivals.Add(1)
}
select {
case <-r.Context().Done():
case <-release:
Expand Down Expand Up @@ -269,11 +292,17 @@ func TestFailedRelayReply_EndToEnd_MAG4481(t *testing.T) {
})

t.Run("hung upstream is the router's 504", func(t *testing.T) {
read := startFailedReplyRouter(t, iface, hangingUpstream(t))()
read := startFailedReplyRouter(t, iface, hangingUpstream(t, nil))()
require.Equal(t, http.StatusGatewayTimeout, read.status, read.body)
message := routerErrorMessage(t, iface, read.body)
require.True(t, strings.HasPrefix(message, "no answer from upstream within "), message)
require.NotContains(t, message, "context deadline exceeded")
// MAG-3980: the router's own 504 carries the request's headers, as a reply an
// upstream produced does: the GUID, and the upstream it asked. One attempt went out,
// so there is no retry to report.
require.NotEmpty(t, read.guid, "the router's 504 carries no Lava-Guid")
require.Equal(t, "upstream-1", read.header.Get(common.PROVIDER_ADDRESS_HEADER_NAME))
require.Empty(t, read.header.Get(common.RETRY_COUNT_HEADER_NAME))
})

t.Run("masked mode", func(t *testing.T) {
Expand All @@ -297,3 +326,70 @@ func TestFailedRelayReply_EndToEnd_MAG4481(t *testing.T) {
})
}
}

// MAG-3980: a request whose budget runs out while every upstream is quiet is answered by the
// router itself, and that reply carries the request's headers like any other failed reply. It used
// to return before they were built, so the client had no Lava-Guid to quote for the failure that
// is hardest to diagnose, and no Lava-Retries or Lava-Provider-Address to say what was tried.
func TestFailedRelayReply_BudgetExpiryKeepsTheRequestHeaders_MAG3980(t *testing.T) {
if !rand.Initialized() {
rand.InitRandomSeed()
}
// Two windows: the ticker fires once inside the budget and hedges to the other upstream, with a
// window to spare. Its next tick finds both upstreams busy and sends nothing.
originalTimeout := common.DefaultTimeout
common.DefaultTimeout = 2 * failedReplyRelayTimeout
t.Cleanup(func() { common.DefaultTimeout = originalTimeout })

jsonRPC := failedReplyInterfaces()[0]
require.Equal(t, "jsonrpc", jsonRPC.name)

var arrivals atomic.Int32
read := startFailedReplyRouter(t, jsonRPC, hangingUpstream(t, &arrivals), hangingUpstream(t, &arrivals))()
require.Equal(t, http.StatusGatewayTimeout, read.status, read.body)
require.EqualValues(t, 2, arrivals.Load(), "setup: the ticker must have hedged to the other upstream, or this measures a single attempt")

require.NotEmpty(t, read.guid, "the reply carries no Lava-Guid")
require.Contains(t, read.body, read.guid, "the header names the request the body names")
require.Equal(t, "1", read.header.Get(common.RETRY_COUNT_HEADER_NAME), "two attempts went out")
require.ElementsMatch(t, []string{"upstream-1", "upstream-2"},
strings.Split(read.header.Get(common.PROVIDER_ADDRESS_HEADER_NAME), ","),
"both upstreams were asked; nothing answered, so nothing follows them")
require.Equal(t, "true", read.header.Get(common.LAVA_HEDGE_TRIGGERED_HEADER))
}

// MAG-3980: a cross-validated request whose budget runs out while every upstream is quiet fails as
// one with too few answers does. Its reply names the reason, insufficient-responses, as every
// other cross-validation failure does, and every upstream asked is still pending.
func TestFailedRelayReply_BudgetExpiryOfACrossValidatedRequestNamesItsReason_MAG3980(t *testing.T) {
if !rand.Initialized() {
rand.InitRandomSeed()
}
originalTimeout := common.DefaultTimeout
common.DefaultTimeout = 2 * failedReplyRelayTimeout
t.Cleanup(func() { common.DefaultTimeout = originalTimeout })

jsonRPC := failedReplyInterfaces()[0]
require.Equal(t, "jsonrpc", jsonRPC.name)
// Three upstreams for a threshold of two: the 429 cases above leave upstream-1 held off for a
// while, and selection leaves a held-off upstream out while another is ready.
jsonRPC.headers = map[string]string{
common.CROSS_VALIDATION_HEADER_MAX_PARTICIPANTS: "3",
common.CROSS_VALIDATION_HEADER_AGREEMENT_THRESHOLD: "2",
}
// A cross-validated attempt is not cut off with the request (MAG-2187), and its own budget is the
// request's, so it could give up just before the request does and be recorded as an error. Its
// upstreams' node-url timeout keeps every attempt out past the request's budget: nothing is back.
jsonRPC.upstreamTimeout = 5 * failedReplyRelayTimeout

read := startFailedReplyRouter(t, jsonRPC, hangingUpstream(t, nil), hangingUpstream(t, nil), hangingUpstream(t, nil))()
require.Equal(t, http.StatusGatewayTimeout, read.status, read.body)
require.NotEmpty(t, read.guid, "the reply carries no Lava-Guid")
require.Equal(t, "failed", read.header.Get(common.CROSS_VALIDATION_STATUS_HEADER_NAME))
queried := read.header.Get(common.CROSS_VALIDATION_ALL_PROVIDERS_HEADER_NAME)
require.NotEmpty(t, queried)
require.Equal(t, queried, read.header.Get(common.CROSS_VALIDATION_PENDING_PROVIDERS_HEADER),
"every upstream asked is still out")
require.Equal(t, common.CrossValidationReasonInsufficientResponses, read.header.Get(common.CROSS_VALIDATION_FAILURE_REASON_HEADER),
"a failed cross-validation names its reason")
}
57 changes: 57 additions & 0 deletions protocol/rpcsmartrouter/hedge_loser_retry_count_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
Expand Down Expand Up @@ -466,6 +467,62 @@ func TestRESTListener_HedgedLoserCountsAsARetry(t *testing.T) {
}
}

// MAG-3778: every upstream is slower than the attempt window, so the ticker hedges while the first
// attempt is still running, and the first attempt answers before the hedge can. The hedge is cut
// off with no result, and the reply still counts it: Lava-Retries counts every attempt the router
// sent (MAG-1818), not the attempts that came back, so it reports one retry and names the hedge
// before the upstream that answered.
func TestJsonRpcListener_FirstAttemptBeatingItsHedgeCountsTheHedge(t *testing.T) {
rand.InitRandomSeed()
// Twice the window: the hedge goes out one window in, while the first attempt is still running,
// and the first attempt answers a whole window before the hedge could.
latency := 2 * failedReplyRelayTimeout
var arrivals atomic.Int32
hedgeCut := make(chan struct{})
markHedgeCut := sync.OnceFunc(func() { close(hedgeCut) })
slowUpstream := func(blockNumber string) string {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
arrivals.Add(1)
// Read to the end first: only then does the server watch the connection, and see the
// router hang up.
_, _ = io.Copy(io.Discard, r.Body)
select {
case <-time.After(latency):
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":77,"result":"` + blockNumber + `"}`))
case <-r.Context().Done():
markHedgeCut()
}
}))
t.Cleanup(upstream.Close)
return upstream.URL
}

jsonRPC := failedReplyInterfaces()[0]
require.Equal(t, "jsonrpc", jsonRPC.name)
read := startFailedReplyRouter(t, jsonRPC, slowUpstream("0x1"), slowUpstream("0x2"))()

require.Equal(t, http.StatusOK, read.status, read.body)
require.EqualValues(t, 2, arrivals.Load(), "setup: the ticker must have hedged to the other upstream")
require.Equal(t, "true", read.header.Get(common.LAVA_HEDGE_TRIGGERED_HEADER))
// Either upstream may be asked first; the body says which one answered.
winner, hedge := "upstream-1", "upstream-2"
if strings.Contains(read.body, `"0x2"`) {
winner, hedge = hedge, winner
}
require.Equal(t, "1", read.header.Get(common.RETRY_COUNT_HEADER_NAME),
"two attempts went out, and the one that lost was still running when the reply was built")
require.Equal(t, hedge+","+winner, read.header.Get(common.PROVIDER_ADDRESS_HEADER_NAME),
"the hedge is named, before the upstream that answered")

// The router cut the hedge off rather than the hedge answering: it had a window left to run.
select {
case <-hedgeCut:
case <-time.After(latency):
t.Fatal("the router never cancelled the hedge")
}
}

// fixedOrderOptimizer picks upstreams in a fixed order, skipping the ones a request already
// used, so a test can say which upstream a first attempt goes to and which one a hedge reaches
// without pinning: a pinned request is answered by its upstream or fails, and never hedges
Expand Down
11 changes: 9 additions & 2 deletions protocol/rpcsmartrouter/relay_end_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,9 @@ func TestRelayEnd_ExitPaths(t *testing.T) {
check: func(t *testing.T, _ *common.RelayResult, err error) { require.ErrorIs(t, err, errUnknownWriteOutcome) },
},
{
// Nothing comes back at all: the router returns no result, and the client still gets 500.
// Nothing comes back at all: the router answers for itself, and the client still gets
// 500. That reply carries the request's headers, as a write some upstream answered does
// (MAG-3980): the GUID, and every upstream the write went to.
name: "write_unknown_every_upstream_silent",
fanOut: true,
nodes: func(*testing.T) []relayEndNode {
Expand All @@ -374,7 +376,12 @@ func TestRelayEnd_ExitPaths(t *testing.T) {
timeout: 1500 * time.Millisecond,
check: func(t *testing.T, result *common.RelayResult, err error) {
require.ErrorIs(t, err, errUnknownWriteOutcome)
require.Nil(t, result)
require.NotNil(t, result, "the reply carries no headers")
require.Equal(t, http.StatusInternalServerError, result.StatusCode)
guid, _ := replyHeader(result.GetReply().GetMetadata(), common.GUID_HEADER_NAME)
require.Equal(t, strconv.FormatUint(relayEndCaseGUID("write_unknown_every_upstream_silent"), 10), guid)
providers, _ := replyHeader(result.GetReply().GetMetadata(), common.PROVIDER_ADDRESS_HEADER_NAME)
require.ElementsMatch(t, []string{"node-a", "node-b"}, strings.Split(providers, ","))
},
},
{
Expand Down
Loading