feat(rpc): add rpc pool for endpoint fallback

README.md

@@ -100,3 +100,51 @@ Challenger node types:
 - host
 - child
 ```
+
+## RPC Pool Configuration
+
+To enhance reliability, the bots support multiple RPC endpoints for L1, L2, and DA nodes. This allows for automatic fallback and retries if an endpoint becomes unavailable.
+
+### Endpoint Configuration
+
+In your configuration file (`~/.opinit/[bot-name].json`), you can specify a list of RPC addresses for each node:
+
+```json
+{
+  "l1_node": {
+    "chain_id": "testnet-l1-1",
+    "bech32_prefix": "init",
+    "rpc_addresses": [
+      "tcp://doi-rpc:26657",
+      "tcp://another-l1-rpc:26657"
+    ]
+  },
+  "l2_node": {
+    "chain_id": "testnet-l2-1",
+    "bech32_prefix": "init",
+    "rpc_addresses": [
+      "tcp://moro-rpc:27657",
+      "tcp://another-l2-rpc:27657"
+    ]
+  },
+  "da_node": {
+    "chain_id": "testnet-l1-1",
+    "bech32_prefix": "init",
+    "rpc_addresses": [
+      "tcp://rene-rpc:26657",
+      "tcp://another-da-rpc:26657"
+    ]
+  }
+}
+```
+
+The bot will try the endpoints in the order they are listed. If a request to the first endpoint fails, it will automatically fall back to the next one in the list.
+
+### Timeout Configuration
+
+You can configure the timeout for each RPC request using the `RPC_TIMEOUT_SECONDS` environment variable. The value is in seconds. If the variable is not set, the default timeout is 5 seconds.
+
+```bash
+export RPC_TIMEOUT_SECONDS=10
+opinitd start [bot-name]
+```

challenger/README.md

@@ -28,12 +28,18 @@ To configure the Challenger, fill in the values in the `~/.opinit/challenger.jso
   "l1_node": {
     "chain_id": "testnet-l1-1",
     "bech32_prefix": "init",
-    "rpc_address": "tcp://localhost:26657",
+    "rpc_addresses": [
+      "tcp://doi-rpc:26657",
+      "tcp://localhost:26657"
+    ],
   },
   "l2_node": {
     "chain_id": "testnet-l2-1",
     "bech32_prefix": "init",
-    "rpc_address": "tcp://localhost:27657",
+    "rpc_addresses": [
+      "tcp://moro-rpc:26657",
+      "tcp://localhost:27657"
+    ],
   },
   // DisableAutoSetL1Height is the flag to disable the automatic setting of the l1 height.
   // If it is false, it will finds the optimal height and sets l1_start_height automatically

challenger/challenger.go

@@ -22,9 +22,13 @@ import (
 
 var _ bottypes.Bot = &Challenger{}
 
-// Executor charges the execution of the bridge between the host and the child chain
-// - relay l1 deposit messages to l2
-// - generate l2 output root and submit to l1
+// Challenger is responsible for:
+// Verifying that the MsgInitiateTokenDeposit event is properly relayed to MsgFinalizeTokenDeposit
+// Checking whether MsgInitiateTokenDeposit was relayed on time
+// Verifying that the Oracle data is properly relayed to MsgUpdateOracle
+// Checking whether Oracle was relayed on time
+// Verifying that the OutputRoot submitted with MsgProposeOutput is correct
+// Checking whether next MsgProposeOutput was submitted on time
 type Challenger struct {
 	host  *host.Host
 	child *child.Child
@@ -84,6 +88,12 @@ func NewChallenger(cfg *challengertypes.Config, db types.DB, sv *server.Server)
 }
 
 func (c *Challenger) Initialize(ctx types.Context) error {
+	// Initialize the child's query client first so we can query bridge info
+	err := c.child.InitializeQueryClient(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize child query client")
+	}
+
 	childBridgeInfo, err := c.child.QueryBridgeInfo(ctx)
 	if err != nil {
 		return err
@@ -92,6 +102,12 @@ func (c *Challenger) Initialize(ctx types.Context) error {
 		return errors.New("bridge info is not set")
 	}
 
+	// Initialize the host's query client so we can query bridge config
+	err = c.host.InitializeQueryClient(ctx)
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize host query client")
+	}
+
 	bridgeInfo, err := c.host.QueryBridgeConfig(ctx, childBridgeInfo.BridgeId)
 	if err != nil {
 		return err
@@ -179,8 +195,25 @@ func (c *Challenger) Start(ctx types.Context) error {
 		return c.challengeHandler(ctx)
 	})
 
-	c.host.Start(ctx)
-	c.child.Start(ctx)
+	// Start components in separate goroutines to ensure proper shutdown handling
+	ctx.ErrGrp().Go(func() (err error) {
+		defer func() {
+			ctx.Logger().Info("host stopped")
+		}()
+		c.host.Start(ctx)
+		<-ctx.Done()
+		return nil
+	})
+
+	ctx.ErrGrp().Go(func() (err error) {
+		defer func() {
+			ctx.Logger().Info("child stopped")
+		}()
+		c.child.Start(ctx)
+		<-ctx.Done()
+		return nil
+	})
+
 	return ctx.ErrGrp().Wait()
 }
 

challenger/types/config.go

@@ -8,9 +8,9 @@ import (
 )
 
 type NodeConfig struct {
-	ChainID      string `json:"chain_id"`
-	Bech32Prefix string `json:"bech32_prefix"`
-	RPCAddress   string `json:"rpc_address"`
+	ChainID      string   `json:"chain_id"`
+	Bech32Prefix string   `json:"bech32_prefix"`
+	RPCAddresses []string `json:"rpc_addresses"`
 }
 
 func (nc NodeConfig) Validate() error {
@@ -20,7 +20,7 @@ func (nc NodeConfig) Validate() error {
 	if nc.Bech32Prefix == "" {
 		return errors.New("bech32 prefix is required")
 	}
-	if nc.RPCAddress == "" {
+	if len(nc.RPCAddresses) == 0 {
 		return errors.New("RPC address is required")
 	}
 	return nil
@@ -66,13 +66,13 @@ func DefaultConfig() *Config {
 		L1Node: NodeConfig{
 			ChainID:      "testnet-l1-1",
 			Bech32Prefix: "init",
-			RPCAddress:   "tcp://localhost:26657",
+			RPCAddresses: []string{"tcp://localhost:26657"},
 		},
 
 		L2Node: NodeConfig{
 			ChainID:      "testnet-l2-1",
 			Bech32Prefix: "init",
-			RPCAddress:   "tcp://localhost:27657",
+			RPCAddresses: []string{"tcp://localhost:27657"},
 		},
 		DisableAutoSetL1Height: false,
 		L1StartHeight:          1,
@@ -114,7 +114,7 @@ func (cfg Config) Validate() error {
 func (cfg Config) L1NodeConfig() nodetypes.NodeConfig {
 	nc := nodetypes.NodeConfig{
 		ChainID:      cfg.L1Node.ChainID,
-		RPC:          cfg.L1Node.RPCAddress,
+		RPC:          cfg.L1Node.RPCAddresses,
 		ProcessType:  nodetypes.PROCESS_TYPE_DEFAULT,
 		Bech32Prefix: cfg.L1Node.Bech32Prefix,
 	}
@@ -124,7 +124,7 @@ func (cfg Config) L1NodeConfig() nodetypes.NodeConfig {
 func (cfg Config) L2NodeConfig() nodetypes.NodeConfig {
 	nc := nodetypes.NodeConfig{
 		ChainID:      cfg.L2Node.ChainID,
-		RPC:          cfg.L2Node.RPCAddress,
+		RPC:          cfg.L2Node.RPCAddresses,
 		ProcessType:  nodetypes.PROCESS_TYPE_DEFAULT,
 		Bech32Prefix: cfg.L2Node.Bech32Prefix,
 	}

cmd/opinitd/db.go

@@ -58,8 +58,14 @@ v0.1.9-2: Fill block hash of finalized tree
 					return err
 				}
 
+				rpcTimeout, err := cmd.Flags().GetDuration(flagRPCTimeout)
+				if err != nil {
+					return err
+				}
+
 				baseCtx := types.NewContext(cmdCtx, ctx.logger.Named(string(bottypes.BotTypeExecutor)), ctx.homePath).
-					WithPollingInterval(interval)
+					WithPollingInterval(interval).
+					WithRPCTimeout(rpcTimeout)
 
 				configPath, err := getConfigPath(cmd, ctx.homePath, string(bottypes.BotTypeExecutor))
 				if err != nil {
@@ -79,7 +85,7 @@ v0.1.9-2: Fill block hash of finalized tree
 					return err
 				}
 
-				rpcClient, err := rpcclient.NewRPCClient(cdc, l2Config.RPC)
+				rpcClient, err := rpcclient.NewRPCClient(baseCtx, cdc, l2Config.RPC, baseCtx.Logger().Named("migration-rpcclient"))
 				if err != nil {
 					return err
 				}
@@ -106,5 +112,6 @@ v0.1.9-2: Fill block hash of finalized tree
 	}
 	cmd = configFlag(ctx.v, cmd)
 	cmd.Flags().Duration(flagPollingInterval, 100*time.Millisecond, "Polling interval in milliseconds")
+	cmd.Flags().Duration(flagRPCTimeout, 5*time.Second, "RPC timeout duration")
 	return cmd
 }

cmd/opinitd/start.go

@@ -19,6 +19,7 @@ import (
 
 const (
 	flagPollingInterval = "polling-interval"
+	flagRPCTimeout      = "rpc-timeout"
 )
 
 func startCmd(cmdCtx *cmdContext) *cobra.Command {
@@ -62,9 +63,15 @@ Currently supported bots:
 				return err
 			}
 
+			rpcTimeout, err := cmd.Flags().GetDuration(flagRPCTimeout)
+			if err != nil {
+				return err
+			}
+
 			baseCtx := types.NewContext(ctx, cmdCtx.logger.Named(string(botType)), cmdCtx.homePath).
 				WithErrGrp(errGrp).
-				WithPollingInterval(interval)
+				WithPollingInterval(interval).
+				WithRPCTimeout(rpcTimeout)
 			err = bot.Initialize(baseCtx)
 			if err != nil {
 				return err
@@ -75,6 +82,7 @@ Currently supported bots:
 
 	cmd = configFlag(cmdCtx.v, cmd)
 	cmd.Flags().Duration(flagPollingInterval, 100*time.Millisecond, "Polling interval in milliseconds")
+	cmd.Flags().Duration(flagRPCTimeout, 5*time.Second, "RPC timeout duration")
 	return cmd
 }
 

cmd/opinitd/tx.go

@@ -226,7 +226,7 @@ func QueryBridgeId(ctx types.Context, cfg *executortypes.Config) (uint64, error)
 		return 0, err
 	}
 
-	l2RpcClient, err := rpcclient.NewRPCClient(cdc, l2Config.RPC)
+	l2RpcClient, err := rpcclient.NewRPCClient(ctx, cdc, l2Config.RPC, ctx.Logger().Named("l2-rpcclient"))
 	if err != nil {
 		return 0, err
 	}
@@ -247,7 +247,7 @@ func l1ProposerAccount(ctx types.Context, cfg *executortypes.Config, bridgeId ui
 		return nil, err
 	}
 
-	rpcClient, err := rpcclient.NewRPCClient(cdc, l1Config.RPC)
+	rpcClient, err := rpcclient.NewRPCClient(ctx, cdc, l1Config.RPC, ctx.Logger().Named("l1-rpcclient"))
 	if err != nil {
 		return nil, err
 	}
@@ -272,7 +272,7 @@ func l2BroadcasterAccount(ctx types.Context, cfg *executortypes.Config) (*broadc
 		return nil, err
 	}
 
-	rpcClient, err := rpcclient.NewRPCClient(cdc, l2Config.RPC)
+	rpcClient, err := rpcclient.NewRPCClient(ctx, cdc, l2Config.RPC, ctx.Logger().Named("l2-rpcclient"))
 	if err != nil {
 		return nil, err
 	}

e2e/helper.go

@@ -684,7 +684,7 @@ func (op OPTestHelper) ExecutorConfig() *executortypes.Config {
 		L1Node: executortypes.NodeConfig{
 			ChainID:       op.Initia.Config().ChainID,
 			Bech32Prefix:  op.Initia.Config().Bech32Prefix,
-			RPCAddress:    fmt.Sprintf("http://%s:26657", op.Initia.GetFullNode().HostName()),
+			RPCAddresses:  []string{fmt.Sprintf("http://%s:26657", op.Initia.GetFullNode().HostName())},
 			GasPrice:      op.Initia.Config().GasPrices,
 			GasAdjustment: op.Initia.Config().GasAdjustment,
 			TxTimeout:     60,
@@ -693,7 +693,7 @@ func (op OPTestHelper) ExecutorConfig() *executortypes.Config {
 		L2Node: executortypes.NodeConfig{
 			ChainID:       op.Minitia.Config().ChainID,
 			Bech32Prefix:  op.Minitia.Config().Bech32Prefix,
-			RPCAddress:    fmt.Sprintf("http://%s:26657", op.Minitia.GetFullNode().HostName()),
+			RPCAddresses:  []string{fmt.Sprintf("http://%s:26657", op.Minitia.GetFullNode().HostName())},
 			GasPrice:      "",
 			GasAdjustment: op.Minitia.Config().GasAdjustment,
 			TxTimeout:     60,
@@ -702,7 +702,7 @@ func (op OPTestHelper) ExecutorConfig() *executortypes.Config {
 		DANode: executortypes.NodeConfig{
 			ChainID:       op.DA.Config().ChainID,
 			Bech32Prefix:  op.DA.Config().Bech32Prefix,
-			RPCAddress:    fmt.Sprintf("http://%s:26657", op.DA.GetFullNode().HostName()),
+			RPCAddresses:  []string{fmt.Sprintf("http://%s:26657", op.DA.GetFullNode().HostName())},
 			GasPrice:      op.DA.Config().GasPrices,
 			GasAdjustment: op.DA.Config().GasAdjustment,
 			TxTimeout:     60,

executor/README.md

@@ -26,23 +26,32 @@ To configure the Executor, fill in the values in the `~/.opinit/executor.json` f
   "l1_node": {
     "chain_id": "testnet-l1-1",
     "bech32_prefix": "init",
-    "rpc_address": "tcp://localhost:26657",
+    "rpc_addresses": [
+      "tcp://doi-rpc.com",
+      "tcp://localhost:26657"
+    ],
     "gas_price": "0.15uinit",
     "gas_adjustment": 1.5,
     "tx_timeout": 60
   },
   "l2_node": {
     "chain_id": "testnet-l2-1",
     "bech32_prefix": "init",
-    "rpc_address": "tcp://localhost:27657",
+    "rpc_addresses": [
+      "tcp://moro-rpc:27657",
+      "tcp://localhost:27657"
+    ],
     "gas_price": "",
     "gas_adjustment": 1.5,
     "tx_timeout": 60
   },
   "da_node": {
     "chain_id": "testnet-l1-1",
     "bech32_prefix": "init",
-    "rpc_address": "tcp://localhost:26657",
+    "rpc_addresses": [
+      "tcp://rene-rpc:26657",
+      "tcp://localhost:26657"
+    ],
     "gas_price": "0.15uinit",
     "gas_adjustment": 1.5,
     "tx_timeout": 60
@@ -320,7 +329,7 @@ If the batch info registered in the chain is changed to change the account or DA
 
 ```go
 {
- RPCAddress string `json:"rpc_address"`
+ RPCAddresses []string `json:"rpc_addresses"`
  GasPrice string `json:"gas_price"`
  GasAdjustment string `json:"gas_adjustment"`
  ChainID string `json:"chain_id"`

executor/batchsubmitter/batch_test.go

@@ -272,7 +272,10 @@ func TestFinalizeBatch(t *testing.T) {
 	require.NoError(t, err)
 
 	mockCaller := mockclient.NewMockCaller()
-	rpcClient := rpcclient.NewRPCClientWithClient(appCodec, client.NewWithCaller(mockCaller))
+	testLogger, _ := zap.NewDevelopment()
+	testCtx := types.NewContext(context.Background(), testLogger, "/tmp").WithRPCTimeout(5 * time.Second)
+	rpcClient, err := rpcclient.NewRPCClientWithClient(testCtx, appCodec, client.NewWithCaller(mockCaller), []string{"http://localhost:26657"}, testLogger)
+	require.NoError(t, err)
 	batchNode := node.NewTestNode(nodetypes.NodeConfig{}, batchDB, appCodec, txConfig, rpcClient, nil)
 
 	hostCdc, _, err := hostprovider.GetCodec("init")
@@ -753,7 +756,10 @@ func TestSubmitGenesis(t *testing.T) {
 	require.NoError(t, err)
 
 	mockCaller := mockclient.NewMockCaller()
-	rpcClient := rpcclient.NewRPCClientWithClient(appCodec, client.NewWithCaller(mockCaller))
+	testLogger, _ := zap.NewDevelopment()
+	testCtx := types.NewContext(context.Background(), testLogger, "/tmp").WithRPCTimeout(5 * time.Second)
+	rpcClient, err := rpcclient.NewRPCClientWithClient(testCtx, appCodec, client.NewWithCaller(mockCaller), []string{"http://localhost:26657"}, testLogger)
+	require.NoError(t, err)
 	batchNode := node.NewTestNode(nodetypes.NodeConfig{}, batchDB, appCodec, txConfig, rpcClient, nil)
 
 	hostCdc, _, err := hostprovider.GetCodec("init")

executor/batchsubmitter/handler_test.go

@@ -46,7 +46,10 @@ func TestRawBlockHandler(t *testing.T) {
 	require.NoError(t, err)
 
 	mockCaller := mockclient.NewMockCaller()
-	rpcClient := rpcclient.NewRPCClientWithClient(appCodec, client.NewWithCaller(mockCaller))
+	testLogger, _ := zap.NewDevelopment()
+	testCtx := types.NewContext(context.Background(), testLogger, "/tmp").WithRPCTimeout(5 * time.Second)
+	rpcClient, err := rpcclient.NewRPCClientWithClient(testCtx, appCodec, client.NewWithCaller(mockCaller), []string{"http://localhost:26657"}, testLogger)
+	require.NoError(t, err)
 	batchNode := node.NewTestNode(nodetypes.NodeConfig{}, batchDB, appCodec, txConfig, rpcClient, nil)
 
 	hostCdc, _, err := hostprovider.GetCodec("init")