refactor(core): read the blockchain through an interface

cmd/api/main.go

@@ -12,6 +12,7 @@ import (
 	"github.com/tonkeeper/opentonapi/pkg/blockchain"
 	"github.com/tonkeeper/opentonapi/pkg/blockchain/indexer"
 	"github.com/tonkeeper/opentonapi/pkg/config"
+	"github.com/tonkeeper/opentonapi/pkg/core"
 	"github.com/tonkeeper/opentonapi/pkg/litestorage"
 	"github.com/tonkeeper/opentonapi/pkg/pyth"
 	"github.com/tonkeeper/opentonapi/pkg/spam"
@@ -59,7 +60,7 @@ func main() {
 	pythFeeds := pyth.GetUpdatedWithFallback(context.Background(), log)
 	storage, err := litestorage.NewLiteStorage(
 		log,
-		client,
+		core.LiteAPIClient(client),
 		litestorage.WithPreloadBlocks([]tongo.BlockID{
 			tongo.MustParseBlockID("(0,8000000000000000,72945279)"),
 		}),
@@ -92,7 +93,7 @@ func main() {
 	if err != nil {
 		log.Fatal("failed to create api handler", zap.Error(err))
 	}
-	idx := indexer.New(log, client)
+	idx := indexer.New(log, core.LiteAPIClient(client))
 	go idx.Run(context.TODO(), []chan indexer.IDandBlock{
 		storageBlockCh,
 	})

pkg/addressbook/addressbook_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 	"github.com/tonkeeper/opentonapi/pkg/config"
+	"github.com/tonkeeper/opentonapi/pkg/core"
 	"github.com/tonkeeper/opentonapi/pkg/litestorage"
 	"github.com/tonkeeper/tongo/liteapi"
 	"github.com/tonkeeper/tongo/ton"
@@ -91,7 +92,7 @@ func TestSearchAttachedAccountsByPrefix(t *testing.T) {
 	client, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 	require.NoError(t, err, "Failed to create lite API client")
 
-	liteStorage, err := litestorage.NewLiteStorage(logger, client)
+	liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(client))
 	require.NoError(t, err, "Failed to create lite storage")
 
 	book := NewAddressBook(logger, config.AddressPath, config.JettonPath, config.CollectionPath, liteStorage)

pkg/api/account_handlers_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/tonkeeper/opentonapi/pkg/core"
 	"github.com/tonkeeper/opentonapi/pkg/spam"
 
 	"github.com/tonkeeper/opentonapi/pkg/chainstate"
@@ -65,7 +66,7 @@ func TestHandler_GetRawAccount(t *testing.T) {
 			logger, _ := zap.NewDevelopment()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 			require.Nil(t, err)
 			book := &mockAddressBook{
 				OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {
@@ -150,7 +151,7 @@ func TestHandler_GetBlockchainRawAccounts(t *testing.T) {
 			logger, _ := zap.NewDevelopment()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 			require.Nil(t, err)
 			h := &Handler{
 				storage: liteStorage,
@@ -201,7 +202,7 @@ func TestHandler_GetAccount(t *testing.T) {
 			logger, _ := zap.NewDevelopment()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 			require.Nil(t, err)
 			book := &mockAddressBook{
 				OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {
@@ -274,7 +275,7 @@ func TestHandler_GetAccounts(t *testing.T) {
 			logger, _ := zap.NewDevelopment()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 			require.Nil(t, err)
 			book := &mockAddressBook{
 				OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {
@@ -360,7 +361,7 @@ func TestHandler_GetTransactions(t *testing.T) {
 			logger, _ := zap.NewDevelopment()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 			require.Nil(t, err)
 			book := &mockAddressBook{
 				OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {

pkg/api/blockchain_handlers_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/tonkeeper/opentonapi/pkg/addressbook"
+	"github.com/tonkeeper/opentonapi/pkg/core"
 	"github.com/tonkeeper/tongo"
 
 	"github.com/stretchr/testify/require"
@@ -25,7 +26,7 @@ func TestHandler_GetRawBlockchainConfig(t *testing.T) {
 	logger := zap.L()
 	cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 	require.Nil(t, err)
-	liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+	liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 	require.Nil(t, err)
 	book := &mockAddressBook{
 		OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {
@@ -99,7 +100,7 @@ func TestHandler_GetRawBlockchainConfigFromBlock(t *testing.T) {
 			logger := zap.L()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 			require.Nil(t, err)
 			book := &mockAddressBook{
 				OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {
@@ -157,7 +158,7 @@ func TestHandler_GetBlockchainConfigFromBlock(t *testing.T) {
 			logger := zap.L()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 			require.Nil(t, err)
 			book := &mockAddressBook{
 				OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {
@@ -187,7 +188,7 @@ func TestHandler_GetBlockchainValidators(t *testing.T) {
 	logger := zap.L()
 	cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 	require.Nil(t, err)
-	liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+	liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 	require.Nil(t, err)
 	book := &mockAddressBook{
 		OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {
@@ -242,7 +243,7 @@ func TestHandler_GetBlockchainBlock(t *testing.T) {
 			logger := zap.L()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 			require.Nil(t, err)
 			book := &mockAddressBook{
 				OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {

pkg/api/decode_message_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/tonkeeper/opentonapi/pkg/addressbook"
+	"github.com/tonkeeper/opentonapi/pkg/core"
 	"github.com/tonkeeper/tongo"
 
 	"github.com/stretchr/testify/require"
@@ -43,7 +44,7 @@ func TestHandler_DecodeMessage(t *testing.T) {
 			logger, _ := zap.NewDevelopment()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 			require.Nil(t, err)
 			book := &mockAddressBook{
 				OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {

pkg/api/event_handlers_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/tonkeeper/opentonapi/pkg/addressbook"
 	"github.com/tonkeeper/opentonapi/pkg/bath"
+	"github.com/tonkeeper/opentonapi/pkg/core"
 	"github.com/tonkeeper/opentonapi/pkg/spam"
 	"github.com/tonkeeper/tongo"
 
@@ -87,7 +88,7 @@ func TestHandler_EmulateMessageToAccountEvent(t *testing.T) {
 			logger, _ := zap.NewDevelopment()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 			require.Nil(t, err)
 			book := &mockAddressBook{
 				OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {

pkg/api/handler.go

@@ -107,6 +107,7 @@ type Options struct {
 	score                   scoreSource
 	parallelTraceProcessing bool
 	archiveLiteServers      []config.LiteServer
+	archiveClient           rewards.LiteClient
 	publicAPIURL            string
 }
 
@@ -210,6 +211,18 @@ func WithArchiveLiteServers(s []config.LiteServer) Option {
 	}
 }
 
+// WithArchiveClient supplies the blockchain connection the rewards service
+// reads validator history through, instead of NewHandler building one from the
+// servers given to WithArchiveLiteServers. It is how a deployment running its
+// own lightserver pool keeps the rewards service on that pool rather than
+// opening a second one; when set, WithArchiveLiteServers is not used to
+// construct a client.
+func WithArchiveClient(cli rewards.LiteClient) Option {
+	return func(o *Options) {
+		o.archiveClient = cli
+	}
+}
+
 func WithPublicAPIURL(publicAPIURL string) Option {
 	return func(o *Options) {
 		o.publicAPIURL = publicAPIURL
@@ -275,13 +288,24 @@ func NewHandler(logger *zap.Logger, opts ...Option) (*Handler, error) {
 		slog.Warn("unable to detect tongo version", "err", err)
 	}
 	var rwd *rewards.Service
-	if len(options.archiveLiteServers) != 0 {
-		cli, err := rewards.NewClient(options.archiveLiteServers)
-		if err == nil {
-			rwd = rewards.New(cli, options.archiveLiteServers)
-			log.Println("rewards service initialized")
-		} else {
-			log.Println("rewards service unavailable:", err)
+	var stats *rewards.Stats
+	switch {
+	case options.archiveClient != nil:
+		// A supplied client keeps its own connections, so there is no server
+		// list to hand the service for rebuilding one.
+		rwd = rewards.New(options.archiveClient, nil)
+		stats = rewards.NewStatsWithClient(options.archiveClient)
+		log.Println("rewards service initialized on the supplied client")
+	default:
+		stats = rewards.NewStats(liteapi.WithLiteServers(options.archiveLiteServers))
+		if len(options.archiveLiteServers) != 0 {
+			cli, err := rewards.NewClient(options.archiveLiteServers)
+			if err == nil {
+				rwd = rewards.New(cli, options.archiveLiteServers)
+				log.Println("rewards service initialized")
+			} else {
+				log.Println("rewards service unavailable:", err)
+			}
 		}
 	}
 	return &Handler{
@@ -318,7 +342,7 @@ func NewHandler(logger *zap.Logger, opts ...Option) (*Handler, error) {
 		tonConnect:              tonConnect,
 		configPool:              configPool,
 		rewards:                 rwd,
-		stats:                   rewards.NewStats(liteapi.WithLiteServers(options.archiveLiteServers)),
+		stats:                   stats,
 	}, nil
 }
 

pkg/api/jetton_handlers_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/tonkeeper/opentonapi/pkg/addressbook"
+	"github.com/tonkeeper/opentonapi/pkg/core"
 
 	"github.com/stretchr/testify/require"
 	"github.com/tonkeeper/tongo"
@@ -38,7 +39,7 @@ func TestHandler_GetJettonsBalances(t *testing.T) {
 			logger, _ := zap.NewDevelopment()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli, litestorage.WithKnownJettons([]tongo.AccountID{
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli), litestorage.WithKnownJettons([]tongo.AccountID{
 				tongo.MustParseAddress("0:beb5d4638e860ccf7317296e298fde5b35982f4725b0676dc98b1de987b82ebc").ID, // Jetton kingy
 				tongo.MustParseAddress("0:65de083a0007638233b6668354e50e44cd4225f1730d66b8b1f19e5d26690751").ID, // Lavandos
 				tongo.MustParseAddress("0:274b605badfcecca83130b27cd375e6a73233f6e15d782a31dd2a80aff097cc0").ID, // fake jUSDT (with cyrillic T)

pkg/api/staking_handlers_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/tonkeeper/opentonapi/pkg/addressbook"
+	"github.com/tonkeeper/opentonapi/pkg/core"
 	"github.com/tonkeeper/tongo"
 
 	"github.com/stretchr/testify/require"
@@ -41,7 +42,7 @@ func TestHandler_GetStakingPoolInfo(t *testing.T) {
 			logger := zap.L()
 			cli, err := liteapi.NewClient(liteapi.FromEnvsOrMainnet())
 			require.Nil(t, err)
-			liteStorage, err := litestorage.NewLiteStorage(logger, cli)
+			liteStorage, err := litestorage.NewLiteStorage(logger, core.LiteAPIClient(cli))
 			require.Nil(t, err)
 			book := &mockAddressBook{
 				OnGetAddressInfoByAddress: func(a tongo.AccountID) (addressbook.KnownAddress, bool) {

pkg/bath/bath_test.go

@@ -102,7 +102,7 @@ func isFocusedRun() bool {
 func newBathTestStorage(t *testing.T, cli *liteapi.Client, blocks []tongo.BlockID) *litestorage.LiteStorage {
 	t.Helper()
 	storage, err := litestorage.NewLiteStorage(zap.L(),
-		cli,
+		core.LiteAPIClient(cli),
 		litestorage.WithPreloadBlocks(blocks),
 		litestorage.WithPythPriceFeeds(pyth.Default),
 	)

pkg/blockchain/indexer/indexer.go

@@ -7,8 +7,8 @@ import (
 	"time"
 
 	"github.com/sourcegraph/conc/iter"
+	"github.com/tonkeeper/opentonapi/pkg/core"
 	"github.com/tonkeeper/tongo"
-	"github.com/tonkeeper/tongo/liteapi"
 	"github.com/tonkeeper/tongo/tlb"
 	"go.uber.org/zap"
 )
@@ -22,10 +22,10 @@ type chunk struct {
 // Indexer tracks the blockchain and notifies subscribers about new blocks.
 type Indexer struct {
 	logger *zap.Logger
-	cli    *liteapi.Client
+	cli    core.LiteClient
 }
 
-func New(logger *zap.Logger, cli *liteapi.Client) *Indexer {
+func New(logger *zap.Logger, cli core.LiteClient) *Indexer {
 	return &Indexer{
 		cli:    cli,
 		logger: logger,

pkg/blockchain/msg_sender.go

@@ -19,10 +19,18 @@ import (
 
 const ttl = 5 * 60 // in seconds
 
+// MessageSender is one blockchain connection an external message can be
+// broadcast through. MsgSender holds several - one per liteserver - because a
+// message wants to reach as many validators as it can, which is the opposite of
+// what a pool that picks a single best connection does for you.
+type MessageSender interface {
+	SendMessage(ctx context.Context, payload []byte) (uint32, error)
+}
+
 // MsgSender provides a method to send a message to the blockchain.
 type MsgSender struct {
 	logger         *zap.Logger
-	sendingClients []*liteapi.Client
+	sendingClients []MessageSender
 	// receivers get a copy of a message before sending it to the blockchain.
 	// receivers is a read-only map/field.
 	receivers map[string]chan<- ExtInMsgCopy
@@ -95,6 +103,20 @@ func NewMsgSender(logger *zap.Logger, servers []config.LiteServer, receivers map
 	if len(clients) == 0 {
 		return nil, fmt.Errorf("no lite clients available")
 	}
+	senders := make([]MessageSender, 0, len(clients))
+	for _, cli := range clients {
+		senders = append(senders, cli)
+	}
+	return NewMsgSenderWithClients(logger, senders, receivers)
+}
+
+// NewMsgSenderWithClients builds a sender over connections the caller supplies,
+// one per liteserver it wants messages broadcast to, for a deployment whose
+// connections are not ones this package can open from a list of liteservers.
+func NewMsgSenderWithClients(logger *zap.Logger, clients []MessageSender, receivers map[string]chan<- ExtInMsgCopy) (*MsgSender, error) {
+	if len(clients) == 0 {
+		return nil, fmt.Errorf("no lite clients available")
+	}
 
 	msgSender := &MsgSender{
 		sendingClients: clients,

pkg/blockchain/msg_sender_test.go

@@ -76,7 +76,7 @@ func TestMsgSender_send_AllAttemptsFail(t *testing.T) {
 	)
 	require.NoError(t, err)
 
-	ms := &MsgSender{sendingClients: []*liteapi.Client{client}}
+	ms := &MsgSender{sendingClients: []MessageSender{client}}
 	sendCtx, sendCancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer sendCancel()