feat(db2): overall improvements

backend/cmd/store/main.go

@@ -2,8 +2,10 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"os"
+	"strconv"
 
 	"github.com/gobitfly/beaconchain/pkg/commons/db2/database"
 	"github.com/gobitfly/beaconchain/pkg/commons/log"
@@ -15,15 +17,19 @@ func main() {
 	project := args[0]
 	instance := args[1]
 	table := args[2]
+	port, err := strconv.Atoi(args[3])
+	if err != nil {
+		panic(err)
+	}
 
 	bt, err := database.NewBigTable(project, instance, nil)
 	if err != nil {
 		panic(err)
 	}
 	remote := database.NewRemote(database.Wrap(bt, table))
 	go func() {
-		log.Info("starting remote raw store on port 8087")
-		if err := http.ListenAndServe("0.0.0.0:8087", remote.Routes()); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		log.Infof("starting remote raw store on port %d", port)
+		if err := http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", port), remote.Routes()); err != nil && !errors.Is(err, http.ErrServerClosed) {
 			panic(err)
 		}
 	}()

backend/go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/golang/protobuf v1.5.4
 	github.com/gomodule/redigo v1.9.2
+	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/csrf v1.7.2
 	github.com/gorilla/handlers v1.5.2

backend/internal/e2e/data_test.go

@@ -12,20 +12,15 @@ import (
 	"github.com/gobitfly/beaconchain/internal/th"
 	"github.com/gobitfly/beaconchain/pkg/commons/db2/data"
 	"github.com/gobitfly/beaconchain/pkg/commons/db2/database"
-	"github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest"
+	"github.com/gobitfly/beaconchain/pkg/commons/db2/db2test"
 	"github.com/gobitfly/beaconchain/pkg/commons/indexer"
 	"github.com/gobitfly/beaconchain/pkg/commons/rpc"
 	"github.com/gobitfly/beaconchain/pkg/commons/types"
 )
 
 func TestStoreWithBackend(t *testing.T) {
-	clientBT, adminBT := databasetest.NewBigTable(t)
-	bigtable, err := database.NewBigTableWithClient(context.Background(), clientBT, adminBT, data.Schema)
-	if err != nil {
-		t.Fatal(err)
-	}
+	store := db2test.NewDataStore(t)
 
-	store := data.NewStore(database.Wrap(bigtable, data.Table))
 	backend := th.NewBackend(t)
 	_, usdt := backend.DeployToken(t, "usdt", "usdt", backend.BankAccount.From)
 
@@ -49,6 +44,7 @@ func TestStoreWithBackend(t *testing.T) {
 				t.Fatal(i, j, err)
 			}
 			backend.Commit()
+
 			lastBlock, err := backend.Client().BlockNumber(context.Background())
 			if err != nil {
 				t.Fatal(err)
@@ -57,27 +53,15 @@ func TestStoreWithBackend(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			if err := indexer.IndexBlocksWithTransformers(fmt.Sprintf("%d", backend.ChainID), []*types.Eth1Block{block}); err != nil {
+			if err := indexer.IndexBlocks(fmt.Sprintf("%d", backend.ChainID), []*types.Eth1Block{block}); err != nil {
 				t.Fatal(err)
 			}
 		}
 	}
 
 	t.Run("get interactions", func(t *testing.T) {
 		efficiencies := make(map[string]int64)
-		interactions, _, err := store.Get(addresses, nil, 50, data.WithDatabaseStats(func(msg string, args ...any) {
-			var efficiency int64
-			var rowRange string
-			for i := 0; i < len(args); i = i + 2 {
-				if args[i].(string) == database.KeyStatEfficiency {
-					efficiency = args[i+1].(int64)
-				}
-				if args[i].(string) == database.KeyStatRange {
-					rowRange = args[i+1].(string)
-				}
-			}
-			efficiencies[rowRange] = efficiency
-		}))
+		interactions, _, err := store.Get(addresses, nil, 50, data.WithDatabaseStats(getEfficiencies(efficiencies)))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -94,3 +78,19 @@ func TestStoreWithBackend(t *testing.T) {
 		}
 	})
 }
+
+func getEfficiencies(efficiencies map[string]int64) func(msg string, args ...any) {
+	return func(msg string, args ...any) {
+		var efficiency int64
+		var rowRange string
+		for i := 0; i < len(args); i = i + 2 {
+			if args[i].(string) == database.KeyStatEfficiency {
+				efficiency = args[i+1].(int64)
+			}
+			if args[i].(string) == database.KeyStatRange {
+				rowRange = args[i+1].(string)
+			}
+		}
+		efficiencies[rowRange] = efficiency
+	}
+}

backend/pkg/commons/db2/data/data_test.go

@@ -11,7 +11,7 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/gobitfly/beaconchain/pkg/commons/db2/database"
-	"github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest"
+	"github.com/gobitfly/beaconchain/pkg/commons/db2/database/databasetest"
 	"github.com/gobitfly/beaconchain/pkg/commons/types"
 )
 

backend/pkg/commons/db2/data/keys.go

@@ -117,7 +117,7 @@ func transactionKeys(chainID string, transaction *types.Eth1TransactionIndexed,
 
 // key are sorted side (+optional other address), chainID, type, asset
 func transferKeys(chainID string, transaction *types.Eth1ERC20Indexed, index int, logIndex int) (string, []string) {
-	main := "ERC20:<chainID>:<hash>"
+	main := "ERC20:<chainID>:<hash>:<logIndex>"
 	baseKeys := []string{
 		"all:<address>",
 		"all:chainID:<address>:<chainID>",

backend/pkg/commons/db2/database/bigtable_test.go

@@ -16,7 +16,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 
-	"github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest"
+	"github.com/gobitfly/beaconchain/pkg/commons/db2/database/databasetest"
 )
 
 func TestNewBigTable(t *testing.T) {

backend/pkg/commons/db2/database/remote_test.go

@@ -6,7 +6,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest"
+	"github.com/gobitfly/beaconchain/pkg/commons/db2/database/databasetest"
 )
 
 func TestRemote(t *testing.T) {

backend/pkg/commons/db2/rawtest/raw.go → backend/pkg/commons/db2/db2test/db2test.go

@@ -1,4 +1,4 @@
-package rawtest
+package db2test
 
 import (
 	"context"
@@ -8,40 +8,42 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/gobitfly/beaconchain/internal/th"
+	"github.com/gobitfly/beaconchain/pkg/commons/db2/data"
 	"github.com/gobitfly/beaconchain/pkg/commons/db2/database"
-	"github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest"
+	"github.com/gobitfly/beaconchain/pkg/commons/db2/database/databasetest"
 	"github.com/gobitfly/beaconchain/pkg/commons/db2/raw"
 )
 
-func NewRandSeededStore(t *testing.T) (raw.Store, *th.BlockchainBackend) {
-	t.Helper()
+func NewDataStore(t *testing.T) data.Store {
 	client, admin := databasetest.NewBigTable(t)
-	bt, err := database.NewBigTableWithClient(context.Background(), client, admin, raw.Schema)
+	db, err := database.NewBigTableWithClient(context.Background(), client, admin, data.Schema)
 	if err != nil {
 		t.Fatal(err)
 	}
+	return data.NewStore(database.Wrap(db, data.Table))
+}
 
-	db := raw.NewStore(database.Wrap(bt, raw.BlocksRawTable))
-
-	backend := th.NewBackend(t)
-	for i := 0; i < 10; i++ {
-		temp := th.CreateEOA(t)
-		backend.FundOneEther(t, temp.From)
-	}
-	lastBlock, err := backend.Client().BlockNumber(context.Background())
+func NewRawStore(t *testing.T) raw.Store {
+	client, admin := databasetest.NewBigTable(t)
+	db, err := database.NewBigTableWithClient(context.Background(), client, admin, raw.Schema)
 	if err != nil {
 		t.Fatal(err)
 	}
-	var blocks []raw.FullBlockData
-	for i := uint64(0); i <= lastBlock; i++ {
-		blocks = append(blocks, makeRawBlock(t, backend.Endpoint, uint64(backend.ChainID), i))
+	return raw.NewStore(database.Wrap(db, raw.Table))
+}
+
+func NewStores(t *testing.T) (raw.Store, data.Store) {
+	return NewRawStore(t), NewDataStore(t)
+}
+
+func AddBlockToRawStore(t *testing.T, store raw.Store, endpoint string, chainID uint64, blocks []uint64) {
+	var fullBlocks []raw.FullBlockData
+	for _, block := range blocks {
+		fullBlocks = append(fullBlocks, makeRawBlock(t, endpoint, chainID, block))
 	}
-	if err := db.AddBlocks(blocks); err != nil {
+	if err := store.AddBlocks(fullBlocks); err != nil {
 		t.Fatal(err)
 	}
-
-	return db, backend
 }
 
 func makeRawBlock(t *testing.T, endpoint string, chainID uint64, block uint64) raw.FullBlockData {

backend/pkg/commons/db2/raw/client_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 
 	"github.com/gobitfly/beaconchain/pkg/commons/db2/database"
-	"github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest"
+	"github.com/gobitfly/beaconchain/pkg/commons/db2/database/databasetest"
 )
 
 const (
@@ -48,7 +48,7 @@ func TestBigTableClientRealCondition(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			rawStore := NewStore(database.Wrap(bt, BlocksRawTable))
+			rawStore := NewStore(database.Wrap(bt, Table))
 			rpcClient, err := rpc.DialOptions(context.Background(), "http://foo.bar", rpc.WithHTTPClient(&http.Client{
 				Transport: NewBigTableEthRaw(rawStore, chainID),
 			}))
@@ -130,7 +130,7 @@ func BenchmarkRawBigTable(b *testing.B) {
 		b.Fatal(err)
 	}
 
-	rawStore := WithCache(NewStore(database.Wrap(bt, BlocksRawTable)))
+	rawStore := WithCache(NewStore(database.Wrap(bt, Table)))
 	rpcClient, err := rpc.DialOptions(context.Background(), "http://foo.bar", rpc.WithHTTPClient(&http.Client{
 		Transport: NewBigTableEthRaw(rawStore, chainID),
 	}))
@@ -173,7 +173,7 @@ func TestBigTableClient(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			rawStore := NewStore(database.Wrap(bt, BlocksRawTable))
+			rawStore := NewStore(database.Wrap(bt, Table))
 			if err := rawStore.AddBlocks([]FullBlockData{tt.block}); err != nil {
 				t.Fatal(err)
 			}
@@ -237,7 +237,7 @@ func TestBigTableClientWithFallback(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			rawStore := NewStore(database.Wrap(bt, BlocksRawTable))
+			rawStore := NewStore(database.Wrap(bt, Table))
 
 			rpcClient, err := rpc.DialOptions(context.Background(), node, rpc.WithHTTPClient(&http.Client{
 				Transport: NewWithFallback(NewBigTableEthRaw(rawStore, tt.block.ChainID), http.DefaultTransport),