Merge branch 'cosmos/v0.47.x' of https://github.com/forbole/bdjuno in…

…to chains/persistence/mainnet

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Unreleased
 - ([\#610](https://github.com/forbole/bdjuno/pull/610)) Add support for gov `v1` proposals
 - ([\#652](https://github.com/forbole/bdjuno/pull/652)) Update gov module parsing 
+- ([\#702](https://github.com/forbole/bdjuno/pull/702)) Add `message_type` module and store msg types inside `message_type` table, add `messages_by_type` function to allow to query messages by their types
 
 ## Version v4.0.0
 ## Notes

cmd/migrate/cmd.go

@@ -8,13 +8,15 @@ import (
 	"github.com/spf13/cobra"
 
 	v3 "github.com/forbole/bdjuno/v4/cmd/migrate/v3"
+	v5 "github.com/forbole/bdjuno/v4/cmd/migrate/v5"
 )
 
 type Migrator func(parseCfg *parsecmdtypes.Config) error
 
 var (
 	migrations = map[string]Migrator{
 		"v3": v3.RunMigration,
+		"v5": v5.RunMigration,
 	}
 )
 

cmd/migrate/v5/migrate.go

@@ -0,0 +1,43 @@
+package v5
+
+import (
+	"fmt"
+
+	v5db "github.com/forbole/bdjuno/v4/database/migrate/v5"
+	parse "github.com/forbole/juno/v5/cmd/parse/types"
+	"github.com/forbole/juno/v5/database"
+	"github.com/forbole/juno/v5/database/postgresql"
+	"github.com/forbole/juno/v5/types/config"
+)
+
+// RunMigration runs the migrations to v5
+func RunMigration(parseConfig *parse.Config) error {
+	cfg, err := GetConfig()
+	if err != nil {
+		return fmt.Errorf("error while reading config: %s", err)
+	}
+
+	// Migrate the database
+	err = migrateDb(cfg, parseConfig)
+	if err != nil {
+		return fmt.Errorf("error while migrating database: %s", err)
+	}
+
+	return nil
+}
+
+func migrateDb(cfg config.Config, parseConfig *parse.Config) error {
+	// Build the codec
+	encodingConfig := parseConfig.GetEncodingConfigBuilder()()
+
+	// Get the db
+	databaseCtx := database.NewContext(cfg.Database, encodingConfig, parseConfig.GetLogger())
+	db, err := postgresql.Builder(databaseCtx)
+	if err != nil {
+		return fmt.Errorf("error while building the db: %s", err)
+	}
+
+	// Build the migrator and perform the migrations
+	migrator := v5db.NewMigrator(db.(*postgresql.Database))
+	return migrator.Migrate()
+}

cmd/migrate/v5/utils.go

@@ -0,0 +1,29 @@
+package v5
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/forbole/juno/v5/types/config"
+	"gopkg.in/yaml.v3"
+)
+
+// GetConfig returns the configuration reading it from the config.yaml file present inside the home directory
+func GetConfig() (config.Config, error) {
+	file := path.Join(config.HomePath, "config.yaml")
+
+	// Make sure the path exists
+	if _, err := os.Stat(file); os.IsNotExist(err) {
+		return config.Config{}, fmt.Errorf("config file does not exist")
+	}
+
+	bz, err := os.ReadFile(file)
+	if err != nil {
+		return config.Config{}, fmt.Errorf("error while reading config file: %s", err)
+	}
+
+	var cfg config.Config
+	err = yaml.Unmarshal(bz, &cfg)
+	return cfg, err
+}

database/database_test.go

@@ -56,7 +56,7 @@ func (suite *DbTestSuite) SetupTest() {
 		100000,
 		100,
 	)
-	db, err := database.Builder(junodb.NewContext(dbCfg, &codec, logging.DefaultLogger()))
+	db, err := database.Builder(junodb.NewContext(dbCfg, codec, logging.DefaultLogger()))
 	suite.Require().NoError(err)
 
 	bigDipperDb, ok := (db).(*database.Db)
@@ -168,4 +168,4 @@ func (suite *DbTestSuite) getAccount(addr string) sdk.AccAddress {
 	suite.Require().NoError(err)
 
 	return delegator
-}
+}

database/message_type.go

@@ -0,0 +1,16 @@
+package database
+
+import (
+	types "github.com/forbole/bdjuno/v4/types"
+)
+
+// SaveMessageType stores the given message type inside the database
+func (db *Db) SaveMessageType(msg *types.MessageType) error {
+	stmt := `
+INSERT INTO message_type(type, module, label, height) 
+VALUES ($1, $2, $3, $4) 
+ON CONFLICT (type) DO NOTHING`
+
+	_, err := db.SQL.Exec(stmt, msg.Type, msg.Module, msg.Label, msg.Height)
+	return err
+}

database/migrate/v5/migrate.go

@@ -0,0 +1,53 @@
+package v5
+
+import (
+	"fmt"
+
+	utils "github.com/forbole/bdjuno/v4/modules/utils"
+	"github.com/forbole/bdjuno/v4/types"
+)
+
+// Migrate implements database.Migrator
+func (db *Migrator) Migrate() error {
+	msgTypes, err := db.getMsgTypesFromMessageTable()
+	if err != nil {
+		return fmt.Errorf("error while getting message types rows: %s", err)
+	}
+
+	for _, msgType := range msgTypes {
+		// migrate message types
+		err = db.migrateMsgTypes(types.NewMessageType(
+			msgType.Type,
+			utils.GetModuleNameFromTypeURL(msgType.Type),
+			utils.GetMsgFromTypeURL(msgType.Type),
+			msgType.Height))
+
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// getMsgTypesFromMessageTable retrieves messages types stored in database inside message table
+func (db *Migrator) getMsgTypesFromMessageTable() ([]MessageRow, error) {
+	smt := "SELECT DISTINCT ON (type) type, transaction_hash, height FROM message ORDER BY type DESC"
+	var rows []MessageRow
+	err := db.SQL.Select(&rows, smt)
+	if err != nil {
+		return nil, err
+	}
+
+	return rows, nil
+}
+
+// migrateMsgTypes stores the given message type inside the database
+func (db *Migrator) migrateMsgTypes(msg *types.MessageType) error {
+	stmt := `
+INSERT INTO message_type(type, module, label, height) 
+VALUES ($1, $2, $3, $4) 
+ON CONFLICT (type) DO NOTHING`
+
+	_, err := db.SQL.Exec(stmt, msg.Type, msg.Module, msg.Label, msg.Height)
+	return err
+}

database/migrate/v5/migrator.go

@@ -0,0 +1,21 @@
+package v5
+
+import (
+	"github.com/jmoiron/sqlx"
+
+	"github.com/forbole/juno/v5/database"
+	"github.com/forbole/juno/v5/database/postgresql"
+)
+
+var _ database.Migrator = &Migrator{}
+
+// Migrator represents the database migrator that should be used to migrate from v4 of the database to v5
+type Migrator struct {
+	SQL *sqlx.DB
+}
+
+func NewMigrator(db *postgresql.Database) *Migrator {
+	return &Migrator{
+		SQL: db.SQL,
+	}
+}

database/migrate/v5/types.go

@@ -0,0 +1,10 @@
+package v5
+
+type MessageRow struct {
+	TransactionHash           string `db:"transaction_hash"`
+	Index                     int64  `db:"index"`
+	Type                      string `db:"type"`
+	Value                     string `db:"value"`
+	InvolvedAccountsAddresses string `db:"involved_accounts_addresses"`
+	Height                    int64  `db:"height"`
+}

database/schema/00-cosmos.sql

@@ -66,11 +66,21 @@ CREATE INDEX transaction_hash_index ON transaction (hash);
 CREATE INDEX transaction_height_index ON transaction (height);
 CREATE INDEX transaction_partition_id_index ON transaction (partition_id);
 
+CREATE TABLE message_type
+(
+    type      TEXT   NOT NULL UNIQUE,
+    module    TEXT   NOT NULL,
+    label     TEXT   NOT NULL,
+    height    BIGINT NOT NULL
+);
+CREATE INDEX message_type_module_index ON message_type (module);
+CREATE INDEX message_type_type_index ON message_type (type);
+
 CREATE TABLE message
 (
     transaction_hash            TEXT   NOT NULL,
     index                       BIGINT NOT NULL,
-    type                        TEXT   NOT NULL,
+    type                        TEXT   NOT NULL REFERENCES message_type(type),
     value                       JSONB  NOT NULL,
     involved_accounts_addresses TEXT[] NOT NULL,
 
@@ -101,6 +111,17 @@ WHERE (cardinality(types) = 0 OR type = ANY (types))
 ORDER BY height DESC LIMIT "limit" OFFSET "offset"
 $$ LANGUAGE sql STABLE;
 
+CREATE FUNCTION messages_by_type(
+  types text [],
+  "limit" bigint DEFAULT 100,
+  "offset" bigint DEFAULT 0) 
+  RETURNS SETOF message AS 
+$$ 
+SELECT * FROM message
+WHERE (cardinality(types) = 0 OR type = ANY (types))
+ORDER BY height DESC LIMIT "limit" OFFSET "offset" 
+$$ LANGUAGE sql STABLE;
+
 CREATE TABLE pruning
 (
     last_pruned_height BIGINT NOT NULL

database/staking_validators.go

@@ -436,7 +436,7 @@ ON CONFLICT (validator_address) DO UPDATE
 WHERE validator_status.height <= excluded.height`
 	_, err = db.SQL.Exec(statusStmt, statusParams...)
 	if err != nil {
-		return fmt.Errorf("error while stroring validators statuses: %s", err)
+		return fmt.Errorf("error while storing validators statuses: %s", err)
 	}
 
 	return nil

database/staking_validators_test.go

@@ -246,7 +246,7 @@ VALUES ('cosmosvalcons1qqqqrezrl53hujmpdch6d805ac75n220ku09rl',
 func (suite *DbTestSuite) TestGetValidators() {
 	suite.getAccount("cosmos1z4hfrxvlgl4s8u4n5ngjcw8kdqrcv43599amxs")
 	suite.getAccount("cosmos184ma3twcfjqef6k95ne8w2hk80x2kah7vcwy4a")
-	// Inser the test data
+	// Insert the test data
 	queries := []string{
 		`INSERT INTO validator (consensus_address, consensus_pubkey) VALUES ('cosmosvalcons1qqqqrezrl53hujmpdch6d805ac75n220ku09rl', 'cosmosvalconspub1zcjduepq7mft6gfls57a0a42d7uhx656cckhfvtrlmw744jv4q0mvlv0dypskehfk8')`,
 		`INSERT INTO validator (consensus_address, consensus_pubkey) VALUES ('cosmosvalcons1qq92t2l4jz5pt67tmts8ptl4p0jhr6utx5xa8y', 'cosmosvalconspub1zcjduepqe93asg05nlnj30ej2pe3r8rkeryyuflhtfw3clqjphxn4j3u27msrr63nk')`,