Merge pull request #64 from cosmos-gaminghub/develop

Develop

.github/workflows/release.yml

@@ -0,0 +1,40 @@
+name: Upload Release Asset
+on:
+  push:
+    tags:
+    - 'v*'
+jobs:
+  build:
+    name: Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Build project
+        run: |
+          make all
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Release ${{ github.ref }}
+          draft: false
+          prerelease: false
+      - name: Set version
+        id: tag_version
+        run: |
+          VERSION=$(echo ${{ github.ref }} | sed -e "s#refs/tags/##g")
+          echo ::set-output name=version::$VERSION
+      - name: Upload Release Asset
+        id: upload-release-asset 
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: build/exporter
+          asset_name: exporter-${{ steps.tag_version.outputs.version }}
+          asset_content_type: application/octet-stream

Makefile

@@ -3,11 +3,9 @@ all: get_deps build
 get_deps:
 	rm -rf ./build
 	rm -rf ./vendor
-	rm -rf ./build
 	go mod download
 	go mod vendor
 
 build:
-	go build -o build/explorer explorer.go
+	go build -o build/exporter exporter.go
 	go build -o build/cron cron-exporter.go
-

README.md

@@ -2,15 +2,17 @@
 Blockchain exporter for the [CosmosSDK](https://github.com/cosmos/cosmos-sdk) based Blockchain(ex [gaia](https://github.com/cosmos/gaia)). This backend server can be applied to any cosmosSDK based blockchain to sync as mongoDB collections.
 
 **Prerequisites**
-* go1.16.0+
+* go1.17.0+
 * mongoDB 4.4.0+
 * docker (for local usage)
 
 ## Get Started
 - set up mongoDB
 
 - build source
+
 Just run exporter binary after building source.
+
 ```bash
 git clone https://github.com/cosmos-gaminghub/explorer-backend.git
 cd explorer-backend

client/client.go

@@ -82,7 +82,7 @@ func GetValidatorSet(height int64, offset int64) ([]types.ValidatorOfValidatorSe
 	}
 
 	var result types.ValidatorSet
-	validators := make([]types.ValidatorOfValidatorSet, 0, 125)
+	validators := make([]types.ValidatorOfValidatorSet, 0, lcd.DefaultValidatorSetLimit)
 	if err := json.Unmarshal(resBytes, &result); err != nil {
 		logger.Error(fmt.Sprintf("Unmarshal validator set error for height %d", height), logger.String("err", err.Error()))
 	}

exporter/account_transaction.go

@@ -28,14 +28,16 @@ func SaveAccountTransaction(validators []*schema.Validator, transactions []*sche
 func getListAccountAddres(messages string) []string {
 	var list []string
 	var addressPrefix = conf.Get().Db.AddresPrefix
+	var length = len(addressPrefix) + 39 // length off account address
 	for {
 		if strings.Contains(messages, addressPrefix) {
 			index := strings.Index(messages, addressPrefix)
-			address := utils.Convert(addressPrefix, messages[index:index+45])
+
+			address := utils.Convert(addressPrefix, messages[index:index+length])
 			if address != "" {
 				list = append(list, address)
 			}
-			messages = messages[index+45 : len(messages)-1]
+			messages = messages[index+length : len(messages)-1]
 		} else {
 			break
 		}

exporter/exporter.go

@@ -53,6 +53,18 @@ func Start() error {
 		}
 	}()
 
+	go func() {
+		for {
+			fmt.Println("start - sync total missed block for validator")
+			err := syncTotalMissedBlock()
+			if err != nil {
+				fmt.Printf("error - sync total missed block for validator: %v\n", err)
+			}
+			fmt.Println("finish - sync total missed block for validator")
+			time.Sleep(3600 * time.Second)
+		}
+	}()
+
 	for {
 		select {}
 	}
@@ -185,6 +197,15 @@ func syncProposal() error {
 	return nil
 }
 
+func syncTotalMissedBlock() error {
+	validators, err := document.Validator{}.GetValidatorList()
+	if err != nil {
+		return nil
+	}
+	saveTotalMissedBlock(validators)
+	return nil
+}
+
 func processProposal(proposal schema.Proposal) error {
 	depositResult, _ := client.GetProposalDeposits(proposal.ProposalId)
 	if len(depositResult.Deposits) > 0 {

exporter/validator.go

@@ -27,33 +27,24 @@ func GetValidators(vals []types.Validator, validatorSets []types.ValidatorOfVali
 		_, decodeByte, _ := bech32.DecodeAndConvert(consensusAddress)
 		str := base64.StdEncoding.EncodeToString(decodeByte)
 
-		if validator.Description.ImageUrl == "" && validator.Description.Identity != "" {
-			validator.Description.ImageUrl = client.GetImageUrl(validator.Description.Identity)
-		}
-
-		var missedBlockCount int64
-		if consensusAddress != "" {
-			valSigningInfo, err := client.GetValSigningInfo(consensusAddress)
-			if err == nil {
-				missedBlockCount, _ = strconv.ParseInt(valSigningInfo.Info.MissedBlocksCount, 10, 64)
-			}
-		}
+		// if validator.Description.ImageUrl == "" && validator.Description.Identity != "" {
+		// 	validator.Description.ImageUrl = client.GetImageUrl(validator.Description.Identity)
+		// }
 
 		val := &schema.Validator{
-			OperatorAddr:     validator.OperatorAddress,
-			ConsensusAddres:  consensusAddress,
-			ConsensusPubkey:  validator.ConsensusPubkey.Key,
-			AccountAddr:      utils.Convert(conf.Get().Db.AddresPrefix, validator.OperatorAddress),
-			Jailed:           validator.Jailed,
-			Status:           validator.Status,
-			Tokens:           tokens,
-			DelegatorShares:  validator.DelegatorShares,
-			Description:      validator.Description,
-			UnbondingHeight:  validator.UnbondingHeight,
-			UnbondingTime:    validator.UnbondingTime,
-			Commission:       validator.Commission,
-			ProposerAddr:     str,
-			TotalMissedBlock: missedBlockCount,
+			OperatorAddr:    validator.OperatorAddress,
+			ConsensusAddres: consensusAddress,
+			ConsensusPubkey: validator.ConsensusPubkey.Key,
+			AccountAddr:     utils.Convert(conf.Get().Db.AddresPrefix, validator.OperatorAddress),
+			Jailed:          validator.Jailed,
+			Status:          validator.Status,
+			Tokens:          tokens,
+			DelegatorShares: validator.DelegatorShares,
+			Description:     validator.Description,
+			UnbondingHeight: validator.UnbondingHeight,
+			UnbondingTime:   validator.UnbondingTime,
+			Commission:      validator.Commission,
+			ProposerAddr:    str,
 		}
 		validators = append(validators, val)
 	}
@@ -66,3 +57,16 @@ func SaveValidator(validator schema.Validator) (interface{}, error) {
 
 	return orm.Upsert(document.CollectionNmValidator, selector, validator)
 }
+
+func saveTotalMissedBlock(validators []document.Validator) {
+	for _, validator := range validators {
+		var missedBlockCount int64
+		if validator.ConsensusAddress != "" {
+			valSigningInfo, err := client.GetValSigningInfo(validator.ConsensusAddress)
+			if err == nil {
+				missedBlockCount, _ = strconv.ParseInt(valSigningInfo.Info.MissedBlocksCount, 10, 64)
+			}
+		}
+		orm.Update(document.CollectionNmValidator, bson.M{"consensus_address": validator.ConsensusAddress}, bson.M{"$set": bson.M{"total_missed_block": missedBlockCount}})
+	}
+}

go.mod

@@ -1,26 +1,30 @@
 module github.com/cosmos-gaminghub/explorer-backend
 
-go 1.16
+go 1.17
 
 require (
-	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
-	github.com/cosmos/cosmos-sdk v0.42.4
-	github.com/etcd-io/bbolt v1.3.3 // indirect
+	github.com/cosmos/cosmos-sdk v0.44.3
 	github.com/joho/godotenv v1.3.0
 	github.com/pkg/errors v0.9.1
 	github.com/robfig/cron v1.2.0
-	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/stretchr/testify v1.7.0
-	github.com/swaggo/swag v1.7.0
-	github.com/tendermint/tendermint v0.34.10 // indirect
-	go.uber.org/zap v1.16.0
-	golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc
-	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
-	google.golang.org/grpc v1.37.0 // indirect
+	go.uber.org/zap v1.17.0
+	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
+	golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/enigmampc/btcutil v1.0.3-0.20200723161021-e2fb6adb2a25 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)
+
 replace (
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	google.golang.org/grpc => google.golang.org/grpc v1.33.2