From 365a5ee49532a66732f0c91a879aa40bfdcca2b2 Mon Sep 17 00:00:00 2001 From: fountainchen Date: Thu, 11 Aug 2022 22:29:08 +0800 Subject: [PATCH 01/82] [scan]update searcher builder after handle --- .github/workflows/build.yml | 5 - .github/workflows/docker_build_indexer.yml | 8 +- .github/workflows/docker_build_scanapi.yml | 2 +- README.md | 2 +- pom.xml | 2 +- starcoin-commons/pom.xml | 2 +- starcoin-indexer/pom.xml | 4 +- starcoin-scan-api/pom.xml | 2 +- .../starcoin/scan/service/BlockService.java | 28 +---- .../starcoin/scan/service/TokenService.java | 15 ++- .../scan/service/TransactionService.java | 105 ++++++++---------- 11 files changed, 69 insertions(+), 106 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9c3133d..0521449 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,11 +20,6 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 - - name: maven-settings-xml-action - uses: whelk-io/maven-settings-xml-action@v18 - with: - repositories: '[{ "id": "github", "url": "https://maven.pkg.github.com/starcoinorg/*" }]' - servers: '[{ "id": "github", "username": "${{ github.actor }}", "password": "${{ secrets.GIT_PACKAGE_TOKEN }}" }]' - name: Cache Maven packages uses: actions/cache@v1 with: diff --git a/.github/workflows/docker_build_indexer.yml b/.github/workflows/docker_build_indexer.yml index 45a7822..032f5f9 100644 --- a/.github/workflows/docker_build_indexer.yml +++ b/.github/workflows/docker_build_indexer.yml @@ -22,7 +22,7 @@ jobs: id: docker_meta uses: crazy-max/ghaction-docker-meta@v1 with: - images: starcoin/starcoin_indexer,ghcr.io/starcoinorg/starcoin_indexer + images: starcoin/starcoin_indexer tag-sha: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1.6.0 @@ -33,12 +33,6 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GIT_PACKAGE_TOKEN }} - name: maven-settings-xml-action uses: whelk-io/maven-settings-xml-action@v18 with: diff --git a/.github/workflows/docker_build_scanapi.yml b/.github/workflows/docker_build_scanapi.yml index 23012e6..0c8c258 100644 --- a/.github/workflows/docker_build_scanapi.yml +++ b/.github/workflows/docker_build_scanapi.yml @@ -22,7 +22,7 @@ jobs: id: docker_meta uses: crazy-max/ghaction-docker-meta@v1 with: - images: starcoin/starcoin_scan,ghcr.io/starcoinorg/starcoin_scan + images: starcoin/starcoin_scan tag-sha: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1.6.0 diff --git a/README.md b/README.md index 2f49545..909ce2f 100644 --- a/README.md +++ b/README.md @@ -1 +1 @@ -# starcoin-search +# StcScan diff --git a/pom.xml b/pom.xml index 93902f2..2f793f6 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.starcoin - starcoin-search + stcscan pom 1.1 diff --git a/starcoin-commons/pom.xml b/starcoin-commons/pom.xml index 9e1b8b2..f8f1f8d 100644 --- a/starcoin-commons/pom.xml +++ b/starcoin-commons/pom.xml @@ -3,7 +3,7 @@ xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - starcoin-search + stcscan org.starcoin 1.1 diff --git a/starcoin-indexer/pom.xml b/starcoin-indexer/pom.xml index 7f815dd..c8d941a 100644 --- a/starcoin-indexer/pom.xml +++ b/starcoin-indexer/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - starcoin-search + stcscan org.starcoin 1.1 @@ -14,7 +14,7 @@ github - GitHub starcoin-search Apache Maven Packages + GitHub stcscan Apache Maven Packages https://maven.pkg.github.com/starcoinorg/starcoin-indexer diff --git a/starcoin-scan-api/pom.xml b/starcoin-scan-api/pom.xml index 088404f..8383d46 100644 --- a/starcoin-scan-api/pom.xml +++ b/starcoin-scan-api/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - starcoin-search + stcscan org.starcoin 1.1 diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java index 39b0b02..ae57d1d 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java @@ -97,17 +97,9 @@ public Result getRange(String network, int page, int count, int start_hei searchSourceBuilder.query(QueryBuilders.matchAllQuery()); //page size searchSourceBuilder.size(count); - //begin offset - int offset; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS && start_height > 0) { - offset = start_height - (page - 1) * count; - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } + //set offset + TransactionService.setBlockSearchBuildFrom(page, count, start_height, searchSourceBuilder); + searchSourceBuilder.sort("header.number", SortOrder.DESC); searchSourceBuilder.trackTotalHits(true); searchRequest.source(searchSourceBuilder); @@ -207,17 +199,9 @@ public Result getUnclesRange(String network, int page, int count, in searchSourceBuilder.query(QueryBuilders.matchAllQuery()); //page size searchSourceBuilder.size(count); - //begin offset - int offset; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS && start_height > 0) { - offset = start_height - (page - 1) * count; - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } + //set offset + TransactionService.setBlockSearchBuildFrom(page, count, start_height, searchSourceBuilder); + searchSourceBuilder.sort("uncle_block_number", SortOrder.DESC); searchSourceBuilder.trackTotalHits(true); searchRequest.source(searchSourceBuilder); diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java index e01ce06..8eb4cb8 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java @@ -453,18 +453,21 @@ public Result getHoldersByToken(String network, int page, int c SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.ADDRESS_INDEX)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(count); - //begin offset + //set offset + boolean shouldAfter = true; int offset = 0; if (page > 1) { offset = (page - 1) * count; if (offset >= ELASTICSEARCH_MAX_HITS) { - searchSourceBuilder.searchAfter(new Object[]{offset}); - searchSourceBuilder.from(0); - } else { - searchSourceBuilder.from(offset); + shouldAfter = true; } } - searchSourceBuilder.from(offset); + if(shouldAfter) { + searchSourceBuilder.from(0); + searchSourceBuilder.searchAfter(new Object[]{offset}); + }else { + searchSourceBuilder.from(offset); + } TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("type_tag.keyword", tokenType); diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java index 8ed937f..39384e7 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java @@ -96,17 +96,8 @@ public Result getRange(String network, int page, int count //page size searchSourceBuilder.size(count); //begin offset - int offset = 0; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS && start_height > 0) { - offset = start_height - (page - 1) * count; - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } - searchSourceBuilder.from(offset); + setBlockSearchBuildFrom(page, count, start_height, searchSourceBuilder); + searchSourceBuilder.sort("timestamp", SortOrder.DESC); searchRequest.source(searchSourceBuilder); searchSourceBuilder.trackTotalHits(true); @@ -128,16 +119,8 @@ public Result getTxnByStartTime(String network, long start //page size searchSourceBuilder.size(count); //begin offset - int offset = 0; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS) { - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } - searchSourceBuilder.from(offset); + setSearchBuildFrom(page, count, searchSourceBuilder); + searchSourceBuilder.sort("timestamp", SortOrder.DESC); searchRequest.source(searchSourceBuilder); searchSourceBuilder.trackTotalHits(true); @@ -160,17 +143,9 @@ public Result getNFTTxns(String network, long start_time, } //page size searchSourceBuilder.size(count); - //begin offset - int offset = 0; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS) { - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } - searchSourceBuilder.from(offset); + //set offset + setSearchBuildFrom(page, count, searchSourceBuilder); + List termHashes = new ArrayList<>(); for (Event event : events.getContents()) { termHashes.add(event.getTransactionHash()); @@ -191,23 +166,47 @@ public Result getRangePendingTransaction(String network, int //page size searchSourceBuilder.size(count); - //begin offset + //set offset + setBlockSearchBuildFrom(page, count, start_height, searchSourceBuilder); + + searchSourceBuilder.sort("timestamp", SortOrder.DESC); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.trackTotalHits(true); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + return ServiceUtils.getSearchResult(searchResponse, PendingTransaction.class); + } + + static void setBlockSearchBuildFrom(int page, int count, int start_height, SearchSourceBuilder searchSourceBuilder) { int offset = 0; + boolean shouldAfter = false; if (page > 1) { offset = (page - 1) * count; if (offset >= ELASTICSEARCH_MAX_HITS && start_height > 0) { offset = start_height - (page - 1) * count; - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); + shouldAfter = true; } } - searchSourceBuilder.from(offset); - searchSourceBuilder.sort("timestamp", SortOrder.DESC); - searchRequest.source(searchSourceBuilder); - searchSourceBuilder.trackTotalHits(true); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - return ServiceUtils.getSearchResult(searchResponse, PendingTransaction.class); + if(shouldAfter) { + searchSourceBuilder.searchAfter(new Object[]{offset}); + }else { + searchSourceBuilder.from(offset); + } + } + + static void setSearchBuildFrom(int page, int count, SearchSourceBuilder searchSourceBuilder) { + int offset = 0; + boolean shouldAfter = false; + if (page > 1) { + offset = (page - 1) * count; + if (offset >= ELASTICSEARCH_MAX_HITS) { + shouldAfter = true; + } + } + if(shouldAfter) { + searchSourceBuilder.searchAfter(new Object[]{offset}); + }else { + searchSourceBuilder.from(offset); + } } public PendingTransaction getPending(String network, String id) throws IOException { @@ -227,17 +226,9 @@ public Result getRangeTransfers(String network, String typeTag, String receiver, SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); //page size searchSourceBuilder.size(count); - //begin offset - int offset = 0; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS) { - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } - searchSourceBuilder.from(offset); + //set offset + setSearchBuildFrom(page, count, searchSourceBuilder); + searchSourceBuilder.sort("timestamp", SortOrder.DESC); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); if (typeTag != null && typeTag.length() > 0) { @@ -387,12 +378,8 @@ public Result getEventsByAddress(String network, String address, int page SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.TRANSACTION_EVENT_INDEX)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(count); - //begin offset - int offset = 0; - if (page > 1) { - offset = (page - 1) * count; - } - searchSourceBuilder.from(offset); + //set offset + setSearchBuildFrom(page, count, searchSourceBuilder); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); boolQuery.should(QueryBuilders.termQuery("tag_name", ServiceUtils.depositEvent)); From 287ae9aded934088c1114f0428aee2e2c0114eb4 Mon Sep 17 00:00:00 2001 From: fountainchen Date: Fri, 12 Aug 2022 11:50:43 +0800 Subject: [PATCH 02/82] [scan]update searcher builder after handle --- .../scan/controller/TokenController.java | 2 +- .../starcoin/scan/service/TokenService.java | 20 ++++++------------- .../scan/service/TransactionService.java | 9 ++++++++- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TokenController.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TokenController.java index 9488d38..8457d6a 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TokenController.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TokenController.java @@ -27,7 +27,7 @@ public class TokenController { @ApiOperation("get token aggregate stat list") @GetMapping("/{network}/stats/{page}") public Result getAggregate(@PathVariable("network") String network, @PathVariable("page") int page, - @RequestParam(value = "count", required = false, defaultValue = "50") int count) { + @RequestParam(value = "count", required = false, defaultValue = "10") int count) { return tokenService.tokenAggregateList(network, page, count); } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java index 8eb4cb8..1829434 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java @@ -75,7 +75,7 @@ public Result tokenAggregateList(String network, int page, i Result volumes = tokenVolumeList(network, page, count); Map volumeMap = getVolumeMap(network, volumes); //get market cap - Result market = tokenMarketCap(network, page, ELASTICSEARCH_MAX_HITS); + Result market = tokenMarketCap(network, 1, ELASTICSEARCH_MAX_HITS); Map marketMap = getMarketMap(market); List viewList = new ArrayList<>(); for (TokenStatistic tokenStatistic : holderContents) { @@ -94,7 +94,7 @@ public Result tokenAggregateList(String network, int page, i viewList.add(view); } result.setContents(viewList); - result.setTotal(holders.getTotal()); + result.setTotal(market.getTotal()); return result; } @@ -270,8 +270,8 @@ public Result tokenHolderList(String network, int page, int coun searchSourceBuilder.query(QueryBuilders.matchAllQuery()); TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("holders") .field("type_tag.keyword") + .size(ELASTICSEARCH_MAX_HITS) .order(BucketOrder.aggregation("address_holders", false)) -// .size(count) .subAggregation(AggregationBuilders.count("address_holders").field("address.keyword")) .subAggregation(new BucketSortPipelineAggregationBuilder("bucket_field", null).from(offset).size(count)); @@ -297,16 +297,8 @@ public Result tokenMarketCap(String network, int page, int count .must(QueryBuilders.matchAllQuery()); searchSourceBuilder.query(queryBuilder); //page size - int offset = 0; searchSourceBuilder.size(count); - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS) { - searchSourceBuilder.searchAfter(new Object[]{offset}); - } - } - //begin offset - searchSourceBuilder.from(offset); + TransactionService.setSearchBuildFrom(page, count, searchSourceBuilder); searchSourceBuilder.trackTotalHits(true); searchRequest.source(searchSourceBuilder); searchSourceBuilder.timeout(new TimeValue(20, TimeUnit.SECONDS)); @@ -368,7 +360,6 @@ public Result tokenVolumeList(String network, int page, int coun logger.error("get token volume error:", e); return null; } - } public void loadTokenInfo(String network) { @@ -446,6 +437,7 @@ private Result searchStatistic(SearchResponse searchResponse, St } } result.setContents(statistics); + result.setTotal(searchResponse.getHits().getTotalHits().value); return result; } @@ -454,7 +446,7 @@ public Result getHoldersByToken(String network, int page, int c SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(count); //set offset - boolean shouldAfter = true; + boolean shouldAfter = false; int offset = 0; if (page > 1) { offset = (page - 1) * count; diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java index 39384e7..1833162 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java @@ -3,6 +3,7 @@ import com.alibaba.fastjson.JSON; import com.beust.jcommander.internal.Lists; import com.novi.serde.DeserializationError; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchRequest; @@ -101,7 +102,13 @@ public Result getRange(String network, int page, int count searchSourceBuilder.sort("timestamp", SortOrder.DESC); searchRequest.source(searchSourceBuilder); searchSourceBuilder.trackTotalHits(true); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + SearchResponse searchResponse = null; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (ElasticsearchException e) { + logger.error("get txn err:", e); + return Result.EmptyResult; + } return ServiceUtils.getSearchResult(searchResponse, TransactionWithEvent.class); } From 3a7ec86ce9b31b2a3df29399333c83eaec6331a1 Mon Sep 17 00:00:00 2001 From: fountainchen Date: Wed, 14 Dec 2022 21:08:53 +0800 Subject: [PATCH 03/82] [config]update pg connect size --- starcoin-indexer/src/main/resources/application.properties | 2 ++ starcoin-scan-api/src/main/resources/application.properties | 2 ++ 2 files changed, 4 insertions(+) diff --git a/starcoin-indexer/src/main/resources/application.properties b/starcoin-indexer/src/main/resources/application.properties index 5a6aa62..79d5d5d 100644 --- a/starcoin-indexer/src/main/resources/application.properties +++ b/starcoin-indexer/src/main/resources/application.properties @@ -16,6 +16,8 @@ elasticsearch.connectionRequestTimeout=2000 swap.contract.address=${SWAP_CONTRACT_ADDR} starcoin.swap.api.url=${SWAP_API_URL} logging.file.name=logs/indexer.log +spring.datasource.hikari.minimum-idle=1 +spring.datasource.hikari.maximum-pool-size=2 spring.datasource.url=${DS_URL} spring.datasource.username=${DB_USER_NAME} spring.datasource.password=${DB_PWD} diff --git a/starcoin-scan-api/src/main/resources/application.properties b/starcoin-scan-api/src/main/resources/application.properties index 3b116dc..88ab280 100644 --- a/starcoin-scan-api/src/main/resources/application.properties +++ b/starcoin-scan-api/src/main/resources/application.properties @@ -56,6 +56,8 @@ spring.datasource.halley.jdbc-url=${HALLEY_DS_URL} spring.datasource.halley.username=${DB_USER_NAME} spring.datasource.halley.password=${DB_PWD} #default ds config +spring.datasource.hikari.minimum-idle=1 +spring.datasource.hikari.maximum-pool-size=2 spring.datasource.url=${DS_URL} spring.datasource.username=${DB_USER_NAME} spring.datasource.password=${DB_PWD} From 268a104ac6a6689caedbb773c0358e2140eeec1b Mon Sep 17 00:00:00 2001 From: fountainchen Date: Wed, 14 Dec 2022 21:51:15 +0800 Subject: [PATCH 04/82] [config]update pg config for migration --- .../starcoin-indexer-barnard-deployment.yaml | 6 +- ...in-indexer-cmd-handle-main-deployment.yaml | 2 +- .../starcoin-indexer-halley-deployment.yaml | 6 +- .../starcoin-indexer-main-deployment.yaml | 4 +- .../starcoin-indexer-proxima-deployment.yaml | 6 +- ...oin-indexer-repair-barnard-deployment.yaml | 2 +- ...arcoin-indexer-repair-main-deployment.yaml | 8 +-- ...starcoin-indexer-stat-main-deployment.yaml | 4 +- ...starcoin-indexer-swap-main-deployment.yaml | 2 +- .../starcoin-indexer-txn-main-deployment.yaml | 2 +- ...aml => starswap-info-main-deployment.yaml} | 32 +++++----- kube/scan-api/scan-api-deployment.yaml | 14 ++--- kube/scan-api/scan-test-api-deployment.yaml | 58 ------------------- kube/scan-api/service-scan-test-api.yaml | 17 ------ .../scan-api/service-scan-test-hazelcast.yaml | 13 ----- .../starscan-sub-barnard-deployment.yaml | 2 +- .../starscan-sub-halley-deployment.yaml | 2 +- .../starscan-sub-main-deployment.yaml | 2 +- .../starscan-sub-proxima-deployment.yaml | 2 +- 19 files changed, 47 insertions(+), 137 deletions(-) rename kube/indexer/{starcoin-indexer-cmd-handle-barnard-deployment.yaml => starswap-info-main-deployment.yaml} (69%) delete mode 100644 kube/scan-api/scan-test-api-deployment.yaml delete mode 100644 kube/scan-api/service-scan-test-api.yaml delete mode 100644 kube/scan-api/service-scan-test-hazelcast.yaml diff --git a/kube/indexer/starcoin-indexer-barnard-deployment.yaml b/kube/indexer/starcoin-indexer-barnard-deployment.yaml index 6ce023a..2acc41a 100644 --- a/kube/indexer/starcoin-indexer-barnard-deployment.yaml +++ b/kube/indexer/starcoin-indexer-barnard-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: starcoin-indexer - image: starcoin/starcoin_indexer:sha-ba00b10 + image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 env: @@ -26,7 +26,7 @@ spec: - name: NETWORK value: "barnard.0727" - name: BG_TASK_JOBS - value: "indexer,secondary,market_cap,txn_payload,swap_stats" + value: "indexer,secondary,market_cap,txn_payload" - name: TXN_OFFSET value: "0" - name: BULK_SIZE @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml b/kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml index d6ac10d..06e87b3 100644 --- a/kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-halley-deployment.yaml b/kube/indexer/starcoin-indexer-halley-deployment.yaml index 115fda6..4798b30 100644 --- a/kube/indexer/starcoin-indexer-halley-deployment.yaml +++ b/kube/indexer/starcoin-indexer-halley-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: starcoin-indexer - image: starcoin/starcoin_indexer:sha-3af1c2a + image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 env: @@ -26,7 +26,7 @@ spec: - name: NETWORK value: "halley.0727" - name: BG_TASK_JOBS - value: "indexer,secondary,market_cap,txn_payload,swap_stats,txn_global_idx_update" + value: "indexer,secondary,market_cap,txn_payload,txn_global_idx_update" - name: BULK_SIZE value: "80" - name: STARCOIN_ES_PWD @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-main-deployment.yaml b/kube/indexer/starcoin-indexer-main-deployment.yaml index d28e5c6..18300fe 100644 --- a/kube/indexer/starcoin-indexer-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-main-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: starcoin-indexer - image: starcoin/starcoin_indexer:sha-ba00b10 + image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 env: @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-proxima-deployment.yaml b/kube/indexer/starcoin-indexer-proxima-deployment.yaml index 16f6832..fa5c9ea 100644 --- a/kube/indexer/starcoin-indexer-proxima-deployment.yaml +++ b/kube/indexer/starcoin-indexer-proxima-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: starcoin-indexer - image: starcoin/starcoin_indexer:sha-3af1c2a + image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 env: @@ -26,7 +26,7 @@ spec: - name: NETWORK value: "proxima.0727" - name: BG_TASK_JOBS - value: "indexer,secondary,market_cap,txn_payload,swap_stats" + value: "indexer,secondary,market_cap,txn_payload" - name: BULK_SIZE value: "80" - name: STARCOIN_ES_PWD @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml b/kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml index f7a70f0..0a56319 100644 --- a/kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml +++ b/kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-repair-main-deployment.yaml b/kube/indexer/starcoin-indexer-repair-main-deployment.yaml index c7d581a..595e91a 100644 --- a/kube/indexer/starcoin-indexer-repair-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-repair-main-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: starcoin-indexer-repair - image: starcoin/starcoin_indexer:sha-6b3888e + image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 env: @@ -26,7 +26,7 @@ spec: - name: NETWORK value: "main.0727" - name: BG_TASK_JOBS - value: "swap_event_handle,swap_pool_fee_stat" + value: "" - name: TXN_OFFSET value: "0" - name: BULK_SIZE @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD @@ -60,4 +60,4 @@ spec: - name: DB_SCHEMA value: "main" - name: PROGRAM_ARGS - value: "auto_repair 5356700" + value: "auto_repair 9411700" diff --git a/kube/indexer/starcoin-indexer-stat-main-deployment.yaml b/kube/indexer/starcoin-indexer-stat-main-deployment.yaml index 5f5b478..d397b7a 100644 --- a/kube/indexer/starcoin-indexer-stat-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-stat-main-deployment.yaml @@ -26,7 +26,7 @@ spec: - name: NETWORK value: "main.0727" - name: BG_TASK_JOBS - value: "swap_stats" + value: "swap_stats,price_hour,price_stat" - name: TXN_OFFSET value: "0" - name: BULK_SIZE @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-swap.czftx6muneg7.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-swap-main-deployment.yaml b/kube/indexer/starcoin-indexer-swap-main-deployment.yaml index 92d661f..a9cc56d 100644 --- a/kube/indexer/starcoin-indexer-swap-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-swap-main-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-txn-main-deployment.yaml b/kube/indexer/starcoin-indexer-txn-main-deployment.yaml index d4194d9..2bebdfb 100644 --- a/kube/indexer/starcoin-indexer-txn-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-txn-main-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-cmd-handle-barnard-deployment.yaml b/kube/indexer/starswap-info-main-deployment.yaml similarity index 69% rename from kube/indexer/starcoin-indexer-cmd-handle-barnard-deployment.yaml rename to kube/indexer/starswap-info-main-deployment.yaml index 530d9ab..bc57ce3 100644 --- a/kube/indexer/starcoin-indexer-cmd-handle-barnard-deployment.yaml +++ b/kube/indexer/starswap-info-main-deployment.yaml @@ -1,36 +1,34 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-cmd-handle-deployment - namespace: starcoin-barnard + name: starswap-info-deployment + namespace: starcoin-main labels: - app: starcoin-indexer-cmd-handle + app: starswap-info spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer-cmd-handle + app: starswap-info template: metadata: labels: - app: starcoin-indexer-cmd-handle + app: starswap-info spec: containers: - - name: starcoin-indexer-cmd-handle - image: starcoin/starcoin_indexer:sha-ba00b10 + - name: starswap-info + image: starcoin/starcoin_indexer:sha-a66e9ab ports: - containerPort: 8300 env: - name: HOSTS - value: "barnard.seed.starcoin.org" + value: "main.seed.starcoin.org" - name: NETWORK - value: "barnard.0727" + value: "main.0727" - name: BG_TASK_JOBS - value: "" + value: "swap_stats,price_hour,price_stat,swap_pool_fee_stat" - name: TXN_OFFSET value: "0" - - name: BULK_SIZE - value: "100" - name: STARCOIN_ES_PWD valueFrom: secretKeyRef: @@ -42,14 +40,14 @@ spec: value: "https" - name: STARCOIN_ES_PORT value: "443" - - name: STARCOIN_ES_USER - value: elastic + - name: BULK_SIZE + value: "100" - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-swap.czftx6muneg7.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD @@ -58,6 +56,6 @@ spec: name: postgresql-starcoin-yu key: password - name: DB_SCHEMA - value: "barnard" + value: "main" - name: PROGRAM_ARGS - value: "migrate_holder" + value: "" diff --git a/kube/scan-api/scan-api-deployment.yaml b/kube/scan-api/scan-api-deployment.yaml index 8b78da5..ee1a522 100644 --- a/kube/scan-api/scan-api-deployment.yaml +++ b/kube/scan-api/scan-api-deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: app: scan-api spec: - replicas: 3 + replicas: 2 selector: matchLabels: app: scan-api @@ -16,7 +16,7 @@ spec: spec: containers: - name: scan-api - image: starcoin/starcoin_scan:sha-3af1c2a + image: starcoin/starcoin_scan:sha-1223fc3 ports: - name: http containerPort: 8500 @@ -39,15 +39,15 @@ spec: name: elasticsearch-pw-elastic key: password - name: MAIN_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=main" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=main" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: BARNARD_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=barnard" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=barnard" - name: HALLEY_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=halley" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=halley" - name: STARCOIN_USER_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=starcoin_user" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=starcoin_user" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/scan-api/scan-test-api-deployment.yaml b/kube/scan-api/scan-test-api-deployment.yaml deleted file mode 100644 index 305eee7..0000000 --- a/kube/scan-api/scan-test-api-deployment.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: scan-test-api-deployment - labels: - app: scan-test-api -spec: - replicas: 1 - selector: - matchLabels: - app: scan-test-api - template: - metadata: - labels: - app: scan-test-api - spec: - containers: - - name: scan-test-api - image: starcoin/starcoin_scan:sha-3af1c2a - ports: - - name: http - containerPort: 8500 - - name: multicast - containerPort: 5701 - env: - - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" - - name: STARCOIN_ES_PROTOCOL - value: "https" - - name: STARCOIN_ES_PORT - value: "443" - - name: STARCOIN_ES_USER - value: elastic - - name: STARCOIN_ES_INDEX_VERSION - value: ".0727" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - - name: MAIN_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=main" - - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - - name: BARNARD_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=barnard" - - name: HALLEY_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=halley" - - name: STARCOIN_USER_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=starcoin_user" - - name: DB_USER_NAME - value: "starcoin" - - name: DB_PWD - valueFrom: - secretKeyRef: - name: postgresql-starcoin-yu - key: password - diff --git a/kube/scan-api/service-scan-test-api.yaml b/kube/scan-api/service-scan-test-api.yaml deleted file mode 100644 index 1bec2e7..0000000 --- a/kube/scan-api/service-scan-test-api.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: scan-test-api - name: scan-test-api -spec: - ports: - - port: 80 - name: http - protocol: TCP - targetPort: 8500 - selector: - app: scan-test-api - type: LoadBalancer -status: - loadBalancer: { } diff --git a/kube/scan-api/service-scan-test-hazelcast.yaml b/kube/scan-api/service-scan-test-hazelcast.yaml deleted file mode 100644 index 420513f..0000000 --- a/kube/scan-api/service-scan-test-hazelcast.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: scan-test-api - name: hazelcast-service -spec: - selector: - app: scan-test-api - ports: - - name: multicast - port: 5701 - type: LoadBalancer \ No newline at end of file diff --git a/kube/subscribe/starscan-sub-barnard-deployment.yaml b/kube/subscribe/starscan-sub-barnard-deployment.yaml index e93e3ff..9625dd4 100644 --- a/kube/subscribe/starscan-sub-barnard-deployment.yaml +++ b/kube/subscribe/starscan-sub-barnard-deployment.yaml @@ -45,7 +45,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/subscribe/starscan-sub-halley-deployment.yaml b/kube/subscribe/starscan-sub-halley-deployment.yaml index 6fa4e86..672ef7f 100644 --- a/kube/subscribe/starscan-sub-halley-deployment.yaml +++ b/kube/subscribe/starscan-sub-halley-deployment.yaml @@ -50,7 +50,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/subscribe/starscan-sub-main-deployment.yaml b/kube/subscribe/starscan-sub-main-deployment.yaml index c5870ef..6745106 100644 --- a/kube/subscribe/starscan-sub-main-deployment.yaml +++ b/kube/subscribe/starscan-sub-main-deployment.yaml @@ -45,7 +45,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/subscribe/starscan-sub-proxima-deployment.yaml b/kube/subscribe/starscan-sub-proxima-deployment.yaml index 7d3ce5b..e8e71f5 100644 --- a/kube/subscribe/starscan-sub-proxima-deployment.yaml +++ b/kube/subscribe/starscan-sub-proxima-deployment.yaml @@ -45,7 +45,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD From 1736b2d70e0880c4d19e98c15a4d015fa7065054 Mon Sep 17 00:00:00 2001 From: fountainchen Date: Thu, 12 Jan 2023 19:12:58 +0800 Subject: [PATCH 05/82] [config]update config for info site migration --- kube/indexer/starcoin-indexer-barnard-deployment.yaml | 2 +- .../starcoin-indexer-cmd-handle-main-deployment.yaml | 2 +- kube/indexer/starcoin-indexer-halley-deployment.yaml | 2 +- kube/indexer/starcoin-indexer-main-deployment.yaml | 4 ++-- kube/indexer/starcoin-indexer-proxima-deployment.yaml | 2 +- .../indexer/starcoin-indexer-repair-barnard-deployment.yaml | 2 +- kube/indexer/starcoin-indexer-repair-main-deployment.yaml | 2 +- kube/indexer/starcoin-indexer-swap-main-deployment.yaml | 6 +++--- kube/indexer/starcoin-indexer-txn-main-deployment.yaml | 2 +- kube/scan-api/scan-api-deployment.yaml | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/kube/indexer/starcoin-indexer-barnard-deployment.yaml b/kube/indexer/starcoin-indexer-barnard-deployment.yaml index 2acc41a..b666aba 100644 --- a/kube/indexer/starcoin-indexer-barnard-deployment.yaml +++ b/kube/indexer/starcoin-indexer-barnard-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml b/kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml index 06e87b3..6ea7a43 100644 --- a/kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-halley-deployment.yaml b/kube/indexer/starcoin-indexer-halley-deployment.yaml index 4798b30..5c6bdfd 100644 --- a/kube/indexer/starcoin-indexer-halley-deployment.yaml +++ b/kube/indexer/starcoin-indexer-halley-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-main-deployment.yaml b/kube/indexer/starcoin-indexer-main-deployment.yaml index 18300fe..5da5b23 100644 --- a/kube/indexer/starcoin-indexer-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-main-deployment.yaml @@ -26,7 +26,7 @@ spec: - name: NETWORK value: "main.0727" - name: BG_TASK_JOBS - value: "indexer,secondary,market_cap" + value: "indexer,secondary,market_cap,txn_payload" - name: TXN_OFFSET value: "0" - name: BULK_SIZE @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-proxima-deployment.yaml b/kube/indexer/starcoin-indexer-proxima-deployment.yaml index fa5c9ea..bf5911c 100644 --- a/kube/indexer/starcoin-indexer-proxima-deployment.yaml +++ b/kube/indexer/starcoin-indexer-proxima-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml b/kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml index 0a56319..4a46315 100644 --- a/kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml +++ b/kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-repair-main-deployment.yaml b/kube/indexer/starcoin-indexer-repair-main-deployment.yaml index 595e91a..b687e50 100644 --- a/kube/indexer/starcoin-indexer-repair-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-repair-main-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-swap-main-deployment.yaml b/kube/indexer/starcoin-indexer-swap-main-deployment.yaml index a9cc56d..dde80d6 100644 --- a/kube/indexer/starcoin-indexer-swap-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-swap-main-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: starcoin-indexer-swap - image: starcoin/starcoin_indexer:sha-a66e9ab + image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 env: @@ -26,7 +26,7 @@ spec: - name: NETWORK value: "main.0727" - name: BG_TASK_JOBS - value: "price_hour,price_stat" + value: "swap_transaction,swap_stats,price_hour,price_stat,swap_pool_fee_stat" - name: TXN_OFFSET value: "0" - name: BULK_SIZE @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/starcoin-indexer-txn-main-deployment.yaml b/kube/indexer/starcoin-indexer-txn-main-deployment.yaml index 2bebdfb..48fd2ba 100644 --- a/kube/indexer/starcoin-indexer-txn-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-txn-main-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/scan-api/scan-api-deployment.yaml b/kube/scan-api/scan-api-deployment.yaml index ee1a522..436cc20 100644 --- a/kube/scan-api/scan-api-deployment.yaml +++ b/kube/scan-api/scan-api-deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: app: scan-api spec: - replicas: 2 + replicas: 1 selector: matchLabels: app: scan-api From 91303e895c38df8d94b68047f775d92d43b44f4b Mon Sep 17 00:00:00 2001 From: Valentine Reese <100587939+ValentineReese@users.noreply.github.com> Date: Mon, 16 Oct 2023 03:37:59 -0500 Subject: [PATCH 06/82] Initial commit --- README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..a8ede14 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# stc-scan +starcoin stc data scan From 3517caf636137fb0ad2f3c14ed3d31753faa8a2a Mon Sep 17 00:00:00 2001 From: Valentine Reese <100587939+ValentineReese@users.noreply.github.com> Date: Mon, 16 Oct 2023 03:39:17 -0500 Subject: [PATCH 07/82] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a8ede14..83f1e44 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,6 @@ # stc-scan -starcoin stc data scan +Established some independent handles for indexing data with scan to customize statistics data for swap, used in conjunction with the [swap-stat-api](https://github.com/Elements-Studio/swap-stat-api) repository. + + + + From 815a633d1f52be17f3adcdd1cbcf317658ccd01f Mon Sep 17 00:00:00 2001 From: Valentine Reese <100587939+valentinereese@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:10:42 +0800 Subject: [PATCH 08/82] change db url --- kube/indexer/starcoin-indexer-stat-main-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/indexer/starcoin-indexer-stat-main-deployment.yaml b/kube/indexer/starcoin-indexer-stat-main-deployment.yaml index d397b7a..3d920b8 100644 --- a/kube/indexer/starcoin-indexer-stat-main-deployment.yaml +++ b/kube/indexer/starcoin-indexer-stat-main-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-swap.czftx6muneg7.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD From 6e347df557f3fa61d361bed774b45903da9a43bc Mon Sep 17 00:00:00 2001 From: bobong Date: Thu, 13 Jun 2024 21:28:01 +0800 Subject: [PATCH 09/82] [dag inspector api] add index code for dag inspector (cherry picked from commit b0201760a5dffdf81b5a12f7a061e682ec14aa1f) --- pom.xml | 2 +- .../java/org/starcoin/constant/Constant.java | 3 + starcoin-indexer/local.env | 18 ++ .../indexer/handler/DagInspectorHandler.java | 196 ++++++++++++++++++ .../controller/DagInspectorController.java | 22 ++ .../controller/TransactionController.java | 2 +- .../controller/TransactionV2Controller.java | 2 +- .../starcoin/scan/service/BlockService.java | 18 +- .../scan/service/DagInspectorService.java | 120 +++++++++++ .../scan/service/TransactionService.java | 1 + .../starcoin/scan/service/vo/DIAppConfig.java | 4 + .../org/starcoin/scan/service/vo/DIBlock.java | 108 ++++++++++ .../vo/DIBlocksAndEdgesAndHeightGroups.java | 36 ++++ .../org/starcoin/scan/service/vo/DIEdge.java | 52 +++++ .../scan/service/vo/DIHeightGroup.java | 22 ++ .../{ => vo}/TransactionWithEvent.java | 2 +- 16 files changed, 603 insertions(+), 5 deletions(-) create mode 100644 starcoin-indexer/local.env create mode 100644 starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java create mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java create mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java create mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfig.java create mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlock.java create mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroups.java create mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIEdge.java create mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIHeightGroup.java rename starcoin-scan-api/src/main/java/org/starcoin/scan/service/{ => vo}/TransactionWithEvent.java (90%) diff --git a/pom.xml b/pom.xml index 2f793f6..7b3a5a2 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.starcoin sdk - 1.1.25 + 1.2.1 org.springframework.boot diff --git a/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java b/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java index c1b57df..395b1b2 100644 --- a/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java +++ b/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java @@ -15,6 +15,9 @@ public class Constant { public static final String MARKET_CAP_INDEX = "market_cap"; public static final String PAYLOAD_INDEX = "txn_payloads"; public static final String TOKEN_INFO_INDEX = "token_info"; + public static final String DAG_INSPECTOR_NODE = "dag_inspector_node"; + public static final String DAG_INSPECTOR_EDGE = "dag_inspector_edge"; + public static final String DAG_INSPECT_HEIGHT_GROUP = "dag_inspector_height_group"; public static final String EVENT_FILTER_ADDRESS = "0x00000000000000000000000000000001"; diff --git a/starcoin-indexer/local.env b/starcoin-indexer/local.env new file mode 100644 index 0000000..534b7d2 --- /dev/null +++ b/starcoin-indexer/local.env @@ -0,0 +1,18 @@ +HOSTS=halley.seed.starcoin.org +NETWORK=halley.0727 +BG_TASK_JOBS=indexer,secondary,market_cap,txn_payload,txn_global_idx_update +TXN_OFFSET=0 +BULK_SIZE=100 +STARCOIN_ES_PWD= +STARCOIN_ES_URL=localhost +STARCOIN_ES_PROTOCOL=http +STARCOIN_ES_PORT=9200 +STARCOIN_ES_USER= +SWAP_API_URL=https://swap-api.starswap.xyz +SWAP_CONTRACT_ADDR=0x8c109349c6bd91411d6bc962e080c4a3 +DS_URL=jdbc:postgresql://localhost/starcoin +DB_SCHEMA=halley +DB_USER_NAME=kaspa +DB_PWD=kaspa +PROGRAM_ARGS= +# auto_repair 9411700 \ No newline at end of file diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java new file mode 100644 index 0000000..cebf59f --- /dev/null +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java @@ -0,0 +1,196 @@ +package org.starcoin.indexer.handler; + +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.starcoin.bean.*; +import org.starcoin.constant.Constant; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +@Service +public class DagInspectorHandler { + + private static final Logger logger = LoggerFactory.getLogger(DagInspectorHandler.class); + + @Value("${starcoin.network}") + private String network; + + String dagInspectNodeIndex; + String dagInspectEdgeIndex; + String dagInspectHeightGroupIndex; + + @Autowired + private RestHighLevelClient client; + + @PostConstruct + public void initIndexs() throws IOException { + dagInspectNodeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_NODE); + dagInspectEdgeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_EDGE); + dagInspectHeightGroupIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECT_HEIGHT_GROUP); + } + + public void upsertDagInfoFromBlocks(List blockList) { + if (blockList.isEmpty()) { + logger.warn("block list is empty"); + return; + } + BulkRequest bulkRequest = new BulkRequest(); + + blockList.forEach(block -> { + bulkRequest.add(buildNodeRequest(block)); + bulkRequest.add(buildEdgeRequest(block, blockList)); + bulkRequest.add(buildHeightGroupRequest(block)); + }); + + try { + BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT); + logger.info("bulk block result: {}", response.buildFailureMessage()); + } catch (IOException e) { + logger.error("bulk block error:", e); + } + } + + /*** + * Build node index request, include block hash, parents hash, colors, etc. + * @param block + * @return IndexRequest + */ + private IndexRequest buildNodeRequest(Block block) { + IndexRequest request = new IndexRequest(dagInspectNodeIndex); + XContentBuilder builder; + try { + builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.field("block_hash", block.getHeader().getBlockHash()); + builder.field("parents_hash", block.getHeader().getParentsHash()); + // TODO(BobOng): get colors from nodes + } + builder.endObject(); + } catch (Exception e) { + logger.error("build block error:", e); + return null; + } + request.id(String.valueOf(block.getHeader().getBlockHash())).source(builder); + return request; + } + + /** + * Build edge data into elasticsearch, include from_block_id, to_block_id, from_height, to_height, etc. + * + * @param block block from chain data + * @return IndexRequest + */ + private IndexRequest buildEdgeRequest(Block block, List blockList) { + IndexRequest request = new IndexRequest(dagInspectEdgeIndex); + XContentBuilder builder; + try { + builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + buildEdgeFieldFromBlock(builder, block, blockList); + } + builder.endObject(); + } catch (Exception e) { + logger.error("build block error:", e); + return null; + } + request.id(String.valueOf(block.getHeader().getBlockHash())).source(builder); + return request; + } + + private void buildEdgeFieldFromBlockPair(XContentBuilder builder, Block block, Block parentBlock) throws IOException { + builder.startObject(); + + builder.field("from_block_id", block.getHeader().getBlockHash()); + builder.field("to_block_id", parentBlock.getHeader().getBlockHash()); + builder.field("from_height", block.getHeader().getHeight()); + builder.field("to_height", parentBlock.getHeader().getHeight()); + builder.field("to_height_group_index", 0); + + builder.endObject(); + } + + /** + * Build edge field from block, include from_block_id, to_block_id, from_height, to_height, etc. + * + * @param builder + * @param currentBlock + * @param blockList + * @throws IOException + */ + private void buildEdgeFieldFromBlock(XContentBuilder builder, Block currentBlock, List blockList) throws IOException { + BlockHeader currentBlockHeader = currentBlock.getHeader(); + + String parentHash = currentBlockHeader.getParentHash(); + Block parentBlock = blockList + .stream() + .filter(b -> b.getHeader().getBlockHash().equalsIgnoreCase(parentHash)) + .findFirst() + .orElse(null); + if (parentBlock == null) { + // TODO(BobOng): Get parent block from chain data + logger.info("Need to getting the parent block from chain data, parent hash: {}", parentHash); + } + buildEdgeFieldFromBlockPair(builder, currentBlock, parentBlock); + + if (currentBlockHeader.getParentsHash().isEmpty()) { + return; + } + + List filteredParents = currentBlockHeader + .getParentsHash() + .stream() + .filter(h -> h.equalsIgnoreCase(parentHash)) + .collect(Collectors.toList()); + + if (filteredParents.isEmpty()) { + return; + } + + List parentBlocks = blockList + .stream() + .filter(b -> filteredParents.contains(b.getHeader().getBlockHash())) + .collect(Collectors.toList()); + for (Block parentBLock : parentBlocks) { + buildEdgeFieldFromBlockPair(builder, currentBlock, parentBLock); + } + } + + private IndexRequest buildHeightGroupRequest(Block block) { + IndexRequest request = new IndexRequest(dagInspectHeightGroupIndex); + XContentBuilder builder = null; + try { + builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field("height", block.getHeader().getHeight()); + +// builder.field("from_block_id", block.getHeader().getBlockHash()); +// builder.field("to_block_id", parentBlock.getHeader().getBlockHash()); +// builder.field("from_height", block.getHeader().getHeight()); +// builder.field("to_height", parentBlock.getHeader().getHeight()); +// builder.field("to_height_group_index", 0); + builder.endObject(); + + } catch (Exception e) { + logger.error("build block error:", e); + } + request.source(builder); + return request; + } + + +} diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java new file mode 100644 index 0000000..b28d8f9 --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java @@ -0,0 +1,22 @@ +package org.starcoin.scan.controller; + +import io.swagger.annotations.Api; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import org.starcoin.bean.Block; +import org.starcoin.scan.service.DagInspectorService; + +@Api(tags = "dag-inspector") +@RestController +@RequestMapping("v2/dag-inspector") +public class DagInspectorController { + + @Autowired + DagInspectorService dagInspectorService; + + @GetMapping("/{network}/") + public Block getBlock(@PathVariable("network") String network, @RequestParam String id) throws Exception { + // return dagInspectorService.getBlock(network, id); + return null; + } +} diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionController.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionController.java index 12ae212..97c4216 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionController.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionController.java @@ -6,7 +6,7 @@ import org.starcoin.bean.Event; import org.starcoin.bean.PendingTransaction; import org.starcoin.scan.service.TransactionService; -import org.starcoin.scan.service.TransactionWithEvent; +import org.starcoin.scan.service.vo.TransactionWithEvent; import java.io.IOException; diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionV2Controller.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionV2Controller.java index 85e926b..ccd33bf 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionV2Controller.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionV2Controller.java @@ -10,7 +10,7 @@ import org.starcoin.bean.TokenTransfer; import org.starcoin.bean.Transfer; import org.starcoin.scan.service.TransactionService; -import org.starcoin.scan.service.TransactionWithEvent; +import org.starcoin.scan.service.vo.TransactionWithEvent; import java.io.IOException; diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java index ae57d1d..19ccf04 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java @@ -7,7 +7,9 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; @@ -19,6 +21,7 @@ import org.starcoin.bean.Block; import org.starcoin.bean.UncleBlock; import org.starcoin.constant.Constant; +import org.starcoin.scan.service.vo.DIEdge; import java.io.IOException; import java.util.List; @@ -215,5 +218,18 @@ public Result getUnclesRange(String network, int page, int count, in return ServiceUtils.getSearchResult(searchResponse, UncleBlock.class); } - + public List getBlocksBetweenHeights(String network, Integer startHeight, Integer endHeight) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.BLOCK_CONTENT_INDEX)); + RangeQueryBuilder termQueryBuilder = QueryBuilders.rangeQuery("header.number").gte(startHeight).lte(endHeight); + searchRequest.source(new SearchSourceBuilder().query(termQueryBuilder).sort("header.number", SortOrder.ASC)); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("getBlocksBetweenHeights failed, startHeight: {}, endHeight: {}", startHeight, endHeight, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, Block.class); + return result.getContents(); + } } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java new file mode 100644 index 0000000..97b0732 --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -0,0 +1,120 @@ +package org.starcoin.scan.service; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.starcoin.api.Result; +import org.starcoin.bean.Block; +import org.starcoin.constant.Constant; +import org.starcoin.scan.service.vo.*; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +@Service +public class DagInspectorService extends BaseService { + + private static final Logger logger = LoggerFactory.getLogger(BlockService.class); + + @Autowired + BlockService blockService; + + @Autowired + private RestHighLevelClient client; + + public DIBlocksAndEdgesAndHeightGroups getBlocksBetweenHeights(String network, Integer startHeight, Integer endHeight) { + List blockList = blockService.getBlocksBetweenHeights(network, startHeight, endHeight); + + DIBlocksAndEdgesAndHeightGroups groups = new DIBlocksAndEdgesAndHeightGroups(); + groups.setBlocks(blockList.stream().map(block -> { + DIBlock diBlock = new DIBlock(); + diBlock.setBlockHash(block.getHeader().getBlockHash()); + diBlock.setHeight(block.getHeader().getHeight()); + diBlock.setColor("blue"); + diBlock.setInVirtualSelectedParentChain(false); + diBlock.setParentIds(block.getHeader().getParentsHash()); + + // TODO(BobOng): Get selected parent id from chain data + // diBlock.setSelectedParentHash(); + // diBlock.setHeightGroupIndex(); + // diBlock.setDaaScore(); + return diBlock; + }).collect(Collectors.toList())); + + groups.setEdges(getEdgeList(network, startHeight, endHeight)); + + List heights = blockList + .stream() + .map(block -> block.getHeader().getHeight()) + .collect(Collectors.toList()); + + List diHeightGroup = getHeightGroup(network, heights); + groups.setHeightGroups(diHeightGroup); + + return groups; + } + + public DIBlocksAndEdgesAndHeightGroups getBlockHash(String targetHash, Integer heightDifference) { + return new DIBlocksAndEdgesAndHeightGroups(); + } + + public DIBlocksAndEdgesAndHeightGroups getBlockDAAScore(Integer targetDAAScore, Integer heightDifference) { + return new DIBlocksAndEdgesAndHeightGroups(); + } + + public DIBlocksAndEdgesAndHeightGroups getHead(Integer heightDifference) { + return new DIBlocksAndEdgesAndHeightGroups(); + } + + List getEdgeList(String network, Integer startHeight, Integer endHeight) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_EDGE)); + RangeQueryBuilder fromHeightQuery = QueryBuilders.rangeQuery("from_height").gte(startHeight); + RangeQueryBuilder toHeightQuery = QueryBuilders.rangeQuery("to_height").lte(endHeight); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(QueryBuilders.boolQuery().must(fromHeightQuery).must(toHeightQuery)).sort("to_height", SortOrder.ASC); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("getEdgeList failed, startHeight: {}, endHeight: {}", startHeight, endHeight, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, DIEdge.class); + return result.getContents(); + } + + List getHeightGroup(String network, List heights) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECT_HEIGHT_GROUP)); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + for (Long height : heights) { + boolQueryBuilder.should(QueryBuilders.termQuery("height", height)); + } + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("getHeightGroup failed, heights: {}", heights, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, DIHeightGroup.class); + return result.getContents(); + } + + public DIAppConfig getAppConfig() { + return new DIAppConfig(); + } + +} diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java index 1833162..44fad73 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java @@ -25,6 +25,7 @@ import org.starcoin.api.Result; import org.starcoin.bean.*; import org.starcoin.constant.Constant; +import org.starcoin.scan.service.vo.TransactionWithEvent; import org.starcoin.types.AccountAddress; import org.starcoin.types.event.ProposalCreatedEvent; import org.starcoin.utils.ByteUtils; diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfig.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfig.java new file mode 100644 index 0000000..dc01f4c --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfig.java @@ -0,0 +1,4 @@ +package org.starcoin.scan.service.vo; + +public class DIAppConfig { +} diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlock.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlock.java new file mode 100644 index 0000000..1551458 --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlock.java @@ -0,0 +1,108 @@ +package org.starcoin.scan.service.vo; + +import java.util.Vector; + +/** + * Starcoin DAG Inspector Block info + */ +public class DIBlock { + String blockHash; + Integer timestamp; + Vector parentIds; + Long height; + Integer daaScore; + Integer heightGroupIndex; + String selectedParentHash; + String color; + Boolean isInVirtualSelectedParentChain; + Vector mergeSetRedIds; + Vector mergeSetBlueIds; + + public String getBlockHash() { + return blockHash; + } + + public void setBlockHash(String blockHash) { + this.blockHash = blockHash; + } + + public Integer getTimestamp() { + return timestamp; + } + + public void setTimestamp(Integer timestamp) { + this.timestamp = timestamp; + } + + public Vector getParentIds() { + return parentIds; + } + + public void setParentIds(Vector parentIds) { + this.parentIds = parentIds; + } + + public Long getHeight() { + return height; + } + + public void setHeight(Long height) { + this.height = height; + } + + public Integer getDaaScore() { + return daaScore; + } + + public void setDaaScore(Integer daaScore) { + this.daaScore = daaScore; + } + + public Integer getHeightGroupIndex() { + return heightGroupIndex; + } + + public void setHeightGroupIndex(Integer heightGroupIndex) { + this.heightGroupIndex = heightGroupIndex; + } + + public String getSelectedParentHash() { + return selectedParentHash; + } + + public void setSelectedParentHash(String selectedParentHash) { + this.selectedParentHash = selectedParentHash; + } + + public String getColor() { + return color; + } + + public void setColor(String color) { + this.color = color; + } + + public Boolean getInVirtualSelectedParentChain() { + return isInVirtualSelectedParentChain; + } + + public void setInVirtualSelectedParentChain(Boolean inVirtualSelectedParentChain) { + isInVirtualSelectedParentChain = inVirtualSelectedParentChain; + } + + public Vector getMergeSetRedIds() { + return mergeSetRedIds; + } + + public void setMergeSetRedIds(Vector mergeSetRedIds) { + this.mergeSetRedIds = mergeSetRedIds; + } + + public Vector getMergeSetBlueIds() { + return mergeSetBlueIds; + } + + public void setMergeSetBlueIds(Vector mergeSetBlueIds) { + this.mergeSetBlueIds = mergeSetBlueIds; + } +} \ No newline at end of file diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroups.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroups.java new file mode 100644 index 0000000..b3fb61f --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroups.java @@ -0,0 +1,36 @@ +package org.starcoin.scan.service.vo; + +import java.util.List; + +/** + * DAG Inspector query object + */ +public class DIBlocksAndEdgesAndHeightGroups { + List blocks; + List edges; + List heightGroups; + + public List getBlocks() { + return blocks; + } + + public void setBlocks(List blocks) { + this.blocks = blocks; + } + + public List getEdges() { + return edges; + } + + public void setEdges(List edges) { + this.edges = edges; + } + + public List getHeightGroups() { + return heightGroups; + } + + public void setHeightGroups(List heightGroups) { + this.heightGroups = heightGroups; + } +} \ No newline at end of file diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIEdge.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIEdge.java new file mode 100644 index 0000000..1b0b925 --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIEdge.java @@ -0,0 +1,52 @@ +package org.starcoin.scan.service.vo; + +/** + * Starcoin DAG Inspector Edge info + */ +public class DIEdge { + String fromBlockId; + String toBlockId; + Integer fromHeight; + Integer toHeight; + Integer toHeightGroupIndex; + + public String getFromBlockId() { + return fromBlockId; + } + + public void setFromBlockId(String fromBlockId) { + this.fromBlockId = fromBlockId; + } + + public String getToBlockId() { + return toBlockId; + } + + public void setToBlockId(String toBlockId) { + this.toBlockId = toBlockId; + } + + public Integer getFromHeight() { + return fromHeight; + } + + public void setFromHeight(Integer fromHeight) { + this.fromHeight = fromHeight; + } + + public Integer getToHeight() { + return toHeight; + } + + public void setToHeight(Integer toHeight) { + this.toHeight = toHeight; + } + + public Integer getToHeightGroupIndex() { + return toHeightGroupIndex; + } + + public void setToHeightGroupIndex(Integer toHeightGroupIndex) { + this.toHeightGroupIndex = toHeightGroupIndex; + } +} \ No newline at end of file diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIHeightGroup.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIHeightGroup.java new file mode 100644 index 0000000..b5d6f59 --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIHeightGroup.java @@ -0,0 +1,22 @@ +package org.starcoin.scan.service.vo; + +public class DIHeightGroup { + Integer height; + Integer size; + + public Integer getHeight() { + return height; + } + + public void setHeight(Integer height) { + this.height = height; + } + + public Integer getSize() { + return size; + } + + public void setSize(Integer size) { + this.size = size; + } +} diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionWithEvent.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/TransactionWithEvent.java similarity index 90% rename from starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionWithEvent.java rename to starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/TransactionWithEvent.java index e5d90f5..4397a35 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionWithEvent.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/TransactionWithEvent.java @@ -1,4 +1,4 @@ -package org.starcoin.scan.service; +package org.starcoin.scan.service.vo; import org.starcoin.bean.Event; import org.starcoin.bean.Transaction; From 7f5d48c0bbf19150125155921135144e4894f63a Mon Sep 17 00:00:00 2001 From: bobong Date: Sat, 15 Jun 2024 16:16:07 +0800 Subject: [PATCH 10/82] [dag inspector api] Use the new processing logic. The difference between this logic and Kaspa Processing is that it first takes part of the block, and then calculates the inspection drawing information based on the data of this part of the block. (cherry picked from commit 2aadab7662ff43114247db2b1674811feadf9836) --- pom.xml | 2 +- .../org/starcoin/bean/DagInspectorBlock.java | 106 +++++ .../org/starcoin/bean/DagInspectorEdge.java | 58 +++ .../bean/DagInspectorHeightGroup.java | 22 ++ .../indexer/handler/DagInspectorHandler.java | 374 +++++++++++++----- .../indexer/handler/ElasticSearchHandler.java | 5 +- 6 files changed, 464 insertions(+), 103 deletions(-) create mode 100644 starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java create mode 100644 starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java create mode 100644 starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java diff --git a/pom.xml b/pom.xml index 7b3a5a2..7eb3b17 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.starcoin sdk - 1.2.1 + 1.2.2 org.springframework.boot diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java new file mode 100644 index 0000000..82fd261 --- /dev/null +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java @@ -0,0 +1,106 @@ +package org.starcoin.bean; + +import java.util.List; +import java.util.List; + +public class DagInspectorBlock { + String blockHash; + Long timestamp; + List parentIds; + Long height; + Long daaScore; + Integer heightGroupIndex; + String selectedParentHash; + String color; + Boolean isInVirtualSelectedParentChain; + List mergeSetRedIds; + List mergeSetBlueIds; + + public String getBlockHash() { + return blockHash; + } + + public void setBlockHash(String blockHash) { + this.blockHash = blockHash; + } + + public Long getTimestamp() { + return timestamp; + } + + public void setTimestamp(Long timestamp) { + this.timestamp = timestamp; + } + + public List getParentIds() { + return parentIds; + } + + public void setParentIds(List parentIds) { + this.parentIds = parentIds; + } + + public Long getHeight() { + return height; + } + + public void setHeight(Long height) { + this.height = height; + } + + public Long getDaaScore() { + return daaScore; + } + + public void setDaaScore(Long daaScore) { + this.daaScore = daaScore; + } + + public Integer getHeightGroupIndex() { + return heightGroupIndex; + } + + public void setHeightGroupIndex(Integer heightGroupIndex) { + this.heightGroupIndex = heightGroupIndex; + } + + public String getSelectedParentHash() { + return selectedParentHash; + } + + public void setSelectedParentHash(String selectedParentHash) { + this.selectedParentHash = selectedParentHash; + } + + public String getColor() { + return color; + } + + public void setColor(String color) { + this.color = color; + } + + public Boolean getInVirtualSelectedParentChain() { + return isInVirtualSelectedParentChain; + } + + public void setInVirtualSelectedParentChain(Boolean inVirtualSelectedParentChain) { + isInVirtualSelectedParentChain = inVirtualSelectedParentChain; + } + + public List getMergeSetRedIds() { + return mergeSetRedIds; + } + + public void setMergeSetRedIds(List mergeSetRedIds) { + this.mergeSetRedIds = mergeSetRedIds; + } + + public List getMergeSetBlueIds() { + return mergeSetBlueIds; + } + + public void setMergeSetBlueIds(List mergeSetBlueIds) { + this.mergeSetBlueIds = mergeSetBlueIds; + } +} diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java new file mode 100644 index 0000000..7d31995 --- /dev/null +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java @@ -0,0 +1,58 @@ +package org.starcoin.bean; + +public class DagInspectorEdge { + String fromBlockHash; + String toBlockHash; + Long fromHeight; + Long toHeight; + Integer fromHeightGroupIndex; + Integer toHeightGroupIndex; + + public String getFromBlockHash() { + return fromBlockHash; + } + + public void setFromBlockHash(String fromBlockHash) { + this.fromBlockHash = fromBlockHash; + } + + public String getToBlockHash() { + return toBlockHash; + } + + public void setToBlockHash(String toBlockHash) { + this.toBlockHash = toBlockHash; + } + + public Long getFromHeight() { + return fromHeight; + } + + public void setFromHeight(Long fromHeight) { + this.fromHeight = fromHeight; + } + + public Long getToHeight() { + return toHeight; + } + + public void setToHeight(Long toHeight) { + this.toHeight = toHeight; + } + + public Integer getFromHeightGroupIndex() { + return fromHeightGroupIndex; + } + + public void setFromHeightGroupIndex(Integer fromHeightGroupIndex) { + this.fromHeightGroupIndex = fromHeightGroupIndex; + } + + public Integer getToHeightGroupIndex() { + return toHeightGroupIndex; + } + + public void setToHeightGroupIndex(Integer toHeightGroupIndex) { + this.toHeightGroupIndex = toHeightGroupIndex; + } +} diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java new file mode 100644 index 0000000..4191534 --- /dev/null +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java @@ -0,0 +1,22 @@ +package org.starcoin.bean; + +public class DagInspectorHeightGroup { + Long height; + Integer size; + + public Long getHeight() { + return height; + } + + public void setHeight(Long height) { + this.height = height; + } + + public Integer getSize() { + return size; + } + + public void setSize(Integer size) { + this.size = size; + } +} diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java index cebf59f..0f70186 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java @@ -1,23 +1,33 @@ package org.starcoin.indexer.handler; +import com.alibaba.fastjson.JSONObject; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; +import org.starcoin.api.BlockRPCClient; import org.starcoin.bean.*; import org.starcoin.constant.Constant; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import javax.annotation.PostConstruct; import java.io.IOException; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; @Service @@ -25,6 +35,10 @@ public class DagInspectorHandler { private static final Logger logger = LoggerFactory.getLogger(DagInspectorHandler.class); + private static final String NODE_COLOR_GRAY = "gray"; + private static final String NODE_COLOR_RED = "red"; + private static final String NODE_COLOR_BLUE = "blue"; + @Value("${starcoin.network}") private String network; @@ -35,6 +49,9 @@ public class DagInspectorHandler { @Autowired private RestHighLevelClient client; + @Autowired + private BlockRPCClient blockRPCClient; + @PostConstruct public void initIndexs() throws IOException { dagInspectNodeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_NODE); @@ -42,19 +59,66 @@ public void initIndexs() throws IOException { dagInspectHeightGroupIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECT_HEIGHT_GROUP); } - public void upsertDagInfoFromBlocks(List blockList) { + public void upsertDagInfoFromBlocks(List blockList) throws IOException { if (blockList.isEmpty()) { logger.warn("block list is empty"); return; } BulkRequest bulkRequest = new BulkRequest(); - blockList.forEach(block -> { - bulkRequest.add(buildNodeRequest(block)); - bulkRequest.add(buildEdgeRequest(block, blockList)); - bulkRequest.add(buildHeightGroupRequest(block)); + // Handle blockGhostdagDataCache + Map inspBlockMap = bulkLoadInspectorBlock( + blockList.stream() + .map(block -> block.getHeader().getBlockHash()) + .collect(Collectors.toList()) + ); + + List inspHeightGroupList = getGroupHeightSizeFromStorage( + blockList.stream() + .map(block -> block.getHeader().getHeight()) + .distinct() + .collect(Collectors.toList()) + ); + + // Build block node data + blockList.forEach(blockInfo -> { + String currentBlockHash = blockInfo.getHeader().getBlockHash(); + DagInspectorBlock inspecBlock = inspBlockMap.get(currentBlockHash); + if (inspecBlock == null) { + try { + BlockGhostdagData ghostdagData = blockRPCClient.getBlockGhostdagData(currentBlockHash); + inspecBlock = new DagInspectorBlock(); + inspecBlock.setBlockHash(currentBlockHash); + inspecBlock.setTimestamp(blockInfo.getHeader().getTimestamp()); + inspecBlock.setColor(NODE_COLOR_GRAY); + inspecBlock.setDaaScore(ghostdagData.getBlueScore()); + inspecBlock.setHeight(blockInfo.getHeader().getHeight()); + inspecBlock.setSelectedParentHash(ghostdagData.getSelectedParent()); + inspecBlock.setParentIds(blockInfo.getHeader().getParentsHash()); + + Integer groupSize = getHeightGroupSizeOrDefault(inspHeightGroupList, inspecBlock.getHeight(), 0); + inspecBlock.setHeightGroupIndex(groupSize); + updateGroupSize(inspHeightGroupList, inspecBlock.getHeight(), groupSize + 1); + + inspBlockMap.put(currentBlockHash, inspecBlock); + + } catch (JSONRPC2SessionException e) { + throw new RuntimeException(e); + } + } else { + logger.info("Block {} already exists", currentBlockHash); + } }); + List inspectorBlockList = new ArrayList<>(inspBlockMap.values()); + List edgeList = buildEdgeDataFromNodeData(inspectorBlockList); + + // Save all data into storage + bulkRequest.add(buildSaveInspectorBlockRequest(inspectorBlockList)); + bulkRequest.add(buildSaveEdgeRequest(edgeList)); + bulkRequest.add(buildSaveHeightGroupRequest(inspHeightGroupList)); + + // Bulk save try { BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT); logger.info("bulk block result: {}", response.buildFailureMessage()); @@ -63,126 +127,226 @@ public void upsertDagInfoFromBlocks(List blockList) { } } - /*** - * Build node index request, include block hash, parents hash, colors, etc. - * @param block - * @return IndexRequest - */ - private IndexRequest buildNodeRequest(Block block) { - IndexRequest request = new IndexRequest(dagInspectNodeIndex); - XContentBuilder builder; - try { - builder = XContentFactory.jsonBuilder(); - builder.startObject(); - { - builder.field("block_hash", block.getHeader().getBlockHash()); - builder.field("parents_hash", block.getHeader().getParentsHash()); - // TODO(BobOng): get colors from nodes + List buildEdgeDataFromNodeData(List inspecBlockList) { + List edgeList = new ArrayList<>(); + if (inspecBlockList.isEmpty()) { + return edgeList; + } + + for (DagInspectorBlock block : inspecBlockList) { + if (block.getParentIds().isEmpty()) { + continue; } - builder.endObject(); - } catch (Exception e) { - logger.error("build block error:", e); - return null; + + block.getParentIds().forEach(parentHash -> { + DagInspectorBlock parentBlock = inspecBlockList + .stream() + .filter(b -> b.getBlockHash().equalsIgnoreCase(parentHash)) + .findFirst() + .orElse(null); + if (parentBlock == null) { + logger.info("Parent block {} not found", parentHash); + return; + } + DagInspectorEdge edge = new DagInspectorEdge(); + edge.setFromBlockHash(block.getBlockHash()); + edge.setToBlockHash(parentBlock.getBlockHash()); + edge.setFromHeight(block.getHeight()); + edge.setToHeight(parentBlock.getHeight()); + edge.setFromHeightGroupIndex(block.getHeightGroupIndex()); + edge.setToHeightGroupIndex(parentBlock.getHeightGroupIndex()); + edgeList.add(edge); + }); } - request.id(String.valueOf(block.getHeader().getBlockHash())).source(builder); - return request; + return edgeList; } - /** - * Build edge data into elasticsearch, include from_block_id, to_block_id, from_height, to_height, etc. - * - * @param block block from chain data - * @return IndexRequest - */ - private IndexRequest buildEdgeRequest(Block block, List blockList) { - IndexRequest request = new IndexRequest(dagInspectEdgeIndex); - XContentBuilder builder; + public Map bulkLoadInspectorBlock(List blockHashList) { + Map inspBlockMap = new HashMap<>(); + SearchRequest searchRequest = new SearchRequest(dagInspectNodeIndex); + TermsQueryBuilder termQueryBuilder = QueryBuilders.termsQuery("block_hash", blockHashList); + searchRequest.source(new SearchSourceBuilder().query(termQueryBuilder)); try { - builder = XContentFactory.jsonBuilder(); - builder.startObject(); - { - buildEdgeFieldFromBlock(builder, block, blockList); - } - builder.endObject(); - } catch (Exception e) { - logger.error("build block error:", e); - return null; + SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT); + JSONObject obj = JSONObject.parseObject(ServiceUtils.getJsonString(response)); + List hits = obj.getJSONObject("hits").getJSONArray("hits").toJavaList(JSONObject.class); + hits.forEach(hit -> { + JSONObject source = hit.getJSONObject("_source"); + DagInspectorBlock block = new DagInspectorBlock(); + block.setBlockHash(source.getString("block_hash")); + block.setTimestamp(source.getLong("timestamp")); + block.setColor(source.getString("color")); + block.setDaaScore(source.getLong("daa_score")); + block.setHeight(source.getLong("height")); + block.setHeightGroupIndex(source.getInteger("height_group_index")); + block.setSelectedParentHash(source.getString("selected_parent_hash")); + block.setParentIds(source.getJSONArray("parent_ids").toJavaList(String.class)); + block.setInVirtualSelectedParentChain(source.getBoolean("in_virtual_selected_parent_chain")); + block.setMergeSetRedIds(source.getJSONArray("mergeset_red_ids").toJavaList(String.class)); + block.setMergeSetBlueIds(source.getJSONArray("mergeset_blue_ids").toJavaList(String.class)); + inspBlockMap.put(block.getBlockHash(), block); + }); + } catch (IOException e) { + logger.error("bulkLoadInspectorBlock error:", e); } - request.id(String.valueOf(block.getHeader().getBlockHash())).source(builder); - return request; + return inspBlockMap; } - private void buildEdgeFieldFromBlockPair(XContentBuilder builder, Block block, Block parentBlock) throws IOException { - builder.startObject(); - - builder.field("from_block_id", block.getHeader().getBlockHash()); - builder.field("to_block_id", parentBlock.getHeader().getBlockHash()); - builder.field("from_height", block.getHeader().getHeight()); - builder.field("to_height", parentBlock.getHeader().getHeight()); - builder.field("to_height_group_index", 0); - - builder.endObject(); + IndexRequest buildSaveInspectorBlockRequest(List blockList) { + IndexRequest request = new IndexRequest(dagInspectNodeIndex); + if (blockList.isEmpty()) { + return request; + } + blockList.forEach(block -> { + XContentBuilder builder; + try { + builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.field("block_hash", block.getBlockHash()); + builder.field("height", block.getHeight()); + builder.field("timestamp", block.getTimestamp()); + builder.field("selected_parent_hash", block.getSelectedParentHash()); + builder.field("parent_ids", block.getParentIds()); + builder.field("daa_score", block.getDaaScore()); + builder.field("height_group_index", block.getHeightGroupIndex()); + builder.field("in_virtual_selected_parent_chain", block.getInVirtualSelectedParentChain()); + builder.field("mergeset_blue_ids", block.getMergeSetBlueIds()); + builder.field("mergeset_red_ids", block.getMergeSetRedIds()); + builder.field("color", block.getColor()); + } + builder.endObject(); + } catch (Exception e) { + logger.error("build block error:", e); + } + }); + return request; } - /** - * Build edge field from block, include from_block_id, to_block_id, from_height, to_height, etc. - * - * @param builder - * @param currentBlock - * @param blockList - * @throws IOException - */ - private void buildEdgeFieldFromBlock(XContentBuilder builder, Block currentBlock, List blockList) throws IOException { - BlockHeader currentBlockHeader = currentBlock.getHeader(); - - String parentHash = currentBlockHeader.getParentHash(); - Block parentBlock = blockList - .stream() - .filter(b -> b.getHeader().getBlockHash().equalsIgnoreCase(parentHash)) - .findFirst() - .orElse(null); - if (parentBlock == null) { - // TODO(BobOng): Get parent block from chain data - logger.info("Need to getting the parent block from chain data, parent hash: {}", parentHash); + IndexRequest buildSaveEdgeRequest(List edgeList) { + IndexRequest request = new IndexRequest(dagInspectEdgeIndex); + if (edgeList.isEmpty()) { + return request; } - buildEdgeFieldFromBlockPair(builder, currentBlock, parentBlock); + edgeList.forEach(edge -> { + XContentBuilder builder; + try { + builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.field("from_block_hash", edge.getFromBlockHash()); + builder.field("to_block_hash", edge.getToBlockHash()); + builder.field("from_height", edge.getFromHeight()); + builder.field("to_height", edge.getToHeight()); + builder.field("from_group_index", edge.getFromHeightGroupIndex()); + builder.field("to_group_index", edge.getToHeightGroupIndex()); + } + builder.endObject(); + } catch (Exception e) { + logger.error("build block error:", e); + } + }); + return request; + } - if (currentBlockHeader.getParentsHash().isEmpty()) { - return; + IndexRequest buildSaveHeightGroupRequest(List heightGroupList) { + IndexRequest request = new IndexRequest(dagInspectHeightGroupIndex); + if (heightGroupList.isEmpty()) { + return request; } + heightGroupList.forEach(heightGroup -> { + XContentBuilder builder; + try { + builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.field("height", heightGroup.getHeight()); + builder.field("size", heightGroup.getSize()); + } + builder.endObject(); + } catch (Exception e) { + logger.error("build block error:", e); + } + }); + return request; + } - List filteredParents = currentBlockHeader - .getParentsHash() - .stream() - .filter(h -> h.equalsIgnoreCase(parentHash)) - .collect(Collectors.toList()); - - if (filteredParents.isEmpty()) { - return; + Integer getHeightGroupSizeOrDefault(List groupList, Long height, Integer defaultSize) { + if (groupList.isEmpty()) { + return defaultSize; } - - List parentBlocks = blockList + DagInspectorHeightGroup group = groupList .stream() - .filter(b -> filteredParents.contains(b.getHeader().getBlockHash())) - .collect(Collectors.toList()); - for (Block parentBLock : parentBlocks) { - buildEdgeFieldFromBlockPair(builder, currentBlock, parentBLock); + .filter(g -> g.getHeight().longValue() == height) + .findFirst() + .orElse(null); + return (group != null) ? group.getSize() : defaultSize; + } + + void updateGroupSize(List groupList, Long height, Integer newSize) { + for (DagInspectorHeightGroup group : groupList) { + if (group.getHeight().longValue() == height) { + group.setSize(newSize); + break; + } } } +// /** +// * Build edge field from block, include from_block_id, to_block_id, from_height, to_height, etc. +// * +// * @param builder +// * @param currentBlock +// * @param blockList +// * @throws IOException +// */ +// private void buildEdgeFieldFromBlock(XContentBuilder builder, Block currentBlock, List blockList) throws IOException { +// BlockHeader currentBlockHeader = currentBlock.getHeader(); +// +// String parentHash = currentBlockHeader.getParentHash(); +// Block parentBlock = blockList +// .stream() +// .filter(b -> b.getHeader().getBlockHash().equalsIgnoreCase(parentHash)) +// .findFirst() +// .orElse(null); +// if (parentBlock == null) { +// // TODO(BobOng): Get parent block from chain data +// logger.info("Need to getting the parent block from chain data, parent hash: {}", parentHash); +// } +// buildEdgeFieldFromBlockPair(builder, currentBlock, parentBlock); +// +// if (currentBlockHeader.getParentsHash().isEmpty()) { +// return; +// } +// +// List filteredParents = currentBlockHeader +// .getParentsHash() +// .stream() +// .filter(h -> h.equalsIgnoreCase(parentHash)) +// .collect(Collectors.toList()); +// +// if (filteredParents.isEmpty()) { +// return; +// } +// +// List parentBlocks = blockList +// .stream() +// .filter(b -> filteredParents.contains(b.getHeader().getBlockHash())) +// .collect(Collectors.toList()); +// for (Block parentBLock : parentBlocks) { +// buildEdgeFieldFromBlockPair(builder, currentBlock, parentBLock); +// } +// } - private IndexRequest buildHeightGroupRequest(Block block) { + private IndexRequest buildHeightGroup(Block block, Integer blockHeightIndex) { IndexRequest request = new IndexRequest(dagInspectHeightGroupIndex); XContentBuilder builder = null; try { builder = XContentFactory.jsonBuilder(); builder.startObject(); + builder.field("height", block.getHeader().getHeight()); + builder.field("size", blockHeightIndex + 1); -// builder.field("from_block_id", block.getHeader().getBlockHash()); -// builder.field("to_block_id", parentBlock.getHeader().getBlockHash()); -// builder.field("from_height", block.getHeader().getHeight()); -// builder.field("to_height", parentBlock.getHeader().getHeight()); -// builder.field("to_height_group_index", 0); builder.endObject(); } catch (Exception e) { @@ -193,4 +357,14 @@ private IndexRequest buildHeightGroupRequest(Block block) { } + private List getGroupHeightSizeFromStorage(List heights) throws IOException { +// SearchRequest searchRequest = new SearchRequest(dagInspectHeightGroupIndex); +// TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("height", height); +// searchRequest.source(new SearchSourceBuilder().query(termQueryBuilder)); +// SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT); +// JSONObject obj = JSONObject.parseObject(ServiceUtils.getJsonString(response)); +// return obj.getInteger("size"); + return new ArrayList<>(); + } + } diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java index 599183e..29227fa 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java @@ -1,10 +1,10 @@ package org.starcoin.indexer.handler; import com.alibaba.fastjson.JSON; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.novi.serde.Bytes; import com.novi.serde.DeserializationError; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.bouncycastle.util.Arrays; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.DocWriteResponse; @@ -54,6 +54,7 @@ import org.starcoin.constant.Constant; import org.starcoin.indexer.service.AddressHolderService; import org.starcoin.indexer.service.TransactionPayloadService; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import org.starcoin.types.AccountAddress; import org.starcoin.types.StructTag; import org.starcoin.types.TokenCode; @@ -378,7 +379,7 @@ public void bulk(List blockList) { addTokenInfo(tokenInfo, codeStr); //add to cache tokenCache.put(codeStr, tokenInfo); - } catch (JSONRPC2SessionException e) { + } catch (JSONRPC2SessionException | JsonProcessingException e) { logger.error("flush token error:", e); } } From b576c44f58782431be5848b6d8d0b5c219c6e449 Mon Sep 17 00:00:00 2001 From: bobong Date: Sat, 15 Jun 2024 16:19:06 +0800 Subject: [PATCH 11/82] [dag inspector api] Solve the type reference problem of exception handling, that is, `com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException` should be `org.starcoin.jsonrpc.client.JSONRPC2SessionException`; (cherry picked from commit b568d235e689a643ab138c59ae8aa79439da5c6f) --- .../java/org/starcoin/indexer/handler/MarketCapHandle.java | 2 +- .../main/java/org/starcoin/indexer/handler/RepairHandle.java | 5 +++-- .../main/java/org/starcoin/indexer/handler/ServiceUtils.java | 4 ++-- .../java/org/starcoin/indexer/handler/SubscribeHandler.java | 2 +- .../main/java/org/starcoin/indexer/handler/SwapHandle.java | 2 +- .../org/starcoin/indexer/handler/TransactionInfoHandle.java | 2 +- .../starcoin/indexer/handler/TransactionPayloadHandle.java | 2 +- .../main/java/org/starcoin/indexer/handler/TvlService.java | 2 +- .../java/org/starcoin/indexer/handler/UpdateTVLHandle.java | 3 ++- .../java/org/starcoin/indexer/handler/SwapIndexerTest.java | 2 +- .../src/test/java/org/starcoin/indexer/handler/TvlTest.java | 2 +- 11 files changed, 15 insertions(+), 13 deletions(-) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java index 44aa22f..761bd13 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java @@ -1,6 +1,5 @@ package org.starcoin.indexer.handler; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; @@ -20,6 +19,7 @@ import org.starcoin.api.TokenContractRPCClient; import org.starcoin.bean.*; import org.starcoin.constant.Constant; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import javax.annotation.PostConstruct; import java.io.IOException; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java index 55547ec..8adff82 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java @@ -1,6 +1,6 @@ package org.starcoin.indexer.handler; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; +import com.fasterxml.jackson.core.JsonProcessingException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -10,6 +10,7 @@ import org.starcoin.api.TransactionRPCClient; import org.starcoin.bean.Block; import org.starcoin.bean.BlockHeader; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import java.util.ArrayList; import java.util.HashMap; @@ -65,7 +66,7 @@ public boolean autoRepair(long startNumber, int count) { logger.info("repair too fast: {}", startNumber); return false; } - } catch (JSONRPC2SessionException e) { + } catch (JSONRPC2SessionException | JsonProcessingException e) { logger.error("get master error: ", e); return false; } diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java index 6777d19..3df24e6 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java @@ -7,7 +7,6 @@ import com.novi.bcs.BcsDeserializer; import com.novi.serde.Bytes; import com.novi.serde.DeserializationError; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; @@ -23,6 +22,7 @@ import org.starcoin.api.TransactionRPCClient; import org.starcoin.bean.*; import org.starcoin.bean.TransferOffset; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import org.starcoin.utils.ResultWithId; import org.starcoin.types.ModuleId; import org.starcoin.types.ScriptFunction; @@ -180,7 +180,7 @@ static TokenInfo getTokenInfo(StateRPCClient stateRPCClient, String tokenCode) { if (tokenInfo != null) { tokenCache.put(tokenCode, tokenInfo); } - } catch (JSONRPC2SessionException e) { + } catch (JSONRPC2SessionException | JsonProcessingException e) { logger.error("get token info error:", e); } } diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SubscribeHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SubscribeHandler.java index cdd044f..14c61ee 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SubscribeHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SubscribeHandler.java @@ -1,6 +1,5 @@ package org.starcoin.indexer.handler; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import io.reactivex.Flowable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,6 +7,7 @@ import org.starcoin.api.TransactionRPCClient; import org.starcoin.bean.PendingTransaction; import org.starcoin.bean.PendingTransactionNotification; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import org.web3j.protocol.websocket.WebSocketService; import java.net.ConnectException; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SwapHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SwapHandle.java index 9395b8d..34519e0 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SwapHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SwapHandle.java @@ -1,6 +1,5 @@ package org.starcoin.indexer.handler; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.elasticsearch.client.RestHighLevelClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,6 +13,7 @@ import org.starcoin.indexer.service.SwapStatService; import org.starcoin.indexer.service.SwapTxnService; import org.starcoin.indexer.service.TokenStatService; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import org.starcoin.utils.NumberUtils; import org.starcoin.utils.StructTagUtil; import org.starcoin.utils.SwapApiClient; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionInfoHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionInfoHandle.java index 8bc644d..c38d676 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionInfoHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionInfoHandle.java @@ -1,6 +1,5 @@ package org.starcoin.indexer.handler; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; @@ -22,6 +21,7 @@ import org.starcoin.api.TransactionRPCClient; import org.starcoin.bean.Transaction; import org.starcoin.constant.Constant; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import java.io.IOException; import java.util.List; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionPayloadHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionPayloadHandle.java index 5f2cca2..1a8504f 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionPayloadHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionPayloadHandle.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.novi.serde.DeserializationError; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.elasticsearch.client.RestHighLevelClient; import org.quartz.JobExecutionContext; import org.slf4j.Logger; @@ -17,6 +16,7 @@ import org.starcoin.constant.Constant; import org.starcoin.constant.StarcoinNetwork; import org.starcoin.indexer.service.SwapTxnService; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import org.starcoin.types.*; import org.starcoin.types.StructTag; import org.starcoin.types.TransactionPayload; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TvlService.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TvlService.java index f2b0ed5..3d6aff4 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TvlService.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TvlService.java @@ -1,6 +1,5 @@ package org.starcoin.indexer.handler; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -9,6 +8,7 @@ import org.starcoin.bean.ContractCall; import org.starcoin.bean.TokenPairTvl; import org.starcoin.bean.TokenTvlAmount; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import java.math.BigInteger; import java.util.ArrayList; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/UpdateTVLHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/UpdateTVLHandle.java index fc3f100..14adcc8 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/UpdateTVLHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/UpdateTVLHandle.java @@ -1,6 +1,6 @@ package org.starcoin.indexer.handler; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -13,6 +13,7 @@ import org.starcoin.indexer.service.SwapPoolStatService; import org.starcoin.indexer.service.SwapStatService; import org.starcoin.indexer.service.TokenStatService; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import org.starcoin.utils.SwapApiClient; import javax.annotation.PostConstruct; diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java index 5255e65..023759b 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java @@ -1,6 +1,5 @@ package org.starcoin.indexer.handler; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -15,6 +14,7 @@ import org.starcoin.bean.SwapTransaction; import org.starcoin.bean.SwapType; import org.starcoin.indexer.service.SwapTxnService; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import org.starcoin.utils.SwapApiClient; import java.io.IOException; diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java index c88d23c..2985792 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java @@ -1,10 +1,10 @@ package org.starcoin.indexer.handler; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.junit.Test; import org.starcoin.api.ContractRPCClient; import org.starcoin.api.StateRPCClient; import org.starcoin.bean.ContractCall; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import java.net.MalformedURLException; import java.net.URL; From df0c7e2b64e0dedefda75c33a8593eb5c82126d4 Mon Sep 17 00:00:00 2001 From: bobong Date: Sat, 15 Jun 2024 23:31:01 +0800 Subject: [PATCH 12/82] [dag inspector api] Added Dag Inspector related data fetching logic to the API (cherry picked from commit 55d22454e1add961a15608ff8531ad0abebceea8) --- .../indexer/handler/DagInspectorHandler.java | 77 +++------ .../starcoin/scan/service/BlockService.java | 29 ++-- .../scan/service/DagInspectorService.java | 163 +++++++++++++----- .../starcoin/scan/service/vo/DIAppConfig.java | 4 - .../scan/service/vo/DIAppConfigVo.java | 49 ++++++ .../org/starcoin/scan/service/vo/DIBlock.java | 108 ------------ .../vo/DIBlocksAndEdgesAndHeightGroups.java | 36 ---- .../vo/DIBlocksAndEdgesAndHeightGroupsVo.java | 40 +++++ .../org/starcoin/scan/service/vo/DIEdge.java | 52 ------ .../scan/service/vo/DIHeightGroup.java | 22 --- 10 files changed, 250 insertions(+), 330 deletions(-) delete mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfig.java create mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfigVo.java delete mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlock.java delete mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroups.java create mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java delete mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIEdge.java delete mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIHeightGroup.java diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java index 0f70186..78ad7d3 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java @@ -21,6 +21,7 @@ import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import org.starcoin.api.BlockRPCClient; +import org.starcoin.api.Result; import org.starcoin.bean.*; import org.starcoin.constant.Constant; import org.starcoin.jsonrpc.client.JSONRPC2SessionException; @@ -110,12 +111,14 @@ public void upsertDagInfoFromBlocks(List blockList) throws IOException { } }); - List inspectorBlockList = new ArrayList<>(inspBlockMap.values()); - List edgeList = buildEdgeDataFromNodeData(inspectorBlockList); // Save all data into storage + List inspectorBlockList = new ArrayList<>(inspBlockMap.values()); bulkRequest.add(buildSaveInspectorBlockRequest(inspectorBlockList)); + + List edgeList = buildEdgeDataFromNodeData(inspectorBlockList); bulkRequest.add(buildSaveEdgeRequest(edgeList)); + bulkRequest.add(buildSaveHeightGroupRequest(inspHeightGroupList)); // Bulk save @@ -271,6 +274,13 @@ IndexRequest buildSaveHeightGroupRequest(List heightGro return request; } + /** + * Get the size of the group at the specified height, or return the default size if the group does not exist. + * @param groupList + * @param height + * @param defaultSize + * @return + */ Integer getHeightGroupSizeOrDefault(List groupList, Long height, Integer defaultSize) { if (groupList.isEmpty()) { return defaultSize; @@ -291,51 +301,6 @@ void updateGroupSize(List groupList, Long height, Integ } } } -// /** -// * Build edge field from block, include from_block_id, to_block_id, from_height, to_height, etc. -// * -// * @param builder -// * @param currentBlock -// * @param blockList -// * @throws IOException -// */ -// private void buildEdgeFieldFromBlock(XContentBuilder builder, Block currentBlock, List blockList) throws IOException { -// BlockHeader currentBlockHeader = currentBlock.getHeader(); -// -// String parentHash = currentBlockHeader.getParentHash(); -// Block parentBlock = blockList -// .stream() -// .filter(b -> b.getHeader().getBlockHash().equalsIgnoreCase(parentHash)) -// .findFirst() -// .orElse(null); -// if (parentBlock == null) { -// // TODO(BobOng): Get parent block from chain data -// logger.info("Need to getting the parent block from chain data, parent hash: {}", parentHash); -// } -// buildEdgeFieldFromBlockPair(builder, currentBlock, parentBlock); -// -// if (currentBlockHeader.getParentsHash().isEmpty()) { -// return; -// } -// -// List filteredParents = currentBlockHeader -// .getParentsHash() -// .stream() -// .filter(h -> h.equalsIgnoreCase(parentHash)) -// .collect(Collectors.toList()); -// -// if (filteredParents.isEmpty()) { -// return; -// } -// -// List parentBlocks = blockList -// .stream() -// .filter(b -> filteredParents.contains(b.getHeader().getBlockHash())) -// .collect(Collectors.toList()); -// for (Block parentBLock : parentBlocks) { -// buildEdgeFieldFromBlockPair(builder, currentBlock, parentBLock); -// } -// } private IndexRequest buildHeightGroup(Block block, Integer blockHeightIndex) { IndexRequest request = new IndexRequest(dagInspectHeightGroupIndex); @@ -358,13 +323,17 @@ private IndexRequest buildHeightGroup(Block block, Integer blockHeightIndex) { private List getGroupHeightSizeFromStorage(List heights) throws IOException { -// SearchRequest searchRequest = new SearchRequest(dagInspectHeightGroupIndex); -// TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("height", height); -// searchRequest.source(new SearchSourceBuilder().query(termQueryBuilder)); -// SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT); -// JSONObject obj = JSONObject.parseObject(ServiceUtils.getJsonString(response)); -// return obj.getInteger("size"); - return new ArrayList<>(); + List groupList = new ArrayList<>(); + if (heights.isEmpty()) { + return groupList; + } + + SearchRequest searchRequest = new SearchRequest(dagInspectHeightGroupIndex); + TermsQueryBuilder termQueryBuilder = QueryBuilders.termsQuery("height", heights); + searchRequest.source(new SearchSourceBuilder().query(termQueryBuilder)); + SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT); + Result result = ServiceUtils.getSearchResult(response, DagInspectorHeightGroup.class); + return result.getContents(); } } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java index 19ccf04..a68e487 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java @@ -21,7 +21,6 @@ import org.starcoin.bean.Block; import org.starcoin.bean.UncleBlock; import org.starcoin.constant.Constant; -import org.starcoin.scan.service.vo.DIEdge; import java.io.IOException; import java.util.List; @@ -218,18 +217,18 @@ public Result getUnclesRange(String network, int page, int count, in return ServiceUtils.getSearchResult(searchResponse, UncleBlock.class); } - public List getBlocksBetweenHeights(String network, Integer startHeight, Integer endHeight) { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.BLOCK_CONTENT_INDEX)); - RangeQueryBuilder termQueryBuilder = QueryBuilders.rangeQuery("header.number").gte(startHeight).lte(endHeight); - searchRequest.source(new SearchSourceBuilder().query(termQueryBuilder).sort("header.number", SortOrder.ASC)); - SearchResponse searchResponse; - try { - searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - logger.error("getBlocksBetweenHeights failed, startHeight: {}, endHeight: {}", startHeight, endHeight, e); - return null; - } - Result result = ServiceUtils.getSearchResult(searchResponse, Block.class); - return result.getContents(); - } +// public List getBlocksBetweenHeights(String network, Integer startHeight, Integer endHeight) { +// SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.BLOCK_CONTENT_INDEX)); +// RangeQueryBuilder termQueryBuilder = QueryBuilders.rangeQuery("header.number").gte(startHeight).lte(endHeight); +// searchRequest.source(new SearchSourceBuilder().query(termQueryBuilder).sort("header.number", SortOrder.ASC)); +// SearchResponse searchResponse; +// try { +// searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); +// } catch (IOException e) { +// logger.error("getBlocksBetweenHeights failed, startHeight: {}, endHeight: {}", startHeight, endHeight, e); +// return null; +// } +// Result result = ServiceUtils.getSearchResult(searchResponse, Block.class); +// return result.getContents(); +// } } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index 97b0732..5962608 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -7,6 +7,8 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.metrics.Max; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; @@ -14,7 +16,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.starcoin.api.Result; -import org.starcoin.bean.Block; +import org.starcoin.bean.DagInspectorBlock; +import org.starcoin.bean.DagInspectorEdge; +import org.starcoin.bean.DagInspectorHeightGroup; import org.starcoin.constant.Constant; import org.starcoin.scan.service.vo.*; @@ -33,55 +37,87 @@ public class DagInspectorService extends BaseService { @Autowired private RestHighLevelClient client; - public DIBlocksAndEdgesAndHeightGroups getBlocksBetweenHeights(String network, Integer startHeight, Integer endHeight) { - List blockList = blockService.getBlocksBetweenHeights(network, startHeight, endHeight); - - DIBlocksAndEdgesAndHeightGroups groups = new DIBlocksAndEdgesAndHeightGroups(); - groups.setBlocks(blockList.stream().map(block -> { - DIBlock diBlock = new DIBlock(); - diBlock.setBlockHash(block.getHeader().getBlockHash()); - diBlock.setHeight(block.getHeader().getHeight()); - diBlock.setColor("blue"); - diBlock.setInVirtualSelectedParentChain(false); - diBlock.setParentIds(block.getHeader().getParentsHash()); - - // TODO(BobOng): Get selected parent id from chain data - // diBlock.setSelectedParentHash(); - // diBlock.setHeightGroupIndex(); - // diBlock.setDaaScore(); - return diBlock; - }).collect(Collectors.toList())); - groups.setEdges(getEdgeList(network, startHeight, endHeight)); + public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String network, Long startHeight, Long endHeight) { + DIBlocksAndEdgesAndHeightGroupsVo groups = new DIBlocksAndEdgesAndHeightGroupsVo(); - List heights = blockList - .stream() - .map(block -> block.getHeader().getHeight()) - .collect(Collectors.toList()); + List blockList = getBlockList(network, startHeight, endHeight); + groups.setBlocks(blockList); + groups.setEdges(getEdgeList(network, startHeight, endHeight)); - List diHeightGroup = getHeightGroup(network, heights); - groups.setHeightGroups(diHeightGroup); + groups.setHeightGroups(getHeightGroup( + network, + blockList.stream() + .map(DagInspectorBlock::getHeight) + .collect(Collectors.toList())) + ); return groups; } - public DIBlocksAndEdgesAndHeightGroups getBlockHash(String targetHash, Integer heightDifference) { - return new DIBlocksAndEdgesAndHeightGroups(); + public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash(String network, String targetHash, Integer heightDifference) { + DagInspectorBlock block = getBlockWithHashFromStorage(network, targetHash); + Long endHeight = block.getHeight(); + long startHeight = block.getHeight() - heightDifference; + if (startHeight < 0L) { + startHeight = 0L; + } + return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); + } + + public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore(String network, Integer targetDAAScore, Integer heightDifference) { + return new DIBlocksAndEdgesAndHeightGroupsVo(); } - public DIBlocksAndEdgesAndHeightGroups getBlockDAAScore(Integer targetDAAScore, Integer heightDifference) { - return new DIBlocksAndEdgesAndHeightGroups(); + public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDifference) { + long endHeight = getMaxHeightFromStorage(); + long startHeight = endHeight - heightDifference; + if (startHeight < 0L) { + startHeight = 0L; + } + return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } - public DIBlocksAndEdgesAndHeightGroups getHead(Integer heightDifference) { - return new DIBlocksAndEdgesAndHeightGroups(); + /** + * Get block height from Elastic search storage + * + * @param network + * @param startHeight + * @param endHeight + * @return + */ + List getBlockList(String network, Long startHeight, Long endHeight) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_EDGE)); + RangeQueryBuilder heightQuery = QueryBuilders.rangeQuery("height").gte(startHeight).lte(endHeight); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query(QueryBuilders.boolQuery().must(heightQuery)) + .sort("height", SortOrder.ASC); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("getEdgeList failed, startHeight: {}, endHeight: {}", startHeight, endHeight, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); + return result.getContents(); } - List getEdgeList(String network, Integer startHeight, Integer endHeight) { + /** + * Get Edge list from ElasticSearch storage + * @param network + * @param startHeight + * @param endHeight + * @return + */ + List getEdgeList(String network, Long startHeight, Long endHeight) { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_EDGE)); RangeQueryBuilder fromHeightQuery = QueryBuilders.rangeQuery("from_height").gte(startHeight); RangeQueryBuilder toHeightQuery = QueryBuilders.rangeQuery("to_height").lte(endHeight); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(QueryBuilders.boolQuery().must(fromHeightQuery).must(toHeightQuery)).sort("to_height", SortOrder.ASC); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query(QueryBuilders.boolQuery().must(fromHeightQuery).must(toHeightQuery)) + .sort("to_height", SortOrder.ASC); searchRequest.source(sourceBuilder); SearchResponse searchResponse; try { @@ -90,11 +126,17 @@ List getEdgeList(String network, Integer startHeight, Integer endHeight) logger.error("getEdgeList failed, startHeight: {}, endHeight: {}", startHeight, endHeight, e); return null; } - Result result = ServiceUtils.getSearchResult(searchResponse, DIEdge.class); + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorEdge.class); return result.getContents(); } - List getHeightGroup(String network, List heights) { + /** + * Get heights + * @param network + * @param heights + * @return + */ + List getHeightGroup(String network, List heights) { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECT_HEIGHT_GROUP)); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); for (Long height : heights) { @@ -109,12 +151,55 @@ List getHeightGroup(String network, List heights) { logger.error("getHeightGroup failed, heights: {}", heights, e); return null; } - Result result = ServiceUtils.getSearchResult(searchResponse, DIHeightGroup.class); + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorHeightGroup.class); return result.getContents(); } - public DIAppConfig getAppConfig() { - return new DIAppConfig(); + public DIAppConfigVo getAppConfig() { + return new DIAppConfigVo(); + } + + Long getMaxHeightFromStorage() { + final String MAX_HEIGHT_FIELD = "max_height"; + final String HEIGHT_FIELD = "height"; + + try { + SearchRequest searchRequest = new SearchRequest(Constant.DAG_INSPECTOR_NODE); + + // Build the SearchSourceBuilder with max aggregation + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.aggregation(AggregationBuilders.max(MAX_HEIGHT_FIELD).field(HEIGHT_FIELD)); + searchRequest.source(sourceBuilder); + + // Execute the search request + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + + // Handle the response + Max maxHeight = searchResponse.getAggregations().get("max_height"); + return (long)maxHeight.getValue(); + + } catch (IOException e) { + e.printStackTrace(); + } + return 0L; + } + + DagInspectorBlock getBlockWithHashFromStorage(String network, String blockHash) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECT_HEIGHT_GROUP)); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.should(QueryBuilders.termQuery("block_hash", blockHash)); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("getBlockWithHashFromStorage failed, blockHash: {}", blockHash, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); + return result.getContents().get(0); } } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfig.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfig.java deleted file mode 100644 index dc01f4c..0000000 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfig.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.starcoin.scan.service.vo; - -public class DIAppConfig { -} diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfigVo.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfigVo.java new file mode 100644 index 0000000..6b155e4 --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfigVo.java @@ -0,0 +1,49 @@ +package org.starcoin.scan.service.vo; + +public class DIAppConfigVo { + String starcoinVersion; + String processingVersion; + String network; + String apiVersion; + String webVersion; + + public String getStarcoinVersion() { + return starcoinVersion; + } + + public void setStarcoinVersion(String starcoinVersion) { + this.starcoinVersion = starcoinVersion; + } + + public String getProcessingVersion() { + return processingVersion; + } + + public void setProcessingVersion(String processingVersion) { + this.processingVersion = processingVersion; + } + + public String getNetwork() { + return network; + } + + public void setNetwork(String network) { + this.network = network; + } + + public String getApiVersion() { + return apiVersion; + } + + public void setApiVersion(String apiVersion) { + this.apiVersion = apiVersion; + } + + public String getWebVersion() { + return webVersion; + } + + public void setWebVersion(String webVersion) { + this.webVersion = webVersion; + } +} diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlock.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlock.java deleted file mode 100644 index 1551458..0000000 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlock.java +++ /dev/null @@ -1,108 +0,0 @@ -package org.starcoin.scan.service.vo; - -import java.util.Vector; - -/** - * Starcoin DAG Inspector Block info - */ -public class DIBlock { - String blockHash; - Integer timestamp; - Vector parentIds; - Long height; - Integer daaScore; - Integer heightGroupIndex; - String selectedParentHash; - String color; - Boolean isInVirtualSelectedParentChain; - Vector mergeSetRedIds; - Vector mergeSetBlueIds; - - public String getBlockHash() { - return blockHash; - } - - public void setBlockHash(String blockHash) { - this.blockHash = blockHash; - } - - public Integer getTimestamp() { - return timestamp; - } - - public void setTimestamp(Integer timestamp) { - this.timestamp = timestamp; - } - - public Vector getParentIds() { - return parentIds; - } - - public void setParentIds(Vector parentIds) { - this.parentIds = parentIds; - } - - public Long getHeight() { - return height; - } - - public void setHeight(Long height) { - this.height = height; - } - - public Integer getDaaScore() { - return daaScore; - } - - public void setDaaScore(Integer daaScore) { - this.daaScore = daaScore; - } - - public Integer getHeightGroupIndex() { - return heightGroupIndex; - } - - public void setHeightGroupIndex(Integer heightGroupIndex) { - this.heightGroupIndex = heightGroupIndex; - } - - public String getSelectedParentHash() { - return selectedParentHash; - } - - public void setSelectedParentHash(String selectedParentHash) { - this.selectedParentHash = selectedParentHash; - } - - public String getColor() { - return color; - } - - public void setColor(String color) { - this.color = color; - } - - public Boolean getInVirtualSelectedParentChain() { - return isInVirtualSelectedParentChain; - } - - public void setInVirtualSelectedParentChain(Boolean inVirtualSelectedParentChain) { - isInVirtualSelectedParentChain = inVirtualSelectedParentChain; - } - - public Vector getMergeSetRedIds() { - return mergeSetRedIds; - } - - public void setMergeSetRedIds(Vector mergeSetRedIds) { - this.mergeSetRedIds = mergeSetRedIds; - } - - public Vector getMergeSetBlueIds() { - return mergeSetBlueIds; - } - - public void setMergeSetBlueIds(Vector mergeSetBlueIds) { - this.mergeSetBlueIds = mergeSetBlueIds; - } -} \ No newline at end of file diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroups.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroups.java deleted file mode 100644 index b3fb61f..0000000 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroups.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.starcoin.scan.service.vo; - -import java.util.List; - -/** - * DAG Inspector query object - */ -public class DIBlocksAndEdgesAndHeightGroups { - List blocks; - List edges; - List heightGroups; - - public List getBlocks() { - return blocks; - } - - public void setBlocks(List blocks) { - this.blocks = blocks; - } - - public List getEdges() { - return edges; - } - - public void setEdges(List edges) { - this.edges = edges; - } - - public List getHeightGroups() { - return heightGroups; - } - - public void setHeightGroups(List heightGroups) { - this.heightGroups = heightGroups; - } -} \ No newline at end of file diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java new file mode 100644 index 0000000..e93975c --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java @@ -0,0 +1,40 @@ +package org.starcoin.scan.service.vo; + +import org.starcoin.bean.DagInspectorBlock; +import org.starcoin.bean.DagInspectorEdge; +import org.starcoin.bean.DagInspectorHeightGroup; + +import java.util.List; + +/** + * DAG Inspector query object + */ +public class DIBlocksAndEdgesAndHeightGroupsVo { + List blocks; + List edges; + List heightGroups; + + public List getBlocks() { + return blocks; + } + + public void setBlocks(List blocks) { + this.blocks = blocks; + } + + public List getEdges() { + return edges; + } + + public void setEdges(List edges) { + this.edges = edges; + } + + public List getHeightGroups() { + return heightGroups; + } + + public void setHeightGroups(List heightGroups) { + this.heightGroups = heightGroups; + } +} \ No newline at end of file diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIEdge.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIEdge.java deleted file mode 100644 index 1b0b925..0000000 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIEdge.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.starcoin.scan.service.vo; - -/** - * Starcoin DAG Inspector Edge info - */ -public class DIEdge { - String fromBlockId; - String toBlockId; - Integer fromHeight; - Integer toHeight; - Integer toHeightGroupIndex; - - public String getFromBlockId() { - return fromBlockId; - } - - public void setFromBlockId(String fromBlockId) { - this.fromBlockId = fromBlockId; - } - - public String getToBlockId() { - return toBlockId; - } - - public void setToBlockId(String toBlockId) { - this.toBlockId = toBlockId; - } - - public Integer getFromHeight() { - return fromHeight; - } - - public void setFromHeight(Integer fromHeight) { - this.fromHeight = fromHeight; - } - - public Integer getToHeight() { - return toHeight; - } - - public void setToHeight(Integer toHeight) { - this.toHeight = toHeight; - } - - public Integer getToHeightGroupIndex() { - return toHeightGroupIndex; - } - - public void setToHeightGroupIndex(Integer toHeightGroupIndex) { - this.toHeightGroupIndex = toHeightGroupIndex; - } -} \ No newline at end of file diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIHeightGroup.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIHeightGroup.java deleted file mode 100644 index b5d6f59..0000000 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIHeightGroup.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.starcoin.scan.service.vo; - -public class DIHeightGroup { - Integer height; - Integer size; - - public Integer getHeight() { - return height; - } - - public void setHeight(Integer height) { - this.height = height; - } - - public Integer getSize() { - return size; - } - - public void setSize(Integer size) { - this.size = size; - } -} From 02f43ece580e7a740b24892c6f2eb824b18dee84 Mon Sep 17 00:00:00 2001 From: bobong Date: Sun, 16 Jun 2024 21:46:45 +0800 Subject: [PATCH 13/82] [dag inspector api] add dag query from daa score (cherry picked from commit 520277f1679a7d62f914548937a87633d3f3cfef) --- .../scan/service/DagInspectorService.java | 32 +++++++++++++++++-- .../scan/service/DagInspectServiceTest.java | 9 ++++++ 2 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index 5962608..276a138 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -66,7 +66,13 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash(String network, String tar } public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore(String network, Integer targetDAAScore, Integer heightDifference) { - return new DIBlocksAndEdgesAndHeightGroupsVo(); + DagInspectorBlock block = getHeightWithDAAScoreFromStorage(network, targetDAAScore); + Long endHeight = block.getHeight(); + long startHeight = block.getHeight() - heightDifference; + if (startHeight < 0L) { + startHeight = 0L; + } + return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDifference) { @@ -106,6 +112,7 @@ List getBlockList(String network, Long startHeight, Long endH /** * Get Edge list from ElasticSearch storage + * * @param network * @param startHeight * @param endHeight @@ -132,6 +139,7 @@ List getEdgeList(String network, Long startHeight, Long endHei /** * Get heights + * * @param network * @param heights * @return @@ -176,7 +184,7 @@ Long getMaxHeightFromStorage() { // Handle the response Max maxHeight = searchResponse.getAggregations().get("max_height"); - return (long)maxHeight.getValue(); + return (long) maxHeight.getValue(); } catch (IOException e) { e.printStackTrace(); @@ -185,7 +193,7 @@ Long getMaxHeightFromStorage() { } DagInspectorBlock getBlockWithHashFromStorage(String network, String blockHash) { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECT_HEIGHT_GROUP)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.should(QueryBuilders.termQuery("block_hash", blockHash)); @@ -202,4 +210,22 @@ DagInspectorBlock getBlockWithHashFromStorage(String network, String blockHash) return result.getContents().get(0); } + DagInspectorBlock getHeightWithDAAScoreFromStorage(String network, Integer daaScore) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.should(QueryBuilders.termQuery("daa_score", daaScore)); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("getBlockWithHashFromStorage failed, blockHash: {}", daaScore, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); + return result.getContents().get(0); + } + } diff --git a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java new file mode 100644 index 0000000..e75bca5 --- /dev/null +++ b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java @@ -0,0 +1,9 @@ +package org.starcoin.scan.service; + +import org.junit.jupiter.api.Test; + +public class DagInspectServiceTest { + + @Test + public void test +} From 948b209a19627e7153aab5d561b43d7e323942be Mon Sep 17 00:00:00 2001 From: bobong Date: Tue, 18 Jun 2024 15:04:46 +0800 Subject: [PATCH 14/82] [dag inspector api] commit all codes for committing to the right repository (cherry picked from commit 1e75f0657ee6c20f181d95958f341df9395886f5) --- docker-compose/docker-compose.yaml | 24 ++++ starcoin-indexer/local.env | 4 +- .../starcoin/indexer/config/QuartzConfig.java | 21 +++ .../indexer/handler/DagInspectorHandler.java | 3 +- .../indexer/handler/DagInspectorIndexer.java | 136 ++++++++++++++++++ .../indexer/handler/IndexerHandle.java | 8 +- .../indexer/handler/DagInspectorTest.java | 73 ++++++++++ .../indexer/test/IndexerLogicBaseTest.java | 13 ++ .../indexer/utils/SwapApiClientTest.java | 10 +- .../application-integrationtest.properties | 55 +++++-- .../src/test/resources/unit-test.env | 18 +++ .../scan/service/DagInspectorService.java | 12 +- .../scan/service/DagInspectServiceTest.java | 15 +- 13 files changed, 358 insertions(+), 34 deletions(-) create mode 100644 docker-compose/docker-compose.yaml create mode 100644 starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java create mode 100644 starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java create mode 100644 starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexerLogicBaseTest.java create mode 100644 starcoin-indexer/src/test/resources/unit-test.env diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml new file mode 100644 index 0000000..35a0bdd --- /dev/null +++ b/docker-compose/docker-compose.yaml @@ -0,0 +1,24 @@ +version: '3.8' + +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.2 + container_name: elasticsearch + environment: + - node.name=elasticsearch + - cluster.name=docker-cluster + - discovery.type=single-node + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata:/usr/share/elasticsearch/data + ports: + - "9200:9200" + +volumes: + esdata: + driver: local diff --git a/starcoin-indexer/local.env b/starcoin-indexer/local.env index 534b7d2..53e1d2c 100644 --- a/starcoin-indexer/local.env +++ b/starcoin-indexer/local.env @@ -1,6 +1,6 @@ HOSTS=halley.seed.starcoin.org -NETWORK=halley.0727 -BG_TASK_JOBS=indexer,secondary,market_cap,txn_payload,txn_global_idx_update +NETWORK=halley +BG_TASK_JOBS=dag_inspector TXN_OFFSET=0 BULK_SIZE=100 STARCOIN_ES_PWD= diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java index 5d17a49..1c10870 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java @@ -203,6 +203,22 @@ public Trigger priceStatTrigger() { .build(); } + @Bean + public JobDetail dagInspectorJob() { + return JobBuilder.newJob(DagInspectorIndexer.class).withIdentity("dag_inspector").storeDurably().build(); + } + + @Bean + public Trigger dagInspectorTrigger() { + SimpleScheduleBuilder scheduleBuilder = SimpleScheduleBuilder.simpleSchedule() + .withIntervalInSeconds(15) + .repeatForever(); + return TriggerBuilder.newTrigger().forJob(dagInspectorJob()) + .withIdentity("dag_inspector") + .withSchedule(scheduleBuilder) + .build(); + } + @Bean public SchedulerFactoryBean schedulerFactoryBean() { SchedulerFactoryBean schedulerFactoryBean = new SchedulerFactoryBean(); @@ -216,6 +232,7 @@ public SchedulerFactoryBean schedulerFactoryBean() { return schedulerFactoryBean; } + private Properties quartzProperties() { Properties prop = new Properties(); prop.put("org.quartz.scheduler.instanceName", "quartzScheduler");// 调度器的实例名 @@ -281,6 +298,10 @@ public Scheduler scheduler() throws SchedulerException { if (jobSet.contains(job.getKey().getName())) { scheduler.scheduleJob(job, priceStatTrigger()); } + job = dagInspectorJob(); + if (jobSet.contains(job.getKey().getName())) { + scheduler.scheduleJob(job, dagInspectorTrigger()); + } scheduler.start(); return scheduler; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java index 78ad7d3..968f780 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java @@ -118,7 +118,6 @@ public void upsertDagInfoFromBlocks(List blockList) throws IOException { List edgeList = buildEdgeDataFromNodeData(inspectorBlockList); bulkRequest.add(buildSaveEdgeRequest(edgeList)); - bulkRequest.add(buildSaveHeightGroupRequest(inspHeightGroupList)); // Bulk save @@ -226,7 +225,7 @@ IndexRequest buildSaveInspectorBlockRequest(List blockList) { return request; } - IndexRequest buildSaveEdgeRequest(List edgeList) { + private IndexRequest buildSaveEdgeRequest(List edgeList) { IndexRequest request = new IndexRequest(dagInspectEdgeIndex); if (edgeList.isEmpty()) { return request; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java new file mode 100644 index 0000000..893876f --- /dev/null +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java @@ -0,0 +1,136 @@ +package org.starcoin.indexer.handler; + +import org.quartz.JobExecutionContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.quartz.QuartzJobBean; +import org.starcoin.api.BlockRPCClient; +import org.starcoin.bean.Block; +import org.starcoin.bean.BlockHeader; +import org.starcoin.bean.BlockOffset; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +public class DagInspectorIndexer extends QuartzJobBean { + + private static final Logger logger = LoggerFactory.getLogger(DagInspectorIndexer.class); + + @Autowired + private DagInspectorHandler inspectorHandler; + + @Autowired + private ElasticSearchHandler elasticSearchHandler; + + @Autowired + private BlockRPCClient blockRPCClient; + + @Value("${starcoin.indexer.bulk_size}") + private long bulkSize; + + private BlockOffset localBlockOffset; + + private BlockHeader currentHandleHeader; + + @PostConstruct + public void initOffset() { + localBlockOffset = elasticSearchHandler.getRemoteOffset(); + //update current handle header + try { + if (localBlockOffset != null) { + Block block = blockRPCClient.getBlockByHeight(localBlockOffset.getBlockHeight()); + if (block != null) { + currentHandleHeader = block.getHeader(); + } else { + logger.error("init offset block not exist on chain: {}", localBlockOffset); + } + + } else { + logger.warn("offset is null,init reset to genesis"); + currentHandleHeader = blockRPCClient.getBlockByHeight(0).getHeader(); + localBlockOffset = new BlockOffset(0, currentHandleHeader.getBlockHash()); + elasticSearchHandler.setRemoteOffset(localBlockOffset); + logger.info("init offset ok: {}", localBlockOffset); + } + } catch (JSONRPC2SessionException e) { + logger.error("set current header error:", e); + } + } + + @Override + protected void executeInternal(JobExecutionContext jobExecutionContext) { + //read current offset + if (localBlockOffset == null || currentHandleHeader == null) { + initOffset(); + } + BlockOffset remoteBlockOffset = elasticSearchHandler.getRemoteOffset(); + logger.info("current remote offset: {}", remoteBlockOffset); + if (remoteBlockOffset == null) { + logger.warn("offset must not null, please check blocks.mapping!!"); + return; + } + + if (remoteBlockOffset.getBlockHeight() > localBlockOffset.getBlockHeight()) { + logger.info("indexer equalize chain blocks."); + return; + } + + //read head + try { + BlockHeader chainHeader = blockRPCClient.getChainHeader(); + // Calculate bulk size + long headHeight = chainHeader.getHeight(); + long bulkNumber = Math.min(headHeight - localBlockOffset.getBlockHeight(), bulkSize); + int index = 1; + Map blockMap = new HashMap<>(); + while (index <= bulkNumber) { + long readNumber = localBlockOffset.getBlockHeight() + index; + + logger.info("Start Get block number: {}", index); + Block block = blockRPCClient.getBlockByHeight(readNumber); + if (block == null) { + logger.warn("get block null: {}", readNumber); + return; + } + + logger.info("Block number is: {}, block hash is: {}", readNumber, block.getHeader().getBlockHash()); + + for (String parentHash : block.getHeader().getParentsHash()) { + logger.info( + "Start get block hash : {}, parent hash: {}", + block.getHeader().getBlockHash(), + parentHash + ); + + if (!blockMap.containsKey(parentHash)) { + Block parentBlock = blockRPCClient.getBlockByHash(parentHash); + if (parentBlock == null) { + logger.warn("get parent block null: {}", parentHash); + continue; + } + blockMap.put(parentHash, parentBlock); + } + } + //update current header + currentHandleHeader = block.getHeader(); + index++; + logger.info("add block: {}", block.getHeader()); + } + inspectorHandler.upsertDagInfoFromBlocks(new ArrayList<>(blockMap.values())); + + // Update offset + localBlockOffset.setBlockHeight(currentHandleHeader.getHeight()); + localBlockOffset.setBlockHash(currentHandleHeader.getBlockHash()); + elasticSearchHandler.setRemoteOffset(localBlockOffset); + logger.info("indexer update success: {}", localBlockOffset); + } catch (JSONRPC2SessionException | IOException e) { + logger.error("chain header error:", e); + } + } +} diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java index f365305..1a6cf92 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java @@ -1,6 +1,5 @@ package org.starcoin.indexer.handler; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.quartz.JobExecutionContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,8 +11,10 @@ import org.starcoin.bean.Block; import org.starcoin.bean.BlockHeader; import org.starcoin.bean.BlockOffset; +import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import javax.annotation.PostConstruct; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -33,12 +34,14 @@ public class IndexerHandle extends QuartzJobBean { @Autowired private ElasticSearchHandler elasticSearchHandler; + @Autowired private TransactionRPCClient transactionRPCClient; @Autowired private BlockRPCClient blockRPCClient; + @PostConstruct public void initOffset() { localBlockOffset = elasticSearchHandler.getRemoteOffset(); @@ -158,12 +161,13 @@ protected void executeInternal(JobExecutionContext jobExecutionContext) { } //bulk execute elasticSearchHandler.bulk(blockList); + //update offset localBlockOffset.setBlockHeight(currentHandleHeader.getHeight()); localBlockOffset.setBlockHash(currentHandleHeader.getBlockHash()); elasticSearchHandler.setRemoteOffset(localBlockOffset); logger.info("indexer update success: {}", localBlockOffset); - } catch (JSONRPC2SessionException e) { + } catch (JSONRPC2SessionException | IOException e) { logger.error("chain header error:", e); } } diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java new file mode 100644 index 0000000..028007e --- /dev/null +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java @@ -0,0 +1,73 @@ +package org.starcoin.indexer.handler; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.quartz.JobDetail; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.Trigger; +import org.quartz.impl.StdSchedulerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.starcoin.bean.Block; +import org.starcoin.bean.BlockHeader; +import org.starcoin.indexer.test.IndexerLogicBaseTest; + +import java.util.ArrayList; +import java.util.List; +import java.util.Vector; + +import static org.quartz.JobBuilder.newJob; +import static org.quartz.TriggerBuilder.newTrigger; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + + +public class DagInspectorTest extends IndexerLogicBaseTest { + @Autowired + private DagInspectorHandler dagInspectorHandler; + + public CountDownLatch latch; + private Scheduler scheduler; + + Block new_test_block(String block_hash, Long height, String parent_hash, List parents_hash) { + Block block = new Block(); + BlockHeader header = new BlockHeader(); + header.setHeight(height); + header.setBlockHash(block_hash); + header.setParentHash(parent_hash); + header.setParentsHash(new Vector<>(parents_hash)); + block.setHeader(header); + return block; + } + + /** + * Test of testUpsertFromBlocks method, of class DagInspector. + */ + @Test + public void testTestUpsertFromBlocks() throws Exception { + + // Initialize blocks + // Block 1 (Genesis) + // Block 1 <- Block 2 + // Block 1 <- Block 3 + // Block 2 <- Block 4 + // Block 3 <- Block 4 + // Block 4 <- Block 5 + // Block 2 <- Block 6 + // Block 3 <- Block 6 + List blockList = new ArrayList<>(); + blockList.add(new_test_block("block_hash_1", 1L, "", new Vector<>())); + blockList.add(new_test_block("block_hash_2", 2L, "block_hash_1", List.of("block_hash_1"))); + blockList.add(new_test_block("block_hash_3", 2L, "block_hash_1", List.of("block_hash_1"))); + blockList.add(new_test_block("block_hash_4", 3L, "block_hash_3", List.of("block_hash_3"))); + blockList.add(new_test_block("block_hash_5", 4L, "block_hash_4", List.of("block_hash_4"))); + blockList.add(new_test_block("block_hash_6", 5L, "block_hash_2", List.of("block_hash_2", "block_hash_3"))); + dagInspectorHandler.upsertDagInfoFromBlocks(blockList); + } + + +} diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexerLogicBaseTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexerLogicBaseTest.java new file mode 100644 index 0000000..9d74f4b --- /dev/null +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexerLogicBaseTest.java @@ -0,0 +1,13 @@ +package org.starcoin.indexer.test; + +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.starcoin.indexer.IndexerApplication; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = IndexerApplication.class) +@TestPropertySource(locations = "classpath:application-integrationtest.properties") +public class IndexerLogicBaseTest { +} diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/SwapApiClientTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/SwapApiClientTest.java index 7709bab..e6a4e0c 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/SwapApiClientTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/SwapApiClientTest.java @@ -2,21 +2,15 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.TestPropertySource; -import org.starcoin.indexer.IndexerApplication; import org.starcoin.bean.LiquidityPoolInfo; import org.starcoin.bean.SwapToken; +import org.starcoin.indexer.test.IndexerLogicBaseTest; import org.starcoin.utils.SwapApiClient; import java.io.IOException; import java.util.List; -@SpringBootTest( - classes = IndexerApplication.class) -@TestPropertySource( - locations = "classpath:application-integrationtest.properties") -class SwapApiClientTest { +class SwapApiClientTest extends IndexerLogicBaseTest { @Autowired private SwapApiClient swapApiClient; diff --git a/starcoin-indexer/src/test/resources/application-integrationtest.properties b/starcoin-indexer/src/test/resources/application-integrationtest.properties index 63a2905..3893f95 100644 --- a/starcoin-indexer/src/test/resources/application-integrationtest.properties +++ b/starcoin-indexer/src/test/resources/application-integrationtest.properties @@ -1,20 +1,45 @@ -starcoin.seeds=localhost -starcoin.network=main -starcoin.bg_task.jobs= -starcoin.indexer.bulk_size=3 +#starcoin.seeds=localhost +#starcoin.network=main +#starcoin.bg_task.jobs= +#starcoin.indexer.bulk_size=3 +#jasypt.encryptor.bean=encryptorBean +#jasypt.encryptor.password= +#elasticsearch.host=localhost +#elasticsearch.port=9200 +#elasticsearch.protocol=http +#elasticsearch.username=elastic +#elasticsearch.connTimeout=10000 +#elasticsearch.socketTimeout=10000 +#elasticsearch.connectionRequestTimeout=2000 +#swap.contract.address=0x8c109349c6bd91411d6bc962e080c4a3 +#starcoin.swap.api.url=https://swap-api.starswap.xyz +#spring.datasource.url=jdbc:postgresql://localhost/${DB_SCHEMA} +#logging.file.name=logs/indexer.log +#spring.datasource.username=${DB_USER} +#spring.datasource.password=${DB_PWD} +#spring.jpa.properties.hibernate.default_schema=${NETWORK} + +# server.port=8300 +starcoin.seeds=${HOSTS} +starcoin.network=${NETWORK} +starcoin.bg_task.jobs=${BG_TASK_JOBS} +starcoin.indexer.bulk_size=${BULK_SIZE} +starcoin.indexer.txn_offset=${TXN_OFFSET} jasypt.encryptor.bean=encryptorBean -jasypt.encryptor.password= -elasticsearch.host=localhost -elasticsearch.port=9200 -elasticsearch.protocol=http -elasticsearch.username=elastic +jasypt.encryptor.password=${STARCOIN_ES_PWD} +elasticsearch.host=${STARCOIN_ES_URL} +elasticsearch.protocol=${STARCOIN_ES_PROTOCOL} +elasticsearch.port=${STARCOIN_ES_PORT} +elasticsearch.username=${STARCOIN_ES_USER} elasticsearch.connTimeout=10000 elasticsearch.socketTimeout=10000 elasticsearch.connectionRequestTimeout=2000 -swap.contract.address=0x8c109349c6bd91411d6bc962e080c4a3 -starcoin.swap.api.url=https://swap-api.starswap.xyz -spring.datasource.url=jdbc:postgresql://localhost/starcoin +swap.contract.address=${SWAP_CONTRACT_ADDR} +starcoin.swap.api.url=${SWAP_API_URL} logging.file.name=logs/indexer.log -spring.datasource.username=starcoin -spring.datasource.password= -spring.jpa.properties.hibernate.default_schema=main \ No newline at end of file +spring.datasource.hikari.minimum-idle=1 +spring.datasource.hikari.maximum-pool-size=2 +spring.datasource.url=${DS_URL} +spring.datasource.username=${DB_USER_NAME} +spring.datasource.password=${DB_PWD} +spring.jpa.properties.hibernate.default_schema=${DB_SCHEMA} \ No newline at end of file diff --git a/starcoin-indexer/src/test/resources/unit-test.env b/starcoin-indexer/src/test/resources/unit-test.env new file mode 100644 index 0000000..53e1d2c --- /dev/null +++ b/starcoin-indexer/src/test/resources/unit-test.env @@ -0,0 +1,18 @@ +HOSTS=halley.seed.starcoin.org +NETWORK=halley +BG_TASK_JOBS=dag_inspector +TXN_OFFSET=0 +BULK_SIZE=100 +STARCOIN_ES_PWD= +STARCOIN_ES_URL=localhost +STARCOIN_ES_PROTOCOL=http +STARCOIN_ES_PORT=9200 +STARCOIN_ES_USER= +SWAP_API_URL=https://swap-api.starswap.xyz +SWAP_CONTRACT_ADDR=0x8c109349c6bd91411d6bc962e080c4a3 +DS_URL=jdbc:postgresql://localhost/starcoin +DB_SCHEMA=halley +DB_USER_NAME=kaspa +DB_PWD=kaspa +PROGRAM_ARGS= +# auto_repair 9411700 \ No newline at end of file diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index 276a138..a2c969f 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -42,6 +42,10 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String DIBlocksAndEdgesAndHeightGroupsVo groups = new DIBlocksAndEdgesAndHeightGroupsVo(); List blockList = getBlockList(network, startHeight, endHeight); + if (blockList == null || blockList.isEmpty()) { + return groups; + } + groups.setBlocks(blockList); groups.setEdges(getEdgeList(network, startHeight, endHeight)); @@ -92,7 +96,7 @@ public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDiff * @param endHeight * @return */ - List getBlockList(String network, Long startHeight, Long endHeight) { + private List getBlockList(String network, Long startHeight, Long endHeight) { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_EDGE)); RangeQueryBuilder heightQuery = QueryBuilders.rangeQuery("height").gte(startHeight).lte(endHeight); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() @@ -167,7 +171,7 @@ public DIAppConfigVo getAppConfig() { return new DIAppConfigVo(); } - Long getMaxHeightFromStorage() { + private Long getMaxHeightFromStorage() { final String MAX_HEIGHT_FIELD = "max_height"; final String HEIGHT_FIELD = "height"; @@ -192,7 +196,7 @@ Long getMaxHeightFromStorage() { return 0L; } - DagInspectorBlock getBlockWithHashFromStorage(String network, String blockHash) { + private DagInspectorBlock getBlockWithHashFromStorage(String network, String blockHash) { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.should(QueryBuilders.termQuery("block_hash", blockHash)); @@ -210,7 +214,7 @@ DagInspectorBlock getBlockWithHashFromStorage(String network, String blockHash) return result.getContents().get(0); } - DagInspectorBlock getHeightWithDAAScoreFromStorage(String network, Integer daaScore) { + private DagInspectorBlock getHeightWithDAAScoreFromStorage(String network, Integer daaScore) { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.should(QueryBuilders.termQuery("daa_score", daaScore)); diff --git a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java index e75bca5..8facad8 100644 --- a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java +++ b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java @@ -1,9 +1,22 @@ package org.starcoin.scan.service; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.starcoin.scan.service.vo.DIBlocksAndEdgesAndHeightGroupsVo; public class DagInspectServiceTest { + final static String TEST_NETWORK = "halley"; + + @Autowired + DagInspectorService dagInspectorService; + @Test - public void test + public void testGetBlocksAndEdgesAndHeightGroups() { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlocksAndEdgesAndHeightGroups(TEST_NETWORK, 0L, 100L); + Assertions.assertNotEquals(0, vo.getBlocks().size()); + Assertions.assertNotEquals(0, vo.getEdges().size()); + Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + } } From c1910bce2c4677c1ac5fec43ee19c441d3de84e9 Mon Sep 17 00:00:00 2001 From: bobong Date: Tue, 18 Jun 2024 16:02:21 +0800 Subject: [PATCH 15/82] [dag inspector] Remove useless code --- .../indexer/handler/DagInspectorHandler.java | 19 ------------------- .../indexer/handler/DagInspectorTest.java | 17 +---------------- 2 files changed, 1 insertion(+), 35 deletions(-) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java index 968f780..114d5c1 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java @@ -301,25 +301,6 @@ void updateGroupSize(List groupList, Long height, Integ } } - private IndexRequest buildHeightGroup(Block block, Integer blockHeightIndex) { - IndexRequest request = new IndexRequest(dagInspectHeightGroupIndex); - XContentBuilder builder = null; - try { - builder = XContentFactory.jsonBuilder(); - builder.startObject(); - - builder.field("height", block.getHeader().getHeight()); - builder.field("size", blockHeightIndex + 1); - - builder.endObject(); - - } catch (Exception e) { - logger.error("build block error:", e); - } - request.source(builder); - return request; - } - private List getGroupHeightSizeFromStorage(List heights) throws IOException { List groupList = new ArrayList<>(); diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java index 028007e..5dd76d0 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java @@ -1,15 +1,6 @@ package org.starcoin.indexer.handler; -import org.junit.Assert; -import org.junit.Before; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.quartz.JobDetail; -import org.quartz.Scheduler; -import org.quartz.SchedulerException; -import org.quartz.Trigger; -import org.quartz.impl.StdSchedulerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.starcoin.bean.Block; import org.starcoin.bean.BlockHeader; @@ -19,20 +10,14 @@ import java.util.List; import java.util.Vector; -import static org.quartz.JobBuilder.newJob; -import static org.quartz.TriggerBuilder.newTrigger; - import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; + public class DagInspectorTest extends IndexerLogicBaseTest { @Autowired private DagInspectorHandler dagInspectorHandler; - public CountDownLatch latch; - private Scheduler scheduler; - Block new_test_block(String block_hash, Long height, String parent_hash, List parents_hash) { Block block = new Block(); BlockHeader header = new BlockHeader(); From 4d9c892d46942a70ee727f892f6b1c479289afb6 Mon Sep 17 00:00:00 2001 From: bobong Date: Tue, 18 Jun 2024 16:17:41 +0800 Subject: [PATCH 16/82] [dag inspector] revert project name --- pom.xml | 2 +- starcoin-commons/pom.xml | 2 +- starcoin-indexer/pom.xml | 2 +- starcoin-scan-api/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 7eb3b17..92904c5 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 org.starcoin - stcscan + starcoin-search pom 1.1 diff --git a/starcoin-commons/pom.xml b/starcoin-commons/pom.xml index f8f1f8d..9e1b8b2 100644 --- a/starcoin-commons/pom.xml +++ b/starcoin-commons/pom.xml @@ -3,7 +3,7 @@ xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - stcscan + starcoin-search org.starcoin 1.1 diff --git a/starcoin-indexer/pom.xml b/starcoin-indexer/pom.xml index c8d941a..7dd9187 100644 --- a/starcoin-indexer/pom.xml +++ b/starcoin-indexer/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - stcscan + starcoin-search org.starcoin 1.1 diff --git a/starcoin-scan-api/pom.xml b/starcoin-scan-api/pom.xml index 8383d46..088404f 100644 --- a/starcoin-scan-api/pom.xml +++ b/starcoin-scan-api/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - stcscan + starcoin-search org.starcoin 1.1 From d8161eb9e600fd3d873842fae17934e2951011c5 Mon Sep 17 00:00:00 2001 From: bobong Date: Tue, 18 Jun 2024 17:54:53 +0800 Subject: [PATCH 17/82] [dag inspector] fixed save data to Elasticsearch DB --- .../org/starcoin/utils/ExceptionWrap.java | 18 +++ starcoin-indexer/local.env | 3 +- .../indexer/handler/DagInspectorHandler.java | 134 ++++++++---------- 3 files changed, 81 insertions(+), 74 deletions(-) create mode 100644 starcoin-commons/src/main/java/org/starcoin/utils/ExceptionWrap.java diff --git a/starcoin-commons/src/main/java/org/starcoin/utils/ExceptionWrap.java b/starcoin-commons/src/main/java/org/starcoin/utils/ExceptionWrap.java new file mode 100644 index 0000000..6329b80 --- /dev/null +++ b/starcoin-commons/src/main/java/org/starcoin/utils/ExceptionWrap.java @@ -0,0 +1,18 @@ +package org.starcoin.utils; + +import java.util.function.Function; + +@FunctionalInterface +public interface ExceptionWrap { + R apply(T t) throws Exception; + + static Function wrap(ExceptionWrap checkedFunction) { + return t -> { + try { + return checkedFunction.apply(t); + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + } +} diff --git a/starcoin-indexer/local.env b/starcoin-indexer/local.env index 53e1d2c..a5c8382 100644 --- a/starcoin-indexer/local.env +++ b/starcoin-indexer/local.env @@ -1,4 +1,5 @@ -HOSTS=halley.seed.starcoin.org +# HOSTS=halley.seed.starcoin.org +HOSTS=localhost NETWORK=halley BG_TASK_JOBS=dag_inspector TXN_OFFSET=0 diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java index 114d5c1..e35ea46 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java @@ -11,20 +11,19 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; import org.starcoin.api.BlockRPCClient; import org.starcoin.api.Result; import org.starcoin.bean.*; import org.starcoin.constant.Constant; import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import org.starcoin.utils.ExceptionWrap; import javax.annotation.PostConstruct; import java.io.IOException; @@ -113,12 +112,13 @@ public void upsertDagInfoFromBlocks(List blockList) throws IOException { // Save all data into storage - List inspectorBlockList = new ArrayList<>(inspBlockMap.values()); - bulkRequest.add(buildSaveInspectorBlockRequest(inspectorBlockList)); - - List edgeList = buildEdgeDataFromNodeData(inspectorBlockList); - bulkRequest.add(buildSaveEdgeRequest(edgeList)); - bulkRequest.add(buildSaveHeightGroupRequest(inspHeightGroupList)); + buildSaveInspectorBlockRequest( + new ArrayList<>(inspBlockMap.values())).forEach(bulkRequest::add); + buildSaveEdgeRequest( + buildEdgeDataFromNodeData(new ArrayList<>(inspBlockMap.values())) + ).forEach(bulkRequest::add); + buildSaveHeightGroupRequest(inspHeightGroupList) + .forEach(bulkRequest::add); // Bulk save try { @@ -194,87 +194,75 @@ public Map bulkLoadInspectorBlock(List blockH return inspBlockMap; } - IndexRequest buildSaveInspectorBlockRequest(List blockList) { - IndexRequest request = new IndexRequest(dagInspectNodeIndex); + List buildSaveInspectorBlockRequest(List blockList) { if (blockList.isEmpty()) { - return request; + return new ArrayList<>(); } - blockList.forEach(block -> { - XContentBuilder builder; - try { - builder = XContentFactory.jsonBuilder(); - builder.startObject(); - { - builder.field("block_hash", block.getBlockHash()); - builder.field("height", block.getHeight()); - builder.field("timestamp", block.getTimestamp()); - builder.field("selected_parent_hash", block.getSelectedParentHash()); - builder.field("parent_ids", block.getParentIds()); - builder.field("daa_score", block.getDaaScore()); - builder.field("height_group_index", block.getHeightGroupIndex()); - builder.field("in_virtual_selected_parent_chain", block.getInVirtualSelectedParentChain()); - builder.field("mergeset_blue_ids", block.getMergeSetBlueIds()); - builder.field("mergeset_red_ids", block.getMergeSetRedIds()); - builder.field("color", block.getColor()); - } - builder.endObject(); - } catch (Exception e) { - logger.error("build block error:", e); + return blockList.stream().map(ExceptionWrap.wrap(block -> { + IndexRequest request = new IndexRequest(dagInspectNodeIndex); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.field("block_hash", block.getBlockHash()); + builder.field("height", block.getHeight()); + builder.field("timestamp", block.getTimestamp()); + builder.field("selected_parent_hash", block.getSelectedParentHash()); + builder.field("parent_ids", block.getParentIds()); + builder.field("daa_score", block.getDaaScore()); + builder.field("height_group_index", block.getHeightGroupIndex()); + builder.field("in_virtual_selected_parent_chain", block.getInVirtualSelectedParentChain()); + builder.field("mergeset_blue_ids", block.getMergeSetBlueIds()); + builder.field("mergeset_red_ids", block.getMergeSetRedIds()); + builder.field("color", block.getColor()); } - }); - return request; + builder.endObject(); + request.source(builder).id(block.getBlockHash()); + return request; + })).collect(Collectors.toList()); } - private IndexRequest buildSaveEdgeRequest(List edgeList) { - IndexRequest request = new IndexRequest(dagInspectEdgeIndex); + private List buildSaveEdgeRequest(List edgeList) throws IOException { if (edgeList.isEmpty()) { - return request; + return new ArrayList<>(); } - edgeList.forEach(edge -> { - XContentBuilder builder; - try { - builder = XContentFactory.jsonBuilder(); - builder.startObject(); - { - builder.field("from_block_hash", edge.getFromBlockHash()); - builder.field("to_block_hash", edge.getToBlockHash()); - builder.field("from_height", edge.getFromHeight()); - builder.field("to_height", edge.getToHeight()); - builder.field("from_group_index", edge.getFromHeightGroupIndex()); - builder.field("to_group_index", edge.getToHeightGroupIndex()); - } - builder.endObject(); - } catch (Exception e) { - logger.error("build block error:", e); + return edgeList.stream().map(ExceptionWrap.wrap(edge -> { + IndexRequest request = new IndexRequest(dagInspectEdgeIndex); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.field("from_block_hash", edge.getFromBlockHash()); + builder.field("to_block_hash", edge.getToBlockHash()); + builder.field("from_height", edge.getFromHeight()); + builder.field("to_height", edge.getToHeight()); + builder.field("from_group_index", edge.getFromHeightGroupIndex()); + builder.field("to_group_index", edge.getToHeightGroupIndex()); } - }); - return request; + builder.endObject(); + request.source(builder).id(String.format("%s_%s", edge.getFromBlockHash(), edge.getToBlockHash())); + return request; + })).collect(Collectors.toList()); } - IndexRequest buildSaveHeightGroupRequest(List heightGroupList) { - IndexRequest request = new IndexRequest(dagInspectHeightGroupIndex); + List buildSaveHeightGroupRequest(List heightGroupList) throws IOException { if (heightGroupList.isEmpty()) { - return request; + return new ArrayList<>(); } - heightGroupList.forEach(heightGroup -> { - XContentBuilder builder; - try { - builder = XContentFactory.jsonBuilder(); - builder.startObject(); - { - builder.field("height", heightGroup.getHeight()); - builder.field("size", heightGroup.getSize()); - } - builder.endObject(); - } catch (Exception e) { - logger.error("build block error:", e); - } - }); - return request; + return heightGroupList.stream().map(ExceptionWrap.wrap(dagInspectorHeightGroup -> { + IndexRequest request = new IndexRequest(dagInspectHeightGroupIndex); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field("height", dagInspectorHeightGroup.getHeight()); + builder.field("size", dagInspectorHeightGroup.getSize()); + builder.endObject(); + request.id(String.valueOf(dagInspectorHeightGroup.getHeight())).source(builder); + return request; + })).collect(Collectors.toList()); } /** * Get the size of the group at the specified height, or return the default size if the group does not exist. + * * @param groupList * @param height * @param defaultSize From 9be238ca1dd3f9ea3e611615bc7342a6bda91dfc Mon Sep 17 00:00:00 2001 From: bobong Date: Wed, 19 Jun 2024 11:05:28 +0800 Subject: [PATCH 18/82] [dag inspector] fixed write height group --- .../org/starcoin/indexer/handler/DagInspectorHandler.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java index e35ea46..319600e 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java @@ -284,9 +284,13 @@ void updateGroupSize(List groupList, Long height, Integ for (DagInspectorHeightGroup group : groupList) { if (group.getHeight().longValue() == height) { group.setSize(newSize); - break; + return ; } } + DagInspectorHeightGroup group = new DagInspectorHeightGroup(); + group.setHeight(height); + group.setSize(newSize); + groupList.add(group); } From c54e52d758105f14a474306063404bd2935db9a2 Mon Sep 17 00:00:00 2001 From: bobong Date: Wed, 19 Jun 2024 11:18:36 +0800 Subject: [PATCH 19/82] [dag inspector] rename files --- .../org/starcoin/indexer/handler/DagInspectorIndexer.java | 2 +- ...gInspectorHandler.java => DagInspectorIndexerHandler.java} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename starcoin-indexer/src/main/java/org/starcoin/indexer/handler/{DagInspectorHandler.java => DagInspectorIndexerHandler.java} (99%) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java index 893876f..b797646 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java @@ -23,7 +23,7 @@ public class DagInspectorIndexer extends QuartzJobBean { private static final Logger logger = LoggerFactory.getLogger(DagInspectorIndexer.class); @Autowired - private DagInspectorHandler inspectorHandler; + private DagInspectorIndexerHandler inspectorHandler; @Autowired private ElasticSearchHandler elasticSearchHandler; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java similarity index 99% rename from starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java rename to starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index 319600e..a5350bc 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -31,9 +31,9 @@ import java.util.stream.Collectors; @Service -public class DagInspectorHandler { +public class DagInspectorIndexerHandler { - private static final Logger logger = LoggerFactory.getLogger(DagInspectorHandler.class); + private static final Logger logger = LoggerFactory.getLogger(DagInspectorIndexerHandler.class); private static final String NODE_COLOR_GRAY = "gray"; private static final String NODE_COLOR_RED = "red"; From 128ef9c59946851012ba02f82548649f80407ed4 Mon Sep 17 00:00:00 2001 From: bobong Date: Wed, 19 Jun 2024 19:02:50 +0800 Subject: [PATCH 20/82] [dag inspector] fixed indexer handle the parent block in edge save --- .../indexer/handler/DagInspectorIndexer.java | 35 +--- .../handler/DagInspectorIndexerHandler.java | 159 ++++++++++++------ .../indexer/handler/DagInspectorTest.java | 5 +- .../scan/service/DagInspectorService.java | 22 ++- .../scan/service/DagInspectServiceTest.java | 8 + 5 files changed, 134 insertions(+), 95 deletions(-) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java index b797646..4b29ee2 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java @@ -14,9 +14,7 @@ import javax.annotation.PostConstruct; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; +import java.util.*; public class DagInspectorIndexer extends QuartzJobBean { @@ -81,42 +79,27 @@ protected void executeInternal(JobExecutionContext jobExecutionContext) { return; } - //read head + // Read chain header try { BlockHeader chainHeader = blockRPCClient.getChainHeader(); + // Calculate bulk size long headHeight = chainHeader.getHeight(); long bulkNumber = Math.min(headHeight - localBlockOffset.getBlockHeight(), bulkSize); int index = 1; Map blockMap = new HashMap<>(); while (index <= bulkNumber) { - long readNumber = localBlockOffset.getBlockHeight() + index; + long currentBlockHeight = localBlockOffset.getBlockHeight() + index; - logger.info("Start Get block number: {}", index); - Block block = blockRPCClient.getBlockByHeight(readNumber); + logger.info("Start Get block number: {}, currentBlockHeight: {}", index, currentBlockHeight); + Block block = blockRPCClient.getBlockByHeight(currentBlockHeight); if (block == null) { - logger.warn("get block null: {}", readNumber); + logger.warn("get block null: {}", currentBlockHeight); return; } - logger.info("Block number is: {}, block hash is: {}", readNumber, block.getHeader().getBlockHash()); - - for (String parentHash : block.getHeader().getParentsHash()) { - logger.info( - "Start get block hash : {}, parent hash: {}", - block.getHeader().getBlockHash(), - parentHash - ); - - if (!blockMap.containsKey(parentHash)) { - Block parentBlock = blockRPCClient.getBlockByHash(parentHash); - if (parentBlock == null) { - logger.warn("get parent block null: {}", parentHash); - continue; - } - blockMap.put(parentHash, parentBlock); - } - } + blockMap.put(block.getHeader().getBlockHash(), block); + //update current header currentHandleHeader = block.getHeader(); index++; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index a5350bc..fa43a4b 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -1,5 +1,6 @@ package org.starcoin.indexer.handler; +import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -59,21 +60,21 @@ public void initIndexs() throws IOException { dagInspectHeightGroupIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECT_HEIGHT_GROUP); } - public void upsertDagInfoFromBlocks(List blockList) throws IOException { + public void upsertDagInfoFromBlocks(List blockList) throws IOException, JSONRPC2SessionException { if (blockList.isEmpty()) { logger.warn("block list is empty"); return; } BulkRequest bulkRequest = new BulkRequest(); - // Handle blockGhostdagDataCache - Map inspBlockMap = bulkLoadInspectorBlock( + // Handle blockGhostdag Data Cache + Map dagBlockMap = bulkLoadDagBlock( blockList.stream() .map(block -> block.getHeader().getBlockHash()) .collect(Collectors.toList()) ); - List inspHeightGroupList = getGroupHeightSizeFromStorage( + List heightGroupList = getGroupHeightSizeFromStorage( blockList.stream() .map(block -> block.getHeader().getHeight()) .distinct() @@ -83,24 +84,26 @@ public void upsertDagInfoFromBlocks(List blockList) throws IOException { // Build block node data blockList.forEach(blockInfo -> { String currentBlockHash = blockInfo.getHeader().getBlockHash(); - DagInspectorBlock inspecBlock = inspBlockMap.get(currentBlockHash); - if (inspecBlock == null) { + DagInspectorBlock dagBlock = dagBlockMap.get(currentBlockHash); + if (dagBlock == null) { try { BlockGhostdagData ghostdagData = blockRPCClient.getBlockGhostdagData(currentBlockHash); - inspecBlock = new DagInspectorBlock(); - inspecBlock.setBlockHash(currentBlockHash); - inspecBlock.setTimestamp(blockInfo.getHeader().getTimestamp()); - inspecBlock.setColor(NODE_COLOR_GRAY); - inspecBlock.setDaaScore(ghostdagData.getBlueScore()); - inspecBlock.setHeight(blockInfo.getHeader().getHeight()); - inspecBlock.setSelectedParentHash(ghostdagData.getSelectedParent()); - inspecBlock.setParentIds(blockInfo.getHeader().getParentsHash()); - - Integer groupSize = getHeightGroupSizeOrDefault(inspHeightGroupList, inspecBlock.getHeight(), 0); - inspecBlock.setHeightGroupIndex(groupSize); - updateGroupSize(inspHeightGroupList, inspecBlock.getHeight(), groupSize + 1); - - inspBlockMap.put(currentBlockHash, inspecBlock); + dagBlock = new DagInspectorBlock(); + dagBlock.setBlockHash(currentBlockHash); + dagBlock.setTimestamp(blockInfo.getHeader().getTimestamp()); + dagBlock.setColor(NODE_COLOR_GRAY); + dagBlock.setDaaScore(ghostdagData.getBlueScore()); + dagBlock.setHeight(blockInfo.getHeader().getHeight()); + dagBlock.setSelectedParentHash(ghostdagData.getSelectedParent()); + dagBlock.setParentIds(blockInfo.getHeader().getParentsHash()); + // Block is the virtual selected parent chain because the list read from block height + dagBlock.setInVirtualSelectedParentChain(true); + + Integer groupSize = getHeightGroupSizeOrDefault(heightGroupList, dagBlock.getHeight(), 0); + dagBlock.setHeightGroupIndex(groupSize); + updateGroupSize(heightGroupList, dagBlock.getHeight(), groupSize + 1); + + dagBlockMap.put(currentBlockHash, dagBlock); } catch (JSONRPC2SessionException e) { throw new RuntimeException(e); @@ -110,15 +113,12 @@ public void upsertDagInfoFromBlocks(List blockList) throws IOException { } }); + buildSaveDagEdgeRequest(buildEdgeDataFromDagBlockDataMaybeUpate(dagBlockMap, heightGroupList)).forEach(bulkRequest::add); + buildSaveDagHeightGroupRequest(heightGroupList).forEach(bulkRequest::add); // Save all data into storage - buildSaveInspectorBlockRequest( - new ArrayList<>(inspBlockMap.values())).forEach(bulkRequest::add); - buildSaveEdgeRequest( - buildEdgeDataFromNodeData(new ArrayList<>(inspBlockMap.values())) - ).forEach(bulkRequest::add); - buildSaveHeightGroupRequest(inspHeightGroupList) - .forEach(bulkRequest::add); + buildSaveDagBlockRequest(new ArrayList<>(dagBlockMap.values())).forEach(bulkRequest::add); + // Bulk save try { @@ -129,42 +129,47 @@ public void upsertDagInfoFromBlocks(List blockList) throws IOException { } } - List buildEdgeDataFromNodeData(List inspecBlockList) { + List buildEdgeDataFromDagBlockDataMaybeUpate( + Map dagBlockMap, + List heightGroupList + ) throws JSONRPC2SessionException { List edgeList = new ArrayList<>(); - if (inspecBlockList.isEmpty()) { + if (dagBlockMap.isEmpty()) { return edgeList; } - for (DagInspectorBlock block : inspecBlockList) { + List newDagBlocks = new ArrayList<>(); + for (DagInspectorBlock block : dagBlockMap.values()) { if (block.getParentIds().isEmpty()) { continue; } - block.getParentIds().forEach(parentHash -> { - DagInspectorBlock parentBlock = inspecBlockList - .stream() - .filter(b -> b.getBlockHash().equalsIgnoreCase(parentHash)) - .findFirst() - .orElse(null); - if (parentBlock == null) { - logger.info("Parent block {} not found", parentHash); - return; + for (String parentHash : block.getParentIds()) { + DagInspectorBlock parentDagBlock = dagBlockMap.get(parentHash); + if (parentDagBlock == null) { + logger.info("Parent block not found: {} ", parentHash); + parentDagBlock = getDagInspectorBlockInfoFromHash(parentHash, heightGroupList); + + // Put into buffer list + newDagBlocks.add(parentDagBlock); } DagInspectorEdge edge = new DagInspectorEdge(); edge.setFromBlockHash(block.getBlockHash()); - edge.setToBlockHash(parentBlock.getBlockHash()); + edge.setToBlockHash(parentDagBlock.getBlockHash()); edge.setFromHeight(block.getHeight()); - edge.setToHeight(parentBlock.getHeight()); + edge.setToHeight(parentDagBlock.getHeight()); edge.setFromHeightGroupIndex(block.getHeightGroupIndex()); - edge.setToHeightGroupIndex(parentBlock.getHeightGroupIndex()); + edge.setToHeightGroupIndex(parentDagBlock.getHeightGroupIndex()); edgeList.add(edge); - }); + } } + // Put into map + newDagBlocks.forEach(dagBlock -> dagBlockMap.put(dagBlock.getBlockHash(), dagBlock)); return edgeList; } - public Map bulkLoadInspectorBlock(List blockHashList) { - Map inspBlockMap = new HashMap<>(); + public Map bulkLoadDagBlock(List blockHashList) { + Map dagBlockMap = new HashMap<>(); SearchRequest searchRequest = new SearchRequest(dagInspectNodeIndex); TermsQueryBuilder termQueryBuilder = QueryBuilders.termsQuery("block_hash", blockHashList); searchRequest.source(new SearchSourceBuilder().query(termQueryBuilder)); @@ -173,7 +178,7 @@ public Map bulkLoadInspectorBlock(List blockH JSONObject obj = JSONObject.parseObject(ServiceUtils.getJsonString(response)); List hits = obj.getJSONObject("hits").getJSONArray("hits").toJavaList(JSONObject.class); hits.forEach(hit -> { - JSONObject source = hit.getJSONObject("_source"); + JSONObject source = hit.getJSONObject("sourceAsMap"); DagInspectorBlock block = new DagInspectorBlock(); block.setBlockHash(source.getString("block_hash")); block.setTimestamp(source.getLong("timestamp")); @@ -182,19 +187,28 @@ public Map bulkLoadInspectorBlock(List blockH block.setHeight(source.getLong("height")); block.setHeightGroupIndex(source.getInteger("height_group_index")); block.setSelectedParentHash(source.getString("selected_parent_hash")); - block.setParentIds(source.getJSONArray("parent_ids").toJavaList(String.class)); - block.setInVirtualSelectedParentChain(source.getBoolean("in_virtual_selected_parent_chain")); - block.setMergeSetRedIds(source.getJSONArray("mergeset_red_ids").toJavaList(String.class)); - block.setMergeSetBlueIds(source.getJSONArray("mergeset_blue_ids").toJavaList(String.class)); - inspBlockMap.put(block.getBlockHash(), block); + + JSONArray parentIds = source.getJSONArray("parent_ids"); + block.setParentIds(parentIds != null ? parentIds.toJavaList(String.class) : new ArrayList<>()); + + Boolean virtual_selected = source.getBoolean("in_virtual_selected_parent_chain"); + block.setInVirtualSelectedParentChain(virtual_selected != null ? virtual_selected : false); + + JSONArray redIds = source.getJSONArray("mergeset_red_ids"); + block.setMergeSetRedIds(redIds != null ? redIds.toJavaList(String.class) : new ArrayList<>()); + + JSONArray blueIds = source.getJSONArray("mergeset_blue_ids"); + block.setMergeSetBlueIds(blueIds != null ? blueIds.toJavaList(String.class) : new ArrayList<>()); + + dagBlockMap.put(block.getBlockHash(), block); }); } catch (IOException e) { logger.error("bulkLoadInspectorBlock error:", e); } - return inspBlockMap; + return dagBlockMap; } - List buildSaveInspectorBlockRequest(List blockList) { + List buildSaveDagBlockRequest(List blockList) { if (blockList.isEmpty()) { return new ArrayList<>(); } @@ -221,7 +235,7 @@ List buildSaveInspectorBlockRequest(List blockL })).collect(Collectors.toList()); } - private List buildSaveEdgeRequest(List edgeList) throws IOException { + private List buildSaveDagEdgeRequest(List edgeList) throws IOException { if (edgeList.isEmpty()) { return new ArrayList<>(); } @@ -243,7 +257,7 @@ private List buildSaveEdgeRequest(List edgeList) })).collect(Collectors.toList()); } - List buildSaveHeightGroupRequest(List heightGroupList) throws IOException { + List buildSaveDagHeightGroupRequest(List heightGroupList) { if (heightGroupList.isEmpty()) { return new ArrayList<>(); } @@ -284,7 +298,7 @@ void updateGroupSize(List groupList, Long height, Integ for (DagInspectorHeightGroup group : groupList) { if (group.getHeight().longValue() == height) { group.setSize(newSize); - return ; + return; } } DagInspectorHeightGroup group = new DagInspectorHeightGroup(); @@ -308,4 +322,37 @@ private List getGroupHeightSizeFromStorage(List h return result.getContents(); } + + protected DagInspectorBlock getDagInspectorBlockInfoFromHash( + String blockHash, + List heightGroupList + ) throws JSONRPC2SessionException { + + Block blockInfo = blockRPCClient.getBlockByHash(blockHash); + BlockGhostdagData blockGhostdagData = blockRPCClient.getBlockGhostdagData(blockHash); + DagInspectorBlock dagBlock = new DagInspectorBlock(); + + Long blockHeight = blockInfo.getHeader().getHeight(); + + dagBlock.setBlockHash(blockHash); + dagBlock.setTimestamp(blockInfo.getHeader().getTimestamp()); + dagBlock.setColor(NODE_COLOR_GRAY); + dagBlock.setDaaScore(blockGhostdagData.getBlueScore()); + dagBlock.setHeight(blockInfo.getHeader().getHeight()); + dagBlock.setSelectedParentHash(blockGhostdagData.getSelectedParent()); + dagBlock.setParentIds(blockInfo.getHeader().getParentsHash()); + dagBlock.setInVirtualSelectedParentChain(false); + + // Height group list index + Integer groupSize = getHeightGroupSizeOrDefault(heightGroupList, blockHeight, 0); + dagBlock.setHeightGroupIndex(groupSize); + updateGroupSize(heightGroupList, blockHeight, groupSize + 1); + + logger.info("Get block info from hash: {}, height: {}, heightGroupIndex: {}", + blockHash, + blockInfo.getHeader().getHeight(), + dagBlock.getHeightGroupIndex() + ); + return dagBlock; + } } diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java index 5dd76d0..47cc137 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/DagInspectorTest.java @@ -10,13 +10,10 @@ import java.util.List; import java.util.Vector; -import java.util.concurrent.CountDownLatch; - - public class DagInspectorTest extends IndexerLogicBaseTest { @Autowired - private DagInspectorHandler dagInspectorHandler; + private DagInspectorIndexerHandler dagInspectorHandler; Block new_test_block(String block_hash, Long height, String parent_hash, List parents_hash) { Block block = new Block(); diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index a2c969f..67b7bbc 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -49,18 +49,20 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String groups.setBlocks(blockList); groups.setEdges(getEdgeList(network, startHeight, endHeight)); - groups.setHeightGroups(getHeightGroup( - network, + List heightList = blockList.stream() .map(DagInspectorBlock::getHeight) - .collect(Collectors.toList())) - ); + .collect(Collectors.toList()); + groups.setHeightGroups(getHeightGroup(network, heightList)); return groups; } public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash(String network, String targetHash, Integer heightDifference) { DagInspectorBlock block = getBlockWithHashFromStorage(network, targetHash); + if (block == null) { + throw new RuntimeException("Cannot find block by block hash"); + } Long endHeight = block.getHeight(); long startHeight = block.getHeight() - heightDifference; if (startHeight < 0L) { @@ -71,6 +73,9 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash(String network, String tar public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore(String network, Integer targetDAAScore, Integer heightDifference) { DagInspectorBlock block = getHeightWithDAAScoreFromStorage(network, targetDAAScore); + if (block == null) { + throw new RuntimeException("Cannot find block by block hash"); + } Long endHeight = block.getHeight(); long startHeight = block.getHeight() - heightDifference; if (startHeight < 0L) { @@ -97,11 +102,10 @@ public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDiff * @return */ private List getBlockList(String network, Long startHeight, Long endHeight) { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_EDGE)); - RangeQueryBuilder heightQuery = QueryBuilders.rangeQuery("height").gte(startHeight).lte(endHeight); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() - .query(QueryBuilders.boolQuery().must(heightQuery)) - .sort("height", SortOrder.ASC); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery("height").gte(startHeight).lte(endHeight))); + sourceBuilder .sort("height", SortOrder.ASC); searchRequest.source(sourceBuilder); SearchResponse searchResponse; try { diff --git a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java index 8facad8..4ecc748 100644 --- a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java +++ b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java @@ -2,9 +2,17 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.starcoin.scan.ScanApplication; import org.starcoin.scan.service.vo.DIBlocksAndEdgesAndHeightGroupsVo; +@RunWith(SpringRunner.class) +@SpringBootTest(classes = ScanApplication.class) +@TestPropertySource(locations = "classpath:application.properties") public class DagInspectServiceTest { final static String TEST_NETWORK = "halley"; From 4f937ee58c9a44c35f716726fead3ea45f68cc10 Mon Sep 17 00:00:00 2001 From: bobong Date: Thu, 20 Jun 2024 17:41:52 +0800 Subject: [PATCH 21/82] [dag inspector] fixed the unittest of dag API in Starcoin Scan service --- .../org/starcoin/bean/DagInspectorBlock.java | 18 +++- .../org/starcoin/bean/DagInspectorEdge.java | 12 +++ .../bean/DagInspectorHeightGroup.java | 8 ++ .../handler/DagInspectorIndexerHandler.java | 14 +-- .../indexer/handler/SwapIndexerTest.java | 1 - .../scan/service/DagInspectorService.java | 95 +++++++++++-------- .../vo/DIBlocksAndEdgesAndHeightGroupsVo.java | 9 ++ .../scan/service/DagInspectServiceTest.java | 43 +++++++-- 8 files changed, 143 insertions(+), 57 deletions(-) diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java index 82fd261..7af5287 100644 --- a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java @@ -1,6 +1,5 @@ package org.starcoin.bean; -import java.util.List; import java.util.List; public class DagInspectorBlock { @@ -103,4 +102,21 @@ public List getMergeSetBlueIds() { public void setMergeSetBlueIds(List mergeSetBlueIds) { this.mergeSetBlueIds = mergeSetBlueIds; } + + @Override + public String toString() { + return "DagInspectorBlock{" + + "blockHash='" + blockHash + '\'' + + ", timestamp=" + timestamp + + ", parentIds=" + parentIds + + ", height=" + height + + ", daaScore=" + daaScore + + ", heightGroupIndex=" + heightGroupIndex + + ", selectedParentHash='" + selectedParentHash + '\'' + + ", color='" + color + '\'' + + ", isInVirtualSelectedParentChain=" + isInVirtualSelectedParentChain + + ", mergeSetRedIds=" + mergeSetRedIds + + ", mergeSetBlueIds=" + mergeSetBlueIds + + '}'; + } } diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java index 7d31995..08f3c0e 100644 --- a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java @@ -55,4 +55,16 @@ public Integer getToHeightGroupIndex() { public void setToHeightGroupIndex(Integer toHeightGroupIndex) { this.toHeightGroupIndex = toHeightGroupIndex; } + + @Override + public String toString() { + return "DagInspectorEdge{" + + "fromBlockHash='" + fromBlockHash + '\'' + + ", toBlockHash='" + toBlockHash + '\'' + + ", fromHeight=" + fromHeight + + ", toHeight=" + toHeight + + ", fromHeightGroupIndex=" + fromHeightGroupIndex + + ", toHeightGroupIndex=" + toHeightGroupIndex + + '}'; + } } diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java index 4191534..fa24b24 100644 --- a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java @@ -19,4 +19,12 @@ public Integer getSize() { public void setSize(Integer size) { this.size = size; } + + @Override + public String toString() { + return "DagInspectorHeightGroup{" + + "height=" + height + + ", size=" + size + + '}'; + } } diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index fa43a4b..f0e5fd0 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -139,12 +139,12 @@ List buildEdgeDataFromDagBlockDataMaybeUpate( } List newDagBlocks = new ArrayList<>(); - for (DagInspectorBlock block : dagBlockMap.values()) { - if (block.getParentIds().isEmpty()) { + for (DagInspectorBlock dagBlock : dagBlockMap.values()) { + if (dagBlock.getParentIds().isEmpty()) { continue; } - for (String parentHash : block.getParentIds()) { + for (String parentHash : dagBlock.getParentIds()) { DagInspectorBlock parentDagBlock = dagBlockMap.get(parentHash); if (parentDagBlock == null) { logger.info("Parent block not found: {} ", parentHash); @@ -154,12 +154,12 @@ List buildEdgeDataFromDagBlockDataMaybeUpate( newDagBlocks.add(parentDagBlock); } DagInspectorEdge edge = new DagInspectorEdge(); - edge.setFromBlockHash(block.getBlockHash()); + edge.setFromBlockHash(dagBlock.getBlockHash()); edge.setToBlockHash(parentDagBlock.getBlockHash()); - edge.setFromHeight(block.getHeight()); + edge.setFromHeight(dagBlock.getHeight()); edge.setToHeight(parentDagBlock.getHeight()); - edge.setFromHeightGroupIndex(block.getHeightGroupIndex()); - edge.setToHeightGroupIndex(parentDagBlock.getHeightGroupIndex()); + edge.setFromHeightGroupIndex(dagBlock.getHeightGroupIndex() == null ? 0 : dagBlock.getHeightGroupIndex()); + edge.setToHeightGroupIndex(parentDagBlock.getHeightGroupIndex() == null ? 0 :parentDagBlock.getHeightGroupIndex()); edgeList.add(edge); } } diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java index 023759b..b1a343d 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.math.BigDecimal; -import java.util.Calendar; import java.util.List; import static org.starcoin.utils.DateTimeUtils.getTimeStamp; diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index 67b7bbc..315eb6a 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -1,5 +1,6 @@ package org.starcoin.scan.service; +import com.alibaba.fastjson.JSONObject; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -8,8 +9,12 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.Max; +import org.elasticsearch.search.aggregations.metrics.TopHits; +import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,6 +29,7 @@ import java.io.IOException; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; @Service @@ -38,7 +44,7 @@ public class DagInspectorService extends BaseService { private RestHighLevelClient client; - public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String network, Long startHeight, Long endHeight) { + public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String network, Long startHeight, Long endHeight) throws IOException { DIBlocksAndEdgesAndHeightGroupsVo groups = new DIBlocksAndEdgesAndHeightGroupsVo(); List blockList = getBlockList(network, startHeight, endHeight); @@ -49,16 +55,20 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String groups.setBlocks(blockList); groups.setEdges(getEdgeList(network, startHeight, endHeight)); - List heightList = + Set heightList = blockList.stream() .map(DagInspectorBlock::getHeight) - .collect(Collectors.toList()); + .collect(Collectors.toSet()); groups.setHeightGroups(getHeightGroup(network, heightList)); return groups; } - public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash(String network, String targetHash, Integer heightDifference) { + public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash( + String network, + String targetHash, + Integer heightDifference + ) throws IOException { DagInspectorBlock block = getBlockWithHashFromStorage(network, targetHash); if (block == null) { throw new RuntimeException("Cannot find block by block hash"); @@ -71,7 +81,7 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash(String network, String tar return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } - public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore(String network, Integer targetDAAScore, Integer heightDifference) { + public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore(String network, Integer targetDAAScore, Integer heightDifference) throws IOException { DagInspectorBlock block = getHeightWithDAAScoreFromStorage(network, targetDAAScore); if (block == null) { throw new RuntimeException("Cannot find block by block hash"); @@ -84,8 +94,8 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore(String network, Intege return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } - public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDifference) { - long endHeight = getMaxHeightFromStorage(); + public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Integer heightDifference) throws IOException { + long endHeight = getMaxHeightFromStorage(network); long startHeight = endHeight - heightDifference; if (startHeight < 0L) { startHeight = 0L; @@ -101,19 +111,14 @@ public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDiff * @param endHeight * @return */ - private List getBlockList(String network, Long startHeight, Long endHeight) { + private List getBlockList(String network, Long startHeight, Long endHeight) throws IOException { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery("height").gte(startHeight).lte(endHeight))); - sourceBuilder .sort("height", SortOrder.ASC); + sourceBuilder.sort("height", SortOrder.ASC); + sourceBuilder.size(endHeight.intValue() - startHeight.intValue()); searchRequest.source(sourceBuilder); - SearchResponse searchResponse; - try { - searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - logger.error("getEdgeList failed, startHeight: {}, endHeight: {}", startHeight, endHeight, e); - return null; - } + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); return result.getContents(); } @@ -152,13 +157,11 @@ List getEdgeList(String network, Long startHeight, Long endHei * @param heights * @return */ - List getHeightGroup(String network, List heights) { + List getHeightGroup(String network, Set heights) { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECT_HEIGHT_GROUP)); - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for (Long height : heights) { - boolQueryBuilder.should(QueryBuilders.termQuery("height", height)); - } - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query(QueryBuilders.termsQuery("height", heights)); + sourceBuilder.size(heights.size()); searchRequest.source(sourceBuilder); SearchResponse searchResponse; try { @@ -175,27 +178,35 @@ public DIAppConfigVo getAppConfig() { return new DIAppConfigVo(); } - private Long getMaxHeightFromStorage() { - final String MAX_HEIGHT_FIELD = "max_height"; - final String HEIGHT_FIELD = "height"; - - try { - SearchRequest searchRequest = new SearchRequest(Constant.DAG_INSPECTOR_NODE); - - // Build the SearchSourceBuilder with max aggregation - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.aggregation(AggregationBuilders.max(MAX_HEIGHT_FIELD).field(HEIGHT_FIELD)); - searchRequest.source(sourceBuilder); - - // Execute the search request - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - - // Handle the response - Max maxHeight = searchResponse.getAggregations().get("max_height"); - return (long) maxHeight.getValue(); + private Long getMaxHeightFromStorage(String network) throws IOException { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); - } catch (IOException e) { - e.printStackTrace(); + // Build the SearchSourceBuilder with max aggregation + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + TermsAggregationBuilder maxHeightAgg = AggregationBuilders + .terms("max_height_agg") + .field("height") + .size(1); + TopHitsAggregationBuilder topHitsAgg = AggregationBuilders + .topHits("top_hits") + .size(1); + + maxHeightAgg.subAggregation(topHitsAgg); + searchSourceBuilder.aggregation(maxHeightAgg); + searchRequest.source(searchSourceBuilder); + + // Execute the search request + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + Terms maxHeightTerms = searchResponse.getAggregations().get("max_height_agg"); + if (maxHeightTerms.getBuckets().size() > 0) { + Terms.Bucket bucket = maxHeightTerms.getBuckets().get(0); + TopHits topHits = bucket.getAggregations().get("top_hits"); + if (topHits.getHits().getHits().length > 0) { + String topHitsJson = topHits.getHits().getHits()[0].getSourceAsString(); + logger.info("Object with the max height: " + topHitsJson); + DagInspectorBlock block = JSONObject.parseObject(topHitsJson, DagInspectorBlock.class); + return block.getHeight(); + } } return 0L; } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java index e93975c..1ae3ddd 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java @@ -37,4 +37,13 @@ public List getHeightGroups() { public void setHeightGroups(List heightGroups) { this.heightGroups = heightGroups; } + + @Override + public String toString() { + return "DIBlocksAndEdgesAndHeightGroupsVo{" + + "blocks=" + blocks + + ", edges=" + edges + + ", heightGroups=" + heightGroups + + '}'; + } } \ No newline at end of file diff --git a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java index 4ecc748..4ce0a5d 100644 --- a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java +++ b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java @@ -2,17 +2,16 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; import org.starcoin.scan.ScanApplication; import org.starcoin.scan.service.vo.DIBlocksAndEdgesAndHeightGroupsVo; -@RunWith(SpringRunner.class) +import java.io.IOException; + @SpringBootTest(classes = ScanApplication.class) -@TestPropertySource(locations = "classpath:application.properties") +@TestPropertySource(locations = "classpath:application-unittest.properties") public class DagInspectServiceTest { final static String TEST_NETWORK = "halley"; @@ -21,10 +20,42 @@ public class DagInspectServiceTest { DagInspectorService dagInspectorService; @Test - public void testGetBlocksAndEdgesAndHeightGroups() { - DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlocksAndEdgesAndHeightGroups(TEST_NETWORK, 0L, 100L); + public void testGetBlocksAndEdgesAndHeightGroups() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlocksAndEdgesAndHeightGroups(TEST_NETWORK, 10000L, 10050L); + Assertions.assertNotEquals(0, vo.getBlocks().size()); + Assertions.assertNotEquals(0, vo.getEdges().size()); + Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + } + + @Test + public void testGetBlockHash() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlockHash( + TEST_NETWORK, + "0x93a4fc71929be2e435efe682d02260f8dd46824fe90e926e3b3ea5839f31e67c", + 10 + ); + Assertions.assertNotEquals(0, vo.getBlocks().size()); + Assertions.assertNotEquals(0, vo.getEdges().size()); + Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + System.out.println(vo); + } + + @Test + public void testGetBlockDAAScore() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlockDAAScore(TEST_NETWORK, 10409, 1); + Assertions.assertNotEquals(0, vo.getBlocks().size()); + Assertions.assertNotEquals(0, vo.getEdges().size()); + Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + System.out.println(vo); + } + + + @Test + public void testGetHead() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getHead(TEST_NETWORK, 100); Assertions.assertNotEquals(0, vo.getBlocks().size()); Assertions.assertNotEquals(0, vo.getEdges().size()); Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + System.out.println(vo); } } From 1991b4bc98442c616681ef26b9eb9432848f0e3c Mon Sep 17 00:00:00 2001 From: bobong Date: Thu, 20 Jun 2024 17:44:17 +0800 Subject: [PATCH 22/82] [dag inspector] commit the docker-compose.yaml for local enviroments --- docker-compose/docker-compose.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 35a0bdd..d46a3bc 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -1,3 +1,4 @@ +# This composer file is used to configure the local environment for debugging version: '3.8' services: @@ -19,6 +20,25 @@ services: ports: - "9200:9200" + hazelcast: + image: hazelcast/hazelcast:latest + container_name: hazelcast + ports: + - "5701:5701" + environment: + - HZ_CLUSTERNAME=stcscan-hazelcast-cluster + - HZ_NETWORK_JOIN_MULTICAST_ENABLED=false + - HZ_NETWORK_JOIN_TCPIP_ENABLED=true + - HZ_NETWORK_JOIN_TCPIP_MEMBERS=hazelcast + - HZ_CACHE_CODE_CACHE_STATISTICS_ENABLED=true + - HZ_CACHE_CODE_CACHE_EVICTION_SIZE=10000 + - HZ_CACHE_CODE_CACHE_EVICTION_MAX_SIZE_POLICY=ENTRY_COUNT + - HZ_CACHE_CODE_CACHE_EVICTION_EVICTION_POLICY=LFU + - HZ_CACHE_SESSION_STATISTICS_ENABLED=true + - HZ_CACHE_SESSION_EVICTION_SIZE=50000 + - HZ_CACHE_SESSION_EVICTION_MAX_SIZE_POLICY=ENTRY_COUNT + - HZ_CACHE_SESSION_EVICTION_EVICTION_POLICY=LRU + volumes: esdata: driver: local From 2286eaf2fb0cd2e4fadb37d0877262a273d7f3a4 Mon Sep 17 00:00:00 2001 From: bobong Date: Thu, 20 Jun 2024 19:55:54 +0800 Subject: [PATCH 23/82] [config] fixed db address --- kube/indexer/starcoin-indexer-halley-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/indexer/starcoin-indexer-halley-deployment.yaml b/kube/indexer/starcoin-indexer-halley-deployment.yaml index b4f24d3..0c74a09 100644 --- a/kube/indexer/starcoin-indexer-halley-deployment.yaml +++ b/kube/indexer/starcoin-indexer-halley-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD From 70704aab5684caed66d15285386fe1ecf86cc6ee Mon Sep 17 00:00:00 2001 From: bobong Date: Fri, 21 Jun 2024 08:39:06 +0800 Subject: [PATCH 24/82] [repair] add es repair for halley network --- ...coin-indexer-repair-halley-deployment.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 kube/indexer/starcoin-indexer-repair-halley-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-repair-halley-deployment.yaml b/kube/indexer/starcoin-indexer-repair-halley-deployment.yaml new file mode 100644 index 0000000..b9b7f93 --- /dev/null +++ b/kube/indexer/starcoin-indexer-repair-halley-deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: starcoin-indexer-repair-deployment + namespace: starcoin-halley + labels: + app: starcoin-indexer-repair +spec: + replicas: 1 + selector: + matchLabels: + app: starcoin-indexer-repair + template: + metadata: + labels: + app: starcoin-indexer-repair + spec: + containers: + - name: starcoin-indexer-repair + image: starcoin/starcoin_indexer:sha-3af1c2a + ports: + - containerPort: 8300 + env: + - name: HOSTS + value: "halley.seed.starcoin.org" + - name: NETWORK + value: "halley.0727" + - name: BG_TASK_JOBS + value: "indexer,secondary,market_cap,txn_payload" + - name: BULK_SIZE + value: "80" + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch-pw-elastic + key: password + - name: STARCOIN_ES_URL + value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + - name: TXN_OFFSET + value: "451993" + - name: STARCOIN_ES_PROTOCOL + value: "https" + - name: STARCOIN_ES_PORT + value: "443" + - name: STARCOIN_ES_USER + value: elastic + - name: SWAP_API_URL + value: "https://swap-api.starswap.xyz" + - name: SWAP_CONTRACT_ADDR + value: "0xbd7e8be8fae9f60f2f5136433e36a091" + - name: DS_URL + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + - name: DB_USER_NAME + value: "starcoin" + - name: DB_PWD + valueFrom: + secretKeyRef: + name: postgresql-starcoin-yu + key: password + - name: DB_SCHEMA + value: "halley" + - name: PROGRAM_ARGS + value: "auto_repair 1" + #valume: From 057b9c07fda536b3e538612262817e8abfa5fb27 Mon Sep 17 00:00:00 2001 From: bobong Date: Fri, 21 Jun 2024 14:11:07 +0800 Subject: [PATCH 25/82] [db error] upgrade starcoin java sdk to 1.2.2 for halley dag data --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index deece9e..063861d 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.starcoin sdk - 1.1.31 + 1.2.2 org.springframework.boot From 28832127c2736d359575ae89a7e5d3beec6a02ab Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Fri, 21 Jun 2024 22:29:35 +0800 Subject: [PATCH 26/82] [db error] fixed metadata load error --- kube/indexer/starcoin-indexer-halley-deployment.yaml | 6 +++--- kube/indexer/starcoin-indexer-repair-halley-deployment.yaml | 4 ++-- pom.xml | 2 +- .../test/resources/application-integrationtest.properties | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/kube/indexer/starcoin-indexer-halley-deployment.yaml b/kube/indexer/starcoin-indexer-halley-deployment.yaml index 0c74a09..1f10f4b 100644 --- a/kube/indexer/starcoin-indexer-halley-deployment.yaml +++ b/kube/indexer/starcoin-indexer-halley-deployment.yaml @@ -17,14 +17,14 @@ spec: spec: containers: - name: starcoin-indexer - image: starcoin/starcoin_indexer:sha-3af1c2a + image: starcoin/starcoin_indexer:halley_fix_db_1 ports: - containerPort: 8300 env: - name: HOSTS value: "halley.seed.starcoin.org" - name: NETWORK - value: "halley.0727" + value: "halley.20240621" - name: BG_TASK_JOBS value: "indexer,secondary,market_cap,txn_payload,swap_stats,txn_global_idx_update" - name: BULK_SIZE @@ -37,7 +37,7 @@ spec: - name: STARCOIN_ES_URL value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" - name: TXN_OFFSET - value: "451993" + value: "0" - name: STARCOIN_ES_PROTOCOL value: "https" - name: STARCOIN_ES_PORT diff --git a/kube/indexer/starcoin-indexer-repair-halley-deployment.yaml b/kube/indexer/starcoin-indexer-repair-halley-deployment.yaml index b9b7f93..f00aa76 100644 --- a/kube/indexer/starcoin-indexer-repair-halley-deployment.yaml +++ b/kube/indexer/starcoin-indexer-repair-halley-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: starcoin-indexer-repair - image: starcoin/starcoin_indexer:sha-3af1c2a + image: starcoin/starcoin_indexer:halley_fix_db_1 ports: - containerPort: 8300 env: @@ -37,7 +37,7 @@ spec: - name: STARCOIN_ES_URL value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" - name: TXN_OFFSET - value: "451993" + value: "0" - name: STARCOIN_ES_PROTOCOL value: "https" - name: STARCOIN_ES_PORT diff --git a/pom.xml b/pom.xml index 063861d..2ce1d86 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.starcoin sdk - 1.2.2 + 1.2.5 org.springframework.boot diff --git a/starcoin-indexer/src/test/resources/application-integrationtest.properties b/starcoin-indexer/src/test/resources/application-integrationtest.properties index 63a2905..99aade5 100644 --- a/starcoin-indexer/src/test/resources/application-integrationtest.properties +++ b/starcoin-indexer/src/test/resources/application-integrationtest.properties @@ -1,4 +1,4 @@ -starcoin.seeds=localhost +starcoin.seeds=halley.seed.starcoin.org starcoin.network=main starcoin.bg_task.jobs= starcoin.indexer.bulk_size=3 From 185b3c6e9008a68eedaaa94c7345783ee48797de Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Fri, 21 Jun 2024 22:30:53 +0800 Subject: [PATCH 27/82] [db error] fixed metadata load error --- kube/indexer/starcoin-indexer-halley-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/indexer/starcoin-indexer-halley-deployment.yaml b/kube/indexer/starcoin-indexer-halley-deployment.yaml index 1f10f4b..dc226c9 100644 --- a/kube/indexer/starcoin-indexer-halley-deployment.yaml +++ b/kube/indexer/starcoin-indexer-halley-deployment.yaml @@ -24,7 +24,7 @@ spec: - name: HOSTS value: "halley.seed.starcoin.org" - name: NETWORK - value: "halley.20240621" + value: "halley.0727" - name: BG_TASK_JOBS value: "indexer,secondary,market_cap,txn_payload,swap_stats,txn_global_idx_update" - name: BULK_SIZE From dfb03f4d8d433d2b5b74d0e745c1de95b51d91c6 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 22 Jun 2024 13:36:48 +0800 Subject: [PATCH 28/82] [vega net] add indexer kube config file for vega net --- .../starcoin-indexer-vega-deployment.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 kube/indexer/starcoin-indexer-vega-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-vega-deployment.yaml b/kube/indexer/starcoin-indexer-vega-deployment.yaml new file mode 100644 index 0000000..32eaaeb --- /dev/null +++ b/kube/indexer/starcoin-indexer-vega-deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: starcoin-indexer-deployment + namespace: starcoin-vega + labels: + app: starcoin-indexer +spec: + replicas: 1 + selector: + matchLabels: + app: starcoin-indexer + template: + metadata: + labels: + app: starcoin-indexer + spec: + containers: + - name: starcoin-indexer + image: starcoin/starcoin_indexer:halley_fix_db_1 + ports: + - containerPort: 8300 + env: + - name: HOSTS + value: "vega.seed.starcoin.org" + - name: NETWORK + value: "vega.20240622" + - name: BG_TASK_JOBS + value: "indexer,secondary,market_cap,txn_payload,swap_stats,txn_global_idx_update" + - name: BULK_SIZE + value: "80" + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch-pw-elastic + key: password + - name: STARCOIN_ES_URL + value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + - name: TXN_OFFSET + value: "0" + - name: STARCOIN_ES_PROTOCOL + value: "https" + - name: STARCOIN_ES_PORT + value: "443" + - name: STARCOIN_ES_USER + value: elastic + - name: SWAP_API_URL + value: "https://swap-api.starswap.xyz" + - name: SWAP_CONTRACT_ADDR + value: "0xbd7e8be8fae9f60f2f5136433e36a091" + - name: DS_URL + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + - name: DB_USER_NAME + value: "starcoin" + - name: DB_PWD + valueFrom: + secretKeyRef: + name: postgresql-starcoin-yu + key: password + - name: DB_SCHEMA + value: "vega" + - name: PROGRAM_ARGS + value: "" + #valume: From ca5ef6b3ff3c7069d320f5100a32aec01bf2e499 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 22 Jun 2024 20:39:51 +0800 Subject: [PATCH 29/82] [vega net] add indexer kube config file for vega net --- kube/indexer/starcoin-indexer-vega-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/indexer/starcoin-indexer-vega-deployment.yaml b/kube/indexer/starcoin-indexer-vega-deployment.yaml index 32eaaeb..690e388 100644 --- a/kube/indexer/starcoin-indexer-vega-deployment.yaml +++ b/kube/indexer/starcoin-indexer-vega-deployment.yaml @@ -24,7 +24,7 @@ spec: - name: HOSTS value: "vega.seed.starcoin.org" - name: NETWORK - value: "vega.20240622" + value: "vega.0727" - name: BG_TASK_JOBS value: "indexer,secondary,market_cap,txn_payload,swap_stats,txn_global_idx_update" - name: BULK_SIZE From a455f5fb53fe44e231d500f08f52f87137d11bdf Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 22 Jun 2024 21:44:22 +0800 Subject: [PATCH 30/82] [debug] add some logs for debug --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2ce1d86..b25795b 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.starcoin sdk - 1.2.5 + 1.2.6 org.springframework.boot From 50ed2f8f6bf2ff46e12f4c47cccec08c33e88e39 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 22 Jun 2024 23:26:38 +0800 Subject: [PATCH 31/82] [debug] update txn mapping for txn.info index --- kube/mappings/txn_info.mapping.json | 390 ++++++++++++++-------------- 1 file changed, 196 insertions(+), 194 deletions(-) diff --git a/kube/mappings/txn_info.mapping.json b/kube/mappings/txn_info.mapping.json index e0c47d3..d66b588 100644 --- a/kube/mappings/txn_info.mapping.json +++ b/kube/mappings/txn_info.mapping.json @@ -1,228 +1,231 @@ { - "mappings": { - "dynamic": "strict", - "properties": { - "timestamp": { - "type": "date" - }, - "deleted": { - "type": "boolean" - }, - "block_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "mappings" : { + "dynamic" : "strict", + "properties" : { + "block_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "transaction_type": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "block_metadata": { - "properties": { - "author": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "block_metadata" : { + "properties" : { + "author" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "author_auth_key": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "author_auth_key" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "chain_id": { - "type": "long" + "chain_id" : { + "type" : "long" }, - "number": { - "type": "long" + "number" : { + "type" : "long" }, - "parent_gas_used": { - "type": "long" + "parent_gas_used" : { + "type" : "long" }, - "parent_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "parent_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "timestamp": { - "type": "date" + "parents_hash" : { + "type" : "text" + }, + "timestamp" : { + "type" : "date" }, - "uncles": { - "type": "long" + "uncles" : { + "type" : "long" } } }, - "block_number": { - "type": "long" + "block_number" : { + "type" : "long" + }, + "deleted" : { + "type" : "boolean" }, - "event_root_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "event_root_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "events": { - "type": "nested", - "properties": { - "block_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "events" : { + "type" : "nested", + "properties" : { + "block_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "block_number": { - "type": "long" + "block_number" : { + "type" : "long" }, - "data": { - "type": "text" + "data" : { + "type" : "text" }, - "event_key": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "event_key" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "event_seq_number": { - "type": "long" + "event_seq_number" : { + "type" : "long" }, - "transaction_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "transaction_global_index" : { + "type" : "long" + }, + "transaction_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "transaction_index": { - "type": "long" - }, - "transaction_global_index": { - "type": "long" + "transaction_index" : { + "type" : "long" }, - "type_tag": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "type_tag" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } } } }, - "gas_used": { - "type": "long" + "gas_used" : { + "type" : "long" }, - "state_root_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "state_root_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "status": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "status" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "status_content": { - "type": "object", - "dynamic": true + "status_content" : { + "type" : "object", + "dynamic" : "true" + }, + "timestamp" : { + "type" : "date" }, - "transaction_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "transaction_global_index" : { + "type" : "long" + }, + "transaction_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "transaction_index": { - "type": "long" + "transaction_index" : { + "type" : "long" }, - "transaction_global_index": { - "type": "long" + "transaction_type" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } }, - "user_transaction": { - "properties": { - "authenticator": { - "properties": { - "Ed25519": { - "properties": { - "public_key": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "user_transaction" : { + "properties" : { + "authenticator" : { + "properties" : { + "Ed25519" : { + "properties" : { + "public_key" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "signature": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "signature" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } } } }, - "MultiEd25519": { - "properties": { - "public_key": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "MultiEd25519" : { + "properties" : { + "public_key" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "signature": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "signature" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } } @@ -230,53 +233,53 @@ } } }, - "raw_txn": { - "properties": { - "chain_id": { - "type": "long" + "raw_txn" : { + "properties" : { + "chain_id" : { + "type" : "long" }, - "expiration_timestamp_secs": { - "type": "date", - "format": "epoch_second" + "expiration_timestamp_secs" : { + "type" : "date", + "format" : "epoch_second" }, - "gas_token_code": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "gas_token_code" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "gas_unit_price": { - "type": "long" + "gas_unit_price" : { + "type" : "long" }, - "max_gas_amount": { - "type": "long" + "max_gas_amount" : { + "type" : "long" }, - "payload": { - "type": "text" + "payload" : { + "type" : "text" }, - "sender": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "sender" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "sequence_number": { - "type": "long" + "sequence_number" : { + "type" : "long" } } }, - "transaction_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "transaction_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } } @@ -284,5 +287,4 @@ } } } -} - +} \ No newline at end of file From fd67cbbd56bcadd695bbfc0234943eb2af5d6b4c Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sun, 23 Jun 2024 19:06:38 +0800 Subject: [PATCH 32/82] [env] add kibana for local enviroment --- docker-compose/docker-compose.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index d46a3bc..1670730 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -39,6 +39,17 @@ services: - HZ_CACHE_SESSION_EVICTION_MAX_SIZE_POLICY=ENTRY_COUNT - HZ_CACHE_SESSION_EVICTION_EVICTION_POLICY=LRU + kibana: + image: docker.elastic.co/kibana/kibana:7.17.2 + container_name: kibana + environment: + ELASTICSEARCH_URL: http://elasticsearch:9200 + ELASTICSEARCH_HOSTS: http://elasticsearch:9200 + ports: + - 5601:5601 + depends_on: + - elasticsearch + volumes: esdata: driver: local From 107ed193e08e4cbd7c934ab2d9a96b0d8440cf87 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sun, 23 Jun 2024 20:27:16 +0800 Subject: [PATCH 33/82] [indxer] Update ES component template for transaction info mapping --- kube/indexer/starcoin-indexer-vega-deployment.yaml | 2 +- kube/mappings/txn_info.mapping.json | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/kube/indexer/starcoin-indexer-vega-deployment.yaml b/kube/indexer/starcoin-indexer-vega-deployment.yaml index 690e388..a130153 100644 --- a/kube/indexer/starcoin-indexer-vega-deployment.yaml +++ b/kube/indexer/starcoin-indexer-vega-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: starcoin-indexer - image: starcoin/starcoin_indexer:halley_fix_db_1 + image: starcoin/starcoin_indexer:halley_fix_db_2 ports: - containerPort: 8300 env: diff --git a/kube/mappings/txn_info.mapping.json b/kube/mappings/txn_info.mapping.json index d66b588..610d82c 100644 --- a/kube/mappings/txn_info.mapping.json +++ b/kube/mappings/txn_info.mapping.json @@ -282,6 +282,9 @@ "ignore_above" : 256 } } + }, + "decoded_payload": { + "type" : "text" } } } From 0ccccbe196725f623563a4fe575fa65983420c85 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sun, 23 Jun 2024 21:56:14 +0800 Subject: [PATCH 34/82] [indxer] Update ES component template for raw_txn info mapping --- kube/mappings/txn_info.mapping.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kube/mappings/txn_info.mapping.json b/kube/mappings/txn_info.mapping.json index 610d82c..12d0d76 100644 --- a/kube/mappings/txn_info.mapping.json +++ b/kube/mappings/txn_info.mapping.json @@ -271,6 +271,9 @@ }, "sequence_number" : { "type" : "long" + }, + "decoded_payload": { + "type" : "text" } } }, @@ -282,9 +285,6 @@ "ignore_above" : 256 } } - }, - "decoded_payload": { - "type" : "text" } } } From 28dea1791ed212f061783acdf5f0b506a195a70e Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Mon, 24 Jun 2024 15:54:56 +0800 Subject: [PATCH 35/82] [dag inspector] Add ES component template and index template for dag inspector data struct --- .../mappings/dag_inspector_block.mapping.json | 57 +++++++++++++++++++ kube/mappings/dag_inspector_edge.mapping.json | 36 ++++++++++++ kube/mappings/dag_inspector_height_group.json | 12 ++++ kube/mappings/es_pipeline.scripts | 42 ++++++++++++++ .../java/org/starcoin/constant/Constant.java | 2 +- .../handler/DagInspectorIndexerHandler.java | 2 +- .../scan/service/DagInspectorService.java | 9 ++- 7 files changed, 153 insertions(+), 7 deletions(-) create mode 100644 kube/mappings/dag_inspector_block.mapping.json create mode 100644 kube/mappings/dag_inspector_edge.mapping.json create mode 100644 kube/mappings/dag_inspector_height_group.json diff --git a/kube/mappings/dag_inspector_block.mapping.json b/kube/mappings/dag_inspector_block.mapping.json new file mode 100644 index 0000000..0345422 --- /dev/null +++ b/kube/mappings/dag_inspector_block.mapping.json @@ -0,0 +1,57 @@ +{ + "mappings" : { + "properties" : { + "block_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "color" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "daa_score" : { + "type" : "long" + }, + "height" : { + "type" : "long" + }, + "height_group_index" : { + "type" : "long" + }, + "in_virtual_selected_parent_chain" : { + "type" : "boolean" + }, + "parent_ids" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "selected_parent_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "timestamp" : { + "type" : "long" + } + } + } +} \ No newline at end of file diff --git a/kube/mappings/dag_inspector_edge.mapping.json b/kube/mappings/dag_inspector_edge.mapping.json new file mode 100644 index 0000000..00655bf --- /dev/null +++ b/kube/mappings/dag_inspector_edge.mapping.json @@ -0,0 +1,36 @@ +{ + "mappings" : { + "properties" : { + "from_block_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "from_group_index" : { + "type" : "long" + }, + "from_height" : { + "type" : "long" + }, + "to_block_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "to_group_index" : { + "type" : "long" + }, + "to_height" : { + "type" : "long" + } + } + } +} \ No newline at end of file diff --git a/kube/mappings/dag_inspector_height_group.json b/kube/mappings/dag_inspector_height_group.json new file mode 100644 index 0000000..24f6863 --- /dev/null +++ b/kube/mappings/dag_inspector_height_group.json @@ -0,0 +1,12 @@ +{ + "mappings" : { + "properties" : { + "height" : { + "type" : "long" + }, + "size" : { + "type" : "long" + } + } + } +} \ No newline at end of file diff --git a/kube/mappings/es_pipeline.scripts b/kube/mappings/es_pipeline.scripts index 6c7775a..b304611 100644 --- a/kube/mappings/es_pipeline.scripts +++ b/kube/mappings/es_pipeline.scripts @@ -107,6 +107,27 @@ PUT _component_template/block-mappings # copy market_cap.mapping.json to here. } } + + PUT _component_template/dag_inspector_block-mappings + { + "template": { + # copy dag_inspector_block.mapping.json to here. + } + } + + PUT _component_template/dag_inspector_edge-mappings + { + "template": { + # copy dag_inspector_edge.mapping.json to here. + } + } + + PUT _component_template/dag_inspector_edge-mappings + { + "template": { + # copy dag_inspector_edge.mapping.json to here. + } + } 4. PUT _index_template/blocks-template { @@ -189,4 +210,25 @@ PUT _component_template/block-mappings "index_patterns": ["*.market_cap"], "priority": 500, "composed_of": ["market_cap-mappings"] + } + + PUT _index_template/dag_inspector_block-template + { + "index_patterns": ["*.dag_inspector_block"], + "priority": 500, + "composed_of": ["dag_inspector_block-mappings"] + } + + PUT _index_template/dag_inspector_edge-template + { + "index_patterns": ["*.dag_inspector_edge"], + "priority": 500, + "composed_of": ["dag_inspector_edge-mappings"] + } + + PUT _index_template/dag_inspector_height_group-template + { + "index_patterns": ["*.dag_inspector_height_group"], + "priority": 500, + "composed_of": ["dag_inspector_height_group-mappings"] } \ No newline at end of file diff --git a/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java b/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java index 395b1b2..fb404ae 100644 --- a/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java +++ b/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java @@ -15,7 +15,7 @@ public class Constant { public static final String MARKET_CAP_INDEX = "market_cap"; public static final String PAYLOAD_INDEX = "txn_payloads"; public static final String TOKEN_INFO_INDEX = "token_info"; - public static final String DAG_INSPECTOR_NODE = "dag_inspector_node"; + public static final String DAG_INSPECTOR_BLOCK = "dag_inspector_block"; public static final String DAG_INSPECTOR_EDGE = "dag_inspector_edge"; public static final String DAG_INSPECT_HEIGHT_GROUP = "dag_inspector_height_group"; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index f0e5fd0..ea3b030 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -55,7 +55,7 @@ public class DagInspectorIndexerHandler { @PostConstruct public void initIndexs() throws IOException { - dagInspectNodeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_NODE); + dagInspectNodeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_BLOCK); dagInspectEdgeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_EDGE); dagInspectHeightGroupIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECT_HEIGHT_GROUP); } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index 315eb6a..18afa30 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -10,7 +10,6 @@ import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.Max; import org.elasticsearch.search.aggregations.metrics.TopHits; import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -112,7 +111,7 @@ public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Integer heightD * @return */ private List getBlockList(String network, Long startHeight, Long endHeight) throws IOException { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery("height").gte(startHeight).lte(endHeight))); sourceBuilder.sort("height", SortOrder.ASC); @@ -179,7 +178,7 @@ public DIAppConfigVo getAppConfig() { } private Long getMaxHeightFromStorage(String network) throws IOException { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); // Build the SearchSourceBuilder with max aggregation SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -212,7 +211,7 @@ private Long getMaxHeightFromStorage(String network) throws IOException { } private DagInspectorBlock getBlockWithHashFromStorage(String network, String blockHash) { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.should(QueryBuilders.termQuery("block_hash", blockHash)); @@ -230,7 +229,7 @@ private DagInspectorBlock getBlockWithHashFromStorage(String network, String blo } private DagInspectorBlock getHeightWithDAAScoreFromStorage(String network, Integer daaScore) { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_NODE)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.should(QueryBuilders.termQuery("daa_score", daaScore)); From 860ab0c616b3b7cacfc406222a7bd77194cc0198 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Mon, 24 Jun 2024 16:53:18 +0800 Subject: [PATCH 36/82] [dag inspector] Upgrade java sdk to 1.2.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 92904c5..00de068 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.starcoin sdk - 1.2.2 + 1.2.5 org.springframework.boot From 43ed6bb089c399322be9298ac7c1b03968e8f0b0 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Tue, 25 Jun 2024 16:05:45 +0800 Subject: [PATCH 37/82] [dag inspector] fixed the controller api interface --- .../controller/DagInspectorController.java | 49 +++++++++++++++++-- .../scan/service/DagInspectorService.java | 34 ++++++++++--- .../scan/service/DagInspectServiceTest.java | 13 +++-- 3 files changed, 81 insertions(+), 15 deletions(-) diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java index b28d8f9..a14263d 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java @@ -3,8 +3,11 @@ import io.swagger.annotations.Api; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; -import org.starcoin.bean.Block; import org.starcoin.scan.service.DagInspectorService; +import org.starcoin.scan.service.vo.DIAppConfigVo; +import org.starcoin.scan.service.vo.DIBlocksAndEdgesAndHeightGroupsVo; + +import java.util.List; @Api(tags = "dag-inspector") @RestController @@ -14,9 +17,45 @@ public class DagInspectorController { @Autowired DagInspectorService dagInspectorService; - @GetMapping("/{network}/") - public Block getBlock(@PathVariable("network") String network, @RequestParam String id) throws Exception { - // return dagInspectorService.getBlock(network, id); - return null; + @GetMapping("/{network}/blocksBetweenHeights") + public DIBlocksAndEdgesAndHeightGroupsVo getBlocksBetweenHeights( + @PathVariable("network") String network, + @RequestParam Long startHeight, + @RequestParam Long endHeight + ) throws Exception { + return dagInspectorService.getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); + } + + @GetMapping("/{network}/header") + public DIBlocksAndEdgesAndHeightGroupsVo header( + @PathVariable("network") String network, + @RequestParam Long heightDifferent + ) throws Exception { + return dagInspectorService.getHead(network, heightDifferent); + } + + @GetMapping("/{network}/blockHash") + public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash( + @PathVariable("network") String network, + @RequestParam String blockHash, + @RequestParam Long heightDifference + ) throws Exception { + return dagInspectorService.getBlockHash(network, blockHash, heightDifference); + } + + @GetMapping("/{network}/blockDAAScore") + public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore( + @PathVariable("network") String network, + @RequestParam Long daaScore, + @RequestParam Long heightDifference + ) throws Exception { + return dagInspectorService.getBlockDAAScore(network, daaScore, heightDifference); + } + + @GetMapping("/{network}/appConfig") + public DIAppConfigVo getAppConfig( + @PathVariable("network") String network + ) throws Exception { + return dagInspectorService.getAppConfig(network); } } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index 18afa30..b64fd28 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -66,7 +66,7 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash( String network, String targetHash, - Integer heightDifference + Long heightDifference ) throws IOException { DagInspectorBlock block = getBlockWithHashFromStorage(network, targetHash); if (block == null) { @@ -80,7 +80,11 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash( return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } - public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore(String network, Integer targetDAAScore, Integer heightDifference) throws IOException { + public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore( + String network, + Long targetDAAScore, + Long heightDifference + ) throws IOException { DagInspectorBlock block = getHeightWithDAAScoreFromStorage(network, targetDAAScore); if (block == null) { throw new RuntimeException("Cannot find block by block hash"); @@ -93,7 +97,7 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore(String network, Intege return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } - public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Integer heightDifference) throws IOException { + public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDifference) throws IOException { long endHeight = getMaxHeightFromStorage(network); long startHeight = endHeight - heightDifference; if (startHeight < 0L) { @@ -156,7 +160,7 @@ List getEdgeList(String network, Long startHeight, Long endHei * @param heights * @return */ - List getHeightGroup(String network, Set heights) { + protected List getHeightGroup(String network, Set heights) { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECT_HEIGHT_GROUP)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() .query(QueryBuilders.termsQuery("height", heights)); @@ -173,7 +177,7 @@ List getHeightGroup(String network, Set heights) return result.getContents(); } - public DIAppConfigVo getAppConfig() { + public DIAppConfigVo getAppConfig(String network) { return new DIAppConfigVo(); } @@ -228,7 +232,7 @@ private DagInspectorBlock getBlockWithHashFromStorage(String network, String blo return result.getContents().get(0); } - private DagInspectorBlock getHeightWithDAAScoreFromStorage(String network, Integer daaScore) { + private DagInspectorBlock getHeightWithDAAScoreFromStorage(String network, Long daaScore) { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.should(QueryBuilders.termQuery("daa_score", daaScore)); @@ -246,4 +250,22 @@ private DagInspectorBlock getHeightWithDAAScoreFromStorage(String network, Integ return result.getContents().get(0); } + public List getBlocksByHeight(String network, Long height) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.should(QueryBuilders.termQuery("height", height)); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("getBlocksByHeight failed, blockHash: {}", height, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); + return result.getContents(); + } + } diff --git a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java index 4ce0a5d..84b4f61 100644 --- a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java +++ b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java @@ -21,7 +21,8 @@ public class DagInspectServiceTest { @Test public void testGetBlocksAndEdgesAndHeightGroups() throws IOException { - DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlocksAndEdgesAndHeightGroups(TEST_NETWORK, 10000L, 10050L); + DIBlocksAndEdgesAndHeightGroupsVo vo = + dagInspectorService.getBlocksAndEdgesAndHeightGroups(TEST_NETWORK, 10000L, 10050L); Assertions.assertNotEquals(0, vo.getBlocks().size()); Assertions.assertNotEquals(0, vo.getEdges().size()); Assertions.assertNotEquals(0, vo.getHeightGroups().size()); @@ -32,7 +33,7 @@ public void testGetBlockHash() throws IOException { DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlockHash( TEST_NETWORK, "0x93a4fc71929be2e435efe682d02260f8dd46824fe90e926e3b3ea5839f31e67c", - 10 + 10L ); Assertions.assertNotEquals(0, vo.getBlocks().size()); Assertions.assertNotEquals(0, vo.getEdges().size()); @@ -42,7 +43,11 @@ public void testGetBlockHash() throws IOException { @Test public void testGetBlockDAAScore() throws IOException { - DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlockDAAScore(TEST_NETWORK, 10409, 1); + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlockDAAScore( + TEST_NETWORK, + 10409L, + 1L + ); Assertions.assertNotEquals(0, vo.getBlocks().size()); Assertions.assertNotEquals(0, vo.getEdges().size()); Assertions.assertNotEquals(0, vo.getHeightGroups().size()); @@ -52,7 +57,7 @@ public void testGetBlockDAAScore() throws IOException { @Test public void testGetHead() throws IOException { - DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getHead(TEST_NETWORK, 100); + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getHead(TEST_NETWORK, 100L); Assertions.assertNotEquals(0, vo.getBlocks().size()); Assertions.assertNotEquals(0, vo.getEdges().size()); Assertions.assertNotEquals(0, vo.getHeightGroups().size()); From 33ff9982f394ea1e2379eee8e6f65f3f487e9c13 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Tue, 25 Jun 2024 16:07:10 +0800 Subject: [PATCH 38/82] [dag inspector] Change the indexer block fetching process from sequential to parallel --- .../indexer/handler/DagInspectorIndexer.java | 61 +++++++++++++++++++ .../handler/DagInspectorIndexerHandler.java | 3 +- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java index 4b29ee2..f4aacc3 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java @@ -15,6 +15,8 @@ import javax.annotation.PostConstruct; import java.io.IOException; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.LongStream; public class DagInspectorIndexer extends QuartzJobBean { @@ -79,6 +81,63 @@ protected void executeInternal(JobExecutionContext jobExecutionContext) { return; } + // fetchAndProcessBlockSequel(); + fetchAndProcessBlocksParallel(); + } + + public void fetchAndProcessBlocksParallel() { + logger.info("Entered"); + try { + // Read chain header + BlockHeader chainHeader = blockRPCClient.getChainHeader(); + + // Calculate bulk size + long headHeight = chainHeader.getHeight(); + long bulkNumber = Math.min(headHeight - localBlockOffset.getBlockHeight(), bulkSize); + + ConcurrentHashMap blockMap = new ConcurrentHashMap<>(); + + LongStream.rangeClosed(1, bulkNumber).parallel().forEach(index -> { + long currentBlockHeight = localBlockOffset.getBlockHeight() + index; + + logger.info("Start Get block number: {}, currentBlockHeight: {}", index, currentBlockHeight); + Block block; + try { + block = blockRPCClient.getBlockByHeight(currentBlockHeight); + if (block == null) { + logger.warn("get block null: {}", currentBlockHeight); + return; + } + + blockMap.put(block.getHeader().getBlockHash(), block); + + logger.info("add block: {}", block.getHeader()); + } catch (Exception e) { + logger.error("Error getting block at height {}: ", currentBlockHeight, e); + } + }); + + // Process the collected blocks + inspectorHandler.upsertDagInfoFromBlocks(new ArrayList<>(blockMap.values())); + + // Update offset with the last processed block + BlockHeader lastBlockHeader = blockMap.values().stream() + .map(Block::getHeader) + .max(Comparator.comparingLong(BlockHeader::getHeight)).orElseThrow(); + + localBlockOffset.setBlockHeight(lastBlockHeader.getHeight()); + localBlockOffset.setBlockHash(lastBlockHeader.getBlockHash()); + elasticSearchHandler.setRemoteOffset(localBlockOffset); + logger.info("indexer update success: {}", localBlockOffset); + } catch (JSONRPC2SessionException | IOException e) { + logger.error("chain header error:", e); + } finally { + logger.info("Exited"); + } + + } + + public void fetchAndProcessBlockSequel() { // Read chain header try { BlockHeader chainHeader = blockRPCClient.getChainHeader(); @@ -116,4 +175,6 @@ protected void executeInternal(JobExecutionContext jobExecutionContext) { logger.error("chain header error:", e); } } + } + diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index ea3b030..58ebbee 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -97,7 +97,7 @@ public void upsertDagInfoFromBlocks(List blockList) throws IOException, J dagBlock.setSelectedParentHash(ghostdagData.getSelectedParent()); dagBlock.setParentIds(blockInfo.getHeader().getParentsHash()); // Block is the virtual selected parent chain because the list read from block height - dagBlock.setInVirtualSelectedParentChain(true); + dagBlock.setInVirtualSelectedParentChain(ghostdagData.getMergesetBlues().contains(currentBlockHash)); Integer groupSize = getHeightGroupSizeOrDefault(heightGroupList, dagBlock.getHeight(), 0); dagBlock.setHeightGroupIndex(groupSize); @@ -119,7 +119,6 @@ public void upsertDagInfoFromBlocks(List blockList) throws IOException, J // Save all data into storage buildSaveDagBlockRequest(new ArrayList<>(dagBlockMap.values())).forEach(bulkRequest::add); - // Bulk save try { BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT); From cbba8a2a437dfdd226dc125f1387f154a700bd4c Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Tue, 25 Jun 2024 16:10:05 +0800 Subject: [PATCH 39/82] [dag inspector] Added a new return structure to the getBlockByHeight interface to meet the display requirements of stcscan --- .../scan/controller/BlockV2Controller.java | 74 +++++++++++++++---- .../org/starcoin/scan/service/vo/BlockVo.java | 57 ++++++++++++++ 2 files changed, 116 insertions(+), 15 deletions(-) create mode 100644 starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/BlockVo.java diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java index fca6985..722c335 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java @@ -7,9 +7,14 @@ import org.starcoin.api.Result; import org.starcoin.bean.Block; import org.starcoin.bean.BlockInfoEntity; +import org.starcoin.bean.DagInspectorBlock; import org.starcoin.bean.UncleBlock; import org.starcoin.scan.service.BlockInfoService; import org.starcoin.scan.service.BlockService; +import org.starcoin.scan.service.DagInspectorService; +import org.starcoin.scan.service.vo.BlockVo; + +import java.util.List; @Api(tags = "block") @RestController @@ -21,64 +26,103 @@ public class BlockV2Controller { @Autowired private BlockInfoService blockInfoService; + @Autowired + private DagInspectorService dagInspectorService; + @ApiOperation("get block by ID") @GetMapping("/{network}/") - public Block getBlock(@PathVariable("network") String network, @RequestParam String id) throws Exception { + public Block getBlock( + @PathVariable("network") String network, + @RequestParam String id + ) throws Exception { return blockService.getBlock(network, id); } @ApiOperation("get block by hash") @GetMapping("/{network}/hash/{hash}") - public Block getBlockByHash(@PathVariable("network") String network, @PathVariable("hash") String hash) { + public Block getBlockByHash( + @PathVariable("network") String network, + @PathVariable("hash") String hash + ) { return blockService.getBlockByHash(network, hash); } @ApiOperation("get block_info by hash") @GetMapping("/info/{network}/hash/{hash}") - public BlockInfoEntity getBlockInfoByHash(@PathVariable("network") String network, @PathVariable("hash") String hash) { + public BlockInfoEntity getBlockInfoByHash( + @PathVariable("network") String network, + @PathVariable("hash") String hash + ) { return blockInfoService.getBlockInfoByHash(network, hash); } @ApiOperation("get block by height") @GetMapping("/{network}/height/{height}") - public Block getBlockByHeight(@PathVariable("network") String network, @PathVariable("height") long height) { - return blockService.getBlockByHeight(network, height); + public BlockVo getBlockByHeight( + @PathVariable("network") String network, + @PathVariable("height") long height + ) { + BlockVo blockVo = BlockVo.from(blockService.getBlockByHeight(network, height)); + List dagBlocks = dagInspectorService.getBlocksByHeight(network, height); + if (dagBlocks == null || dagBlocks.isEmpty()) { + return blockVo; + } + dagBlocks.stream() + .filter(dagBlock -> dagBlock.getBlockHash().equals(blockVo.getId())) + .findFirst().ifPresent(dagBlock -> { + blockVo.setDaaScore(dagBlock.getDaaScore()); + blockVo.setMergeSetBlueIds(dagBlock.getMergeSetBlueIds()); + blockVo.setHeightGroupIndex(dagBlock.getHeightGroupIndex()); + }); + return blockVo; } @ApiOperation("get block list") @GetMapping("/{network}/page/{page}") - public Result getRangeBlocks(@PathVariable("network") String network, @PathVariable("page") int page, - @RequestParam(value = "count", required = false, defaultValue = "20") int count + public Result getRangeBlocks( + @PathVariable("network") String network, + @PathVariable("page") int page, + @RequestParam(value = "count", required = false, defaultValue = "20") int count ) { return blockService.getRange(network, page, count, 0); } @ApiOperation("get block list by start height") @GetMapping("/{network}/start_height/") - public Result getBlocksStartWith(@PathVariable("network") String network, @RequestParam(value = "start_height", required = false, defaultValue = "0") long start_height, - @RequestParam(value = "page", required = false, defaultValue = "1") int page, - @RequestParam(value = "count", required = false, defaultValue = "20") int count + public Result getBlocksStartWith( + @PathVariable("network") String network, + @RequestParam(value = "start_height", required = false, defaultValue = "0") long start_height, + @RequestParam(value = "page", required = false, defaultValue = "1") int page, + @RequestParam(value = "count", required = false, defaultValue = "20") int count ) { return blockService.getBlocksStartWith(network, start_height, page, count); } @ApiOperation("get uncle block list") @GetMapping("/{network}/uncle/page/{page}") - public Result getRangeUncleBlocks(@PathVariable("network") String network, @PathVariable("page") int page, - @RequestParam(value = "count", required = false, defaultValue = "20") int count, - @RequestParam(value = "total", required = false, defaultValue = "0") int start_height) { + public Result getRangeUncleBlocks( + @PathVariable("network") String network, @PathVariable("page") int page, + @RequestParam(value = "count", required = false, defaultValue = "20") int count, + @RequestParam(value = "total", required = false, defaultValue = "0") int start_height + ) { return blockService.getUnclesRange(network, page, count, start_height); } @ApiOperation("get uncle block by height") @GetMapping("/{network}/uncle/height/{height}") - public UncleBlock getUncleBlockByHeight(@PathVariable("network") String network, @PathVariable("height") long height) { + public UncleBlock getUncleBlockByHeight( + @PathVariable("network") String network, + @PathVariable("height") long height + ) { return blockService.getUncleBlockByHeight(network, height); } @ApiOperation("get uncle block by hash") @GetMapping("/{network}/uncle/hash/{hash}") - public UncleBlock getUncleBlockByHash(@PathVariable("network") String network, @PathVariable("hash") String hash) { + public UncleBlock getUncleBlockByHash( + @PathVariable("network") String network, + @PathVariable("hash") String hash + ) { return blockService.getUncleBlockByHash(network, hash); } } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/BlockVo.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/BlockVo.java new file mode 100644 index 0000000..bb8cd44 --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/BlockVo.java @@ -0,0 +1,57 @@ +package org.starcoin.scan.service.vo; + +import com.alibaba.fastjson.annotation.JSONField; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.starcoin.bean.*; + +import java.util.ArrayList; +import java.util.List; + +public class BlockVo extends Block { + private Long daaScore; + + private Integer heightGroupIndex; + + List mergeSetBlueIds; + + public Long getDaaScore() { + return daaScore; + } + + public void setDaaScore(Long daaScore) { + this.daaScore = daaScore; + } + + public Integer getHeightGroupIndex() { + return heightGroupIndex; + } + + public void setHeightGroupIndex(Integer heightGroupIndex) { + this.heightGroupIndex = heightGroupIndex; + } + + public List getMergeSetBlueIds() { + return mergeSetBlueIds; + } + + public void setMergeSetBlueIds(List mergeSetBlueIds) { + this.mergeSetBlueIds = mergeSetBlueIds; + } + + public static BlockVo from(Block block) { + BlockVo blockVo = new BlockVo(); + + blockVo.setBlockMetadata(block.getBlockMetadata()); + blockVo.setBody(block.getBody()); + blockVo.setUncles(block.getUncles()); + blockVo.setHeader(block.getHeader()); + blockVo.setRaw(block.getRaw()); + blockVo.setBody(block.getBody()); + + blockVo.setDaaScore(0L); + blockVo.setHeightGroupIndex(0); + blockVo.setMergeSetBlueIds(new ArrayList<>()); + + return blockVo; + } +} From 5e40cd56ce0b80e1881ce614b68fcb209fdaeeda Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Tue, 25 Jun 2024 16:17:09 +0800 Subject: [PATCH 40/82] [dag inspector] Remove useless code --- .../org/starcoin/scan/service/BlockService.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java index a68e487..2ae0fd4 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java @@ -216,19 +216,4 @@ public Result getUnclesRange(String network, int page, int count, in } return ServiceUtils.getSearchResult(searchResponse, UncleBlock.class); } - -// public List getBlocksBetweenHeights(String network, Integer startHeight, Integer endHeight) { -// SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.BLOCK_CONTENT_INDEX)); -// RangeQueryBuilder termQueryBuilder = QueryBuilders.rangeQuery("header.number").gte(startHeight).lte(endHeight); -// searchRequest.source(new SearchSourceBuilder().query(termQueryBuilder).sort("header.number", SortOrder.ASC)); -// SearchResponse searchResponse; -// try { -// searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); -// } catch (IOException e) { -// logger.error("getBlocksBetweenHeights failed, startHeight: {}, endHeight: {}", startHeight, endHeight, e); -// return null; -// } -// Result result = ServiceUtils.getSearchResult(searchResponse, Block.class); -// return result.getContents(); -// } } From c66a0fb6d0311738c75825b510f442e576f48d04 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Tue, 25 Jun 2024 16:22:46 +0800 Subject: [PATCH 41/82] [dag inspector] set in vritual selected parent chain --- .../starcoin/indexer/handler/DagInspectorIndexerHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index 58ebbee..b558e6f 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -97,7 +97,7 @@ public void upsertDagInfoFromBlocks(List blockList) throws IOException, J dagBlock.setSelectedParentHash(ghostdagData.getSelectedParent()); dagBlock.setParentIds(blockInfo.getHeader().getParentsHash()); // Block is the virtual selected parent chain because the list read from block height - dagBlock.setInVirtualSelectedParentChain(ghostdagData.getMergesetBlues().contains(currentBlockHash)); + dagBlock.setInVirtualSelectedParentChain(true); Integer groupSize = getHeightGroupSizeOrDefault(heightGroupList, dagBlock.getHeight(), 0); dagBlock.setHeightGroupIndex(groupSize); From 9533817364c34813e3c4a8502af994e9c01eabfa Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Tue, 25 Jun 2024 17:30:07 +0800 Subject: [PATCH 42/82] [dag inspector] Fixed the error of block field name mismatch --- docker-compose/README.md | 10 ++++++ .../handler/DagInspectorIndexerHandler.java | 7 +++-- .../scan/controller/BlockV2Controller.java | 10 ++++-- .../org/starcoin/scan/service/vo/BlockVo.java | 31 ++++++++++++------- 4 files changed, 40 insertions(+), 18 deletions(-) create mode 100644 docker-compose/README.md diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 0000000..2d50be2 --- /dev/null +++ b/docker-compose/README.md @@ -0,0 +1,10 @@ +# 如何构建本地环境 + +## 1. 安装 Docker +## 2. 安装 Docker Compose +## 3. 运行以下命令 +```bash +docker-compose up +``` +## 4. 访问 http://localhost:8080 +``` diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index b558e6f..58dfda4 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -147,7 +147,7 @@ List buildEdgeDataFromDagBlockDataMaybeUpate( DagInspectorBlock parentDagBlock = dagBlockMap.get(parentHash); if (parentDagBlock == null) { logger.info("Parent block not found: {} ", parentHash); - parentDagBlock = getDagInspectorBlockInfoFromHash(parentHash, heightGroupList); + parentDagBlock = getDagInspectorBlockInfoFromHash(parentHash, heightGroupList, false); // Put into buffer list newDagBlocks.add(parentDagBlock); @@ -324,7 +324,8 @@ private List getGroupHeightSizeFromStorage(List h protected DagInspectorBlock getDagInspectorBlockInfoFromHash( String blockHash, - List heightGroupList + List heightGroupList, + boolean isSelectedParentChain ) throws JSONRPC2SessionException { Block blockInfo = blockRPCClient.getBlockByHash(blockHash); @@ -340,7 +341,7 @@ protected DagInspectorBlock getDagInspectorBlockInfoFromHash( dagBlock.setHeight(blockInfo.getHeader().getHeight()); dagBlock.setSelectedParentHash(blockGhostdagData.getSelectedParent()); dagBlock.setParentIds(blockInfo.getHeader().getParentsHash()); - dagBlock.setInVirtualSelectedParentChain(false); + dagBlock.setInVirtualSelectedParentChain(isSelectedParentChain); // Height group list index Integer groupSize = getHeightGroupSizeOrDefault(heightGroupList, blockHeight, 0); diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java index 722c335..8f40a1d 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java @@ -62,7 +62,11 @@ public BlockVo getBlockByHeight( @PathVariable("network") String network, @PathVariable("height") long height ) { - BlockVo blockVo = BlockVo.from(blockService.getBlockByHeight(network, height)); + Block block = blockService.getBlockByHeight(network, height); + if (block == null) { + return new BlockVo(); + } + BlockVo blockVo = BlockVo.from(block); List dagBlocks = dagInspectorService.getBlocksByHeight(network, height); if (dagBlocks == null || dagBlocks.isEmpty()) { return blockVo; @@ -71,8 +75,8 @@ public BlockVo getBlockByHeight( .filter(dagBlock -> dagBlock.getBlockHash().equals(blockVo.getId())) .findFirst().ifPresent(dagBlock -> { blockVo.setDaaScore(dagBlock.getDaaScore()); - blockVo.setMergeSetBlueIds(dagBlock.getMergeSetBlueIds()); - blockVo.setHeightGroupIndex(dagBlock.getHeightGroupIndex()); + blockVo.setMergedBlueset(dagBlock.getMergeSetBlueIds()); + blockVo.setHeightgroupIndex(dagBlock.getHeightGroupIndex()); }); return blockVo; } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/BlockVo.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/BlockVo.java index bb8cd44..d0a9db0 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/BlockVo.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/BlockVo.java @@ -8,11 +8,18 @@ import java.util.List; public class BlockVo extends Block { + + @JSONField(name = "daa_score") + @JsonProperty("daa_score") private Long daaScore; - private Integer heightGroupIndex; + @JSONField(name = "heightgroup_index") + @JsonProperty("heightgroup_index") + private Integer heightgroupIndex; - List mergeSetBlueIds; + @JSONField(name = "merged_blueset") + @JsonProperty("merged_blueset") + List mergedBlueset; public Long getDaaScore() { return daaScore; @@ -22,20 +29,20 @@ public void setDaaScore(Long daaScore) { this.daaScore = daaScore; } - public Integer getHeightGroupIndex() { - return heightGroupIndex; + public Integer getHeightgroupIndex() { + return heightgroupIndex; } - public void setHeightGroupIndex(Integer heightGroupIndex) { - this.heightGroupIndex = heightGroupIndex; + public void setHeightgroupIndex(Integer heightgroupIndex) { + this.heightgroupIndex = heightgroupIndex; } - public List getMergeSetBlueIds() { - return mergeSetBlueIds; + public List getMergedBlueset() { + return mergedBlueset; } - public void setMergeSetBlueIds(List mergeSetBlueIds) { - this.mergeSetBlueIds = mergeSetBlueIds; + public void setMergedBlueset(List mergedBlueset) { + this.mergedBlueset = mergedBlueset; } public static BlockVo from(Block block) { @@ -49,8 +56,8 @@ public static BlockVo from(Block block) { blockVo.setBody(block.getBody()); blockVo.setDaaScore(0L); - blockVo.setHeightGroupIndex(0); - blockVo.setMergeSetBlueIds(new ArrayList<>()); + blockVo.setHeightgroupIndex(0); + blockVo.setMergedBlueset(new ArrayList<>()); return blockVo; } From 0cc2200094b9e6fd2e258ec70aa8f4b59a1ea736 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Tue, 25 Jun 2024 17:52:13 +0800 Subject: [PATCH 43/82] [dag inspector] Add README.md for local environment --- docker-compose/README.md | 57 +++++++++++++++++++++++++++--- docker-compose/docker-compose.yaml | 12 +++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/docker-compose/README.md b/docker-compose/README.md index 2d50be2..8fc417e 100644 --- a/docker-compose/README.md +++ b/docker-compose/README.md @@ -1,10 +1,57 @@ -# 如何构建本地环境 +# How to build local environment -## 1. 安装 Docker -## 2. 安装 Docker Compose -## 3. 运行以下命令 +## 1. Install "Docker" and "Docker Compose" +## 2. Run the command to start the database and components ```bash docker-compose up ``` -## 4. 访问 http://localhost:8080 +## 3. Start starcoin-index project + +### Config the startup environment variable +```dotenv +HOSTS=localhost +NETWORK=halley # select which network to scan +BG_TASK_JOBS=dag_inspector +TXN_OFFSET=0 +BULK_SIZE=100 +STARCOIN_ES_PWD= +STARCOIN_ES_URL=localhost +STARCOIN_ES_PROTOCOL=http +STARCOIN_ES_PORT=9200 +STARCOIN_ES_USER= +SWAP_API_URL=https://swap-api.starswap.xyz +SWAP_CONTRACT_ADDR=0x8c109349c6bd91411d6bc962e080c4a3 +DS_URL=jdbc:postgresql://localhost/starcoin +DB_SCHEMA=halley +DB_USER_NAME=starcoin +DB_PWD=starcoin +PROGRAM_ARGS= +# auto_repair 9411700 +``` + +### Configuration Elasticsearch template +[IMPORTANT!!] Make sure your template has added to Elastic search service before add data, including component template and index template to ES. +Following file: [[es_pipeline.scripts](..%2Fkube%2Fmappings%2Fes_pipeline.scripts)] + +1. Open the 'Kibana' site has been started in the docker-compose environment, usually the url is http://localhost:5601 +2. Navigate to 'Dev Tools' +3. Follow the instructions in the file of giving above to add the template to ES + +## 4. Start starcoin-scan-api project + +### Config the startup enviroment variable +```dotenv +STARCOIN_ES_URL=localhost +STARCOIN_ES_PROTOCOL=http +STARCOIN_ES_PORT=9200 +STARCOIN_ES_USER= +STARCOIN_ES_INDEX_VERSION= +STARCOIN_ES_PWD= +MAIN_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=main +BARNARD_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=barnard +HALLEY_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=halley +DS_URL=jdbc:postgresql://localhost/starcoin +STARCOIN_USER_DS_URL="jdbc:postgresql://localhost/starcoin?currentSchema=starcoin_user" +DB_USER_NAME=starcoin +DB_PWD=starcoin ``` diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 1670730..160a848 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -50,6 +50,18 @@ services: depends_on: - elasticsearch + postgresql: + image: postgres:13.2 + container_name: postgres_db + environment: + POSTGRES_USER: starcoin + POSTGRES_PASSWORD: starcoin + POSTGRES_DB: starcoin + volumes: + - db_data:/var/lib/postgresql/data + ports: + - 5432:5432 + volumes: esdata: driver: local From 5f94f5e216f7438b2ee24e11b83bec625b4dc89b Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Tue, 25 Jun 2024 22:28:21 +0800 Subject: [PATCH 44/82] use block replace blockInfo for easy understanding, because we have class BlockInfo --- .../handler/DagInspectorIndexerHandler.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index 58dfda4..40a5596 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -82,20 +82,20 @@ public void upsertDagInfoFromBlocks(List blockList) throws IOException, J ); // Build block node data - blockList.forEach(blockInfo -> { - String currentBlockHash = blockInfo.getHeader().getBlockHash(); + blockList.forEach(block -> { + String currentBlockHash = block.getHeader().getBlockHash(); DagInspectorBlock dagBlock = dagBlockMap.get(currentBlockHash); if (dagBlock == null) { try { BlockGhostdagData ghostdagData = blockRPCClient.getBlockGhostdagData(currentBlockHash); dagBlock = new DagInspectorBlock(); dagBlock.setBlockHash(currentBlockHash); - dagBlock.setTimestamp(blockInfo.getHeader().getTimestamp()); + dagBlock.setTimestamp(block.getHeader().getTimestamp()); dagBlock.setColor(NODE_COLOR_GRAY); dagBlock.setDaaScore(ghostdagData.getBlueScore()); - dagBlock.setHeight(blockInfo.getHeader().getHeight()); + dagBlock.setHeight(block.getHeader().getHeight()); dagBlock.setSelectedParentHash(ghostdagData.getSelectedParent()); - dagBlock.setParentIds(blockInfo.getHeader().getParentsHash()); + dagBlock.setParentIds(block.getHeader().getParentsHash()); // Block is the virtual selected parent chain because the list read from block height dagBlock.setInVirtualSelectedParentChain(true); @@ -147,7 +147,7 @@ List buildEdgeDataFromDagBlockDataMaybeUpate( DagInspectorBlock parentDagBlock = dagBlockMap.get(parentHash); if (parentDagBlock == null) { logger.info("Parent block not found: {} ", parentHash); - parentDagBlock = getDagInspectorBlockInfoFromHash(parentHash, heightGroupList, false); + parentDagBlock = getDagInspectorBlockFromHash(parentHash, heightGroupList, false); // Put into buffer list newDagBlocks.add(parentDagBlock); @@ -322,25 +322,25 @@ private List getGroupHeightSizeFromStorage(List h } - protected DagInspectorBlock getDagInspectorBlockInfoFromHash( + protected DagInspectorBlock getDagInspectorBlockFromHash( String blockHash, List heightGroupList, boolean isSelectedParentChain ) throws JSONRPC2SessionException { - Block blockInfo = blockRPCClient.getBlockByHash(blockHash); + Block block = blockRPCClient.getBlockByHash(blockHash); BlockGhostdagData blockGhostdagData = blockRPCClient.getBlockGhostdagData(blockHash); DagInspectorBlock dagBlock = new DagInspectorBlock(); - Long blockHeight = blockInfo.getHeader().getHeight(); + Long blockHeight = block.getHeader().getHeight(); dagBlock.setBlockHash(blockHash); - dagBlock.setTimestamp(blockInfo.getHeader().getTimestamp()); + dagBlock.setTimestamp(block.getHeader().getTimestamp()); dagBlock.setColor(NODE_COLOR_GRAY); dagBlock.setDaaScore(blockGhostdagData.getBlueScore()); - dagBlock.setHeight(blockInfo.getHeader().getHeight()); + dagBlock.setHeight(block.getHeader().getHeight()); dagBlock.setSelectedParentHash(blockGhostdagData.getSelectedParent()); - dagBlock.setParentIds(blockInfo.getHeader().getParentsHash()); + dagBlock.setParentIds(block.getHeader().getParentsHash()); dagBlock.setInVirtualSelectedParentChain(isSelectedParentChain); // Height group list index @@ -350,7 +350,7 @@ protected DagInspectorBlock getDagInspectorBlockInfoFromHash( logger.info("Get block info from hash: {}, height: {}, heightGroupIndex: {}", blockHash, - blockInfo.getHeader().getHeight(), + block.getHeader().getHeight(), dagBlock.getHeightGroupIndex() ); return dagBlock; From b1fcd278255ba1e2ffce7f498f688ba66a2723e8 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Tue, 25 Jun 2024 22:38:49 +0800 Subject: [PATCH 45/82] use Math.max replace if expression --- .../scan/service/DagInspectorService.java | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index b64fd28..9ea92bd 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -73,10 +73,7 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash( throw new RuntimeException("Cannot find block by block hash"); } Long endHeight = block.getHeight(); - long startHeight = block.getHeight() - heightDifference; - if (startHeight < 0L) { - startHeight = 0L; - } + long startHeight = Math.max(block.getHeight() - heightDifference, 0L); return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } @@ -90,19 +87,13 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore( throw new RuntimeException("Cannot find block by block hash"); } Long endHeight = block.getHeight(); - long startHeight = block.getHeight() - heightDifference; - if (startHeight < 0L) { - startHeight = 0L; - } + long startHeight = Math.max(block.getHeight() - heightDifference, 0L); return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDifference) throws IOException { long endHeight = getMaxHeightFromStorage(network); - long startHeight = endHeight - heightDifference; - if (startHeight < 0L) { - startHeight = 0L; - } + long startHeight = Math.max(endHeight - heightDifference, 0L); return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } From ce6c784eb9830b40766d58504848186f6464a6e6 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Wed, 26 Jun 2024 00:48:30 +0800 Subject: [PATCH 46/82] add fetchParentsBlock --- .../indexer/handler/DagInspectorIndexer.java | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java index f4aacc3..2b79ec7 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java @@ -146,7 +146,10 @@ public void fetchAndProcessBlockSequel() { long headHeight = chainHeader.getHeight(); long bulkNumber = Math.min(headHeight - localBlockOffset.getBlockHeight(), bulkSize); int index = 1; - Map blockMap = new HashMap<>(); + List blockList = new ArrayList<>(); + long minHeight = localBlockOffset.getBlockHeight(); + Set visit = new HashSet<>(); + Deque deque = new ArrayDeque<>(); while (index <= bulkNumber) { long currentBlockHeight = localBlockOffset.getBlockHeight() + index; @@ -156,15 +159,22 @@ public void fetchAndProcessBlockSequel() { logger.warn("get block null: {}", currentBlockHeight); return; } - - blockMap.put(block.getHeader().getBlockHash(), block); - + visit.add(block.getHeader().getBlockHash()); + fetchParentsBlock(block, visit, deque, blockList, minHeight); + while (!deque.isEmpty()) { + int size = deque.size(); + for (int i = 0; i < size; i++) { + Block block_parent = deque.removeFirst(); + fetchParentsBlock(block_parent, visit, deque, blockList, minHeight); + } + } + blockList.add(block); //update current header currentHandleHeader = block.getHeader(); index++; logger.info("add block: {}", block.getHeader()); } - inspectorHandler.upsertDagInfoFromBlocks(new ArrayList<>(blockMap.values())); + inspectorHandler.upsertDagInfoFromBlocks(blockList); // Update offset localBlockOffset.setBlockHeight(currentHandleHeader.getHeight()); @@ -176,5 +186,19 @@ public void fetchAndProcessBlockSequel() { } } -} - + void fetchParentsBlock(Block block, Set visit, Deque deque, List blockList, long minHeight) throws JSONRPC2SessionException { + for (String parent : block.getBlockMetadata().getParentsHash()) { + if (!visit.contains(parent)) { + visit.add(parent); + Block block_parent = elasticSearchHandler.getBlockContent(parent); + if (block_parent == null) { + block_parent = blockRPCClient.getBlockByHash(parent); + if (block_parent.getHeader().getHeight() >= minHeight) { + deque.addLast(block_parent); + blockList.add(block_parent); + } + } + } + } + } +} \ No newline at end of file From f1a00eed0d69299ccd5d4e7982f4981f3a57dc09 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Wed, 26 Jun 2024 17:14:42 +0800 Subject: [PATCH 47/82] [dag inspector] rename file --- ..._height_group.json => dag_inspector_height_group.mapping.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename kube/mappings/{dag_inspector_height_group.json => dag_inspector_height_group.mapping.json} (100%) diff --git a/kube/mappings/dag_inspector_height_group.json b/kube/mappings/dag_inspector_height_group.mapping.json similarity index 100% rename from kube/mappings/dag_inspector_height_group.json rename to kube/mappings/dag_inspector_height_group.mapping.json From 94e4f3efa39ea3cf0a982ca26301eec769f535d9 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Wed, 26 Jun 2024 17:29:37 +0800 Subject: [PATCH 48/82] [dag inspector] fixed the template --- docker-compose/README.md | 6 ++++++ kube/mappings/es_pipeline.scripts | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker-compose/README.md b/docker-compose/README.md index 8fc417e..527d1b5 100644 --- a/docker-compose/README.md +++ b/docker-compose/README.md @@ -37,6 +37,9 @@ Following file: [[es_pipeline.scripts](..%2Fkube%2Fmappings%2Fes_pipeline.script 2. Navigate to 'Dev Tools' 3. Follow the instructions in the file of giving above to add the template to ES +### Add SQL tables for network +[IMPORTANT!!] Add the [tables](../starcoin-indexer/deploy/create_table.sql) for the network you want to scan, including main, barnard, halley, etc. + ## 4. Start starcoin-scan-api project ### Config the startup enviroment variable @@ -55,3 +58,6 @@ STARCOIN_USER_DS_URL="jdbc:postgresql://localhost/starcoin?currentSchema=starcoi DB_USER_NAME=starcoin DB_PWD=starcoin ``` + +### Add SQL tables for network +[IMPORTANT!!] Add the [tables](../starcoin-scan-api/deploy/create_table.sql) for the network you want to scan, including main, barnard, halley, etc. diff --git a/kube/mappings/es_pipeline.scripts b/kube/mappings/es_pipeline.scripts index b304611..fdae8cc 100644 --- a/kube/mappings/es_pipeline.scripts +++ b/kube/mappings/es_pipeline.scripts @@ -122,10 +122,10 @@ PUT _component_template/block-mappings } } - PUT _component_template/dag_inspector_edge-mappings + PUT _component_template/dag_inspector_height_group-mappings { "template": { - # copy dag_inspector_edge.mapping.json to here. + # copy dag_inspector_height_group.mapping.json to here. } } 4. From 62b1f1d769cb05bf8ce44e16074558434dea4c53 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Wed, 26 Jun 2024 18:33:53 +0800 Subject: [PATCH 49/82] [dag block scan-api] fixed if getBlocksByHeight throw exception --- .../java/org/starcoin/scan/service/DagInspectorService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index 9ea92bd..e01e6e4 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -1,6 +1,7 @@ package org.starcoin.scan.service; import com.alibaba.fastjson.JSONObject; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -105,7 +106,7 @@ public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDiff * @param endHeight * @return */ - private List getBlockList(String network, Long startHeight, Long endHeight) throws IOException { + private List getBlockList(String network, Long startHeight, Long endHeight) throws ElasticsearchException, IOException { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery("height").gte(startHeight).lte(endHeight))); @@ -251,7 +252,7 @@ public List getBlocksByHeight(String network, Long height) { SearchResponse searchResponse; try { searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - } catch (IOException e) { + } catch (IOException | org.elasticsearch.ElasticsearchStatusException e) { logger.error("getBlocksByHeight failed, blockHash: {}", height, e); return null; } From 15fdd63d1a607e9809ee6bbcec2b3d6d81a5814a Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Wed, 26 Jun 2024 19:09:44 +0800 Subject: [PATCH 50/82] [dag block indexer] fixed getting parents error for DFS algorithm --- .../indexer/handler/DagInspectorIndexer.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java index 2b79ec7..4b3cc17 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java @@ -81,8 +81,8 @@ protected void executeInternal(JobExecutionContext jobExecutionContext) { return; } - // fetchAndProcessBlockSequel(); - fetchAndProcessBlocksParallel(); + fetchAndProcessBlockSequel(); + // fetchAndProcessBlocksParallel(); } public void fetchAndProcessBlocksParallel() { @@ -186,8 +186,14 @@ public void fetchAndProcessBlockSequel() { } } - void fetchParentsBlock(Block block, Set visit, Deque deque, List blockList, long minHeight) throws JSONRPC2SessionException { - for (String parent : block.getBlockMetadata().getParentsHash()) { + void fetchParentsBlock( + Block block, + Set visit, + Deque deque, + List blockList, + long minHeight + ) throws JSONRPC2SessionException { + for (String parent : block.getHeader().getParentsHash()) { if (!visit.contains(parent)) { visit.add(parent); Block block_parent = elasticSearchHandler.getBlockContent(parent); From 9ed9e7c95bc1801e0ea1224702866b122f2c859f Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Thu, 27 Jun 2024 17:41:15 +0800 Subject: [PATCH 51/82] [bug fix] Solved the problem that when the indexer parses txn info without transaction, it reports an error and exits, resulting in the block not being added to ES storage --- pom.xml | 2 +- starcoin-indexer/local.env | 4 +- .../starcoin/indexer/config/QuartzConfig.java | 1 + .../indexer/handler/IndexerHandle.java | 5 ++- .../indexer/handler/RepairHandle.java | 9 ++-- .../indexer/handler/ServiceUtils.java | 16 +++++-- .../indexer/test/IndexerLogicBaseTest.java | 3 -- .../indexer/utils/ServiceUtilsTest.java | 45 +++++++++++++++++++ .../application-integrationtest.properties | 35 ++++++++------- .../src/test/resources/unit-test.env | 18 -------- starcoin-scan-api/local.env | 13 ++++++ 11 files changed, 103 insertions(+), 48 deletions(-) create mode 100644 starcoin-indexer/src/test/java/org/starcoin/indexer/utils/ServiceUtilsTest.java delete mode 100644 starcoin-indexer/src/test/resources/unit-test.env create mode 100644 starcoin-scan-api/local.env diff --git a/pom.xml b/pom.xml index b25795b..6a37694 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.starcoin sdk - 1.2.6 + 1.2.7 org.springframework.boot diff --git a/starcoin-indexer/local.env b/starcoin-indexer/local.env index a5c8382..c45dc4d 100644 --- a/starcoin-indexer/local.env +++ b/starcoin-indexer/local.env @@ -13,7 +13,7 @@ SWAP_API_URL=https://swap-api.starswap.xyz SWAP_CONTRACT_ADDR=0x8c109349c6bd91411d6bc962e080c4a3 DS_URL=jdbc:postgresql://localhost/starcoin DB_SCHEMA=halley -DB_USER_NAME=kaspa -DB_PWD=kaspa +DB_USER_NAME=starcoin +DB_PWD=starcoin PROGRAM_ARGS= # auto_repair 9411700 \ No newline at end of file diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java index 1c10870..8ca2b3d 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java @@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; import org.springframework.scheduling.quartz.SchedulerFactoryBean; import org.starcoin.config.SearchJobFactory; import org.starcoin.indexer.handler.*; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java index fc9cb96..b9a6f5b 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java @@ -151,8 +151,11 @@ protected void executeInternal(JobExecutionContext jobExecutionContext) { logger.info("rollback handle ok: {}", localBlockOffset); return; //退出当前任务，重新添加从分叉点之后的block } + //set event - ServiceUtils.addBlockToList(transactionRPCClient, blockList, block); + ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block); + blockList.add(block); + //update current header currentHandleHeader = block.getHeader(); index++; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java index 8adff82..fdf466e 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java @@ -100,7 +100,8 @@ public boolean autoRepair(long startNumber, int count) { if (esBlock == null) { logger.warn("es block not exist: {}", block.getHeader().getHeight()); try { - ServiceUtils.addBlockToList(transactionRPCClient, blockList, block); + ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block); + blockList.add(block); } catch (JSONRPC2SessionException e) { logger.error("add block err:", e); } @@ -109,7 +110,8 @@ public boolean autoRepair(long startNumber, int count) { if (!block.getHeader().getBlockHash().equals(esBlock.getHeader().getBlockHash())) { // fork block try { - ServiceUtils.addBlockToList(transactionRPCClient, blockList, block); + ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block); + blockList.add(block); } catch (JSONRPC2SessionException e) { logger.error("add fix block err:", e); } @@ -141,7 +143,8 @@ public void repair(long blockNumber) { if (!blockOnChain.getHeader().getBlockHash().equals(blockOnEs.getHeader().getBlockHash())) { // update block List blockList = new ArrayList<>(); - ServiceUtils.addBlockToList(transactionRPCClient, blockList, blockOnChain); + ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, blockOnChain); + blockList.add(blockOnChain); elasticSearchHandler.updateBlock(blockList); logger.info("repair ok: {}", blockNumber); } else { diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java index 3df24e6..d355971 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java @@ -220,11 +220,19 @@ public static Map getTokenReserveFromState(StateRPCClient stat return poolReserves; } - static void addBlockToList(TransactionRPCClient transactionRPCClient, List blockList, Block block) throws JSONRPC2SessionException { - List transactionList = transactionRPCClient.getBlockTransactions(block.getHeader().getBlockHash()); - if (transactionList == null) { + public static void fetchTransactionsForBlock(TransactionRPCClient transactionRPCClient, Block block) throws JSONRPC2SessionException { + List transactionList; + try { + transactionList = transactionRPCClient.getBlockTransactions(block.getHeader().getBlockHash()); + } catch (JSONRPC2SessionException e) { + logger.error("get block txn error:", e); return; } + + if (transactionList == null || transactionList.isEmpty()) { + return; + } + for (Transaction transaction : transactionList) { BlockMetadata metadata; Transaction userTransaction = transactionRPCClient.getTransactionByHash(transaction.getTransactionHash()); @@ -268,6 +276,6 @@ static void addBlockToList(TransactionRPCClient transactionRPCClient, List blockHashes = Arrays.asList( + "0x6590d6769e9837e5fbc116daf53efa809f996e16cf55d4a713695094ac79aada", + "0x6b7ca9e9a1aa3f414caab285f214179108d5cf3fc2864efd9e7b5ed4461e7e80", + "0x042218a4750e696942fab2a875479a77f7c3c48049acb400a80c77b24aa06d36", + "0x721c15b57a22687dd90a439d85918f92f49937eef48c7c7afb72a7da43845589", + "0x9a68fd546aec8fdad955c0626cb9b20be121f932bb69f8a523f029248b125f51", + "0x73c0d5b604e130dedf0ecb77102e91de521bf38097b80d3e64c7ed58bc85c232", + "0x9161c9b51a7817cc2a18c190c99b40e96cb7cf4666ad63a7ffa33b2671c3aa31", + "0x020aadcabfbd57cc4c9059d16a91f8bc944b6c5c110d4fc72e6c346658035ecc" + ); + List blocks = blockHashes.stream().map(ExceptionWrap.wrap(blockHash -> { + Block block = new Block(); + BlockHeader header = new BlockHeader(); + header.setBlockHash(blockHash); + block.setHeader(header); + ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block); + System.out.println("Block: " + blockHash); + return block; + })).collect(Collectors.toList()); + blocks.forEach(System.out::println); + + } +} diff --git a/starcoin-indexer/src/test/resources/application-integrationtest.properties b/starcoin-indexer/src/test/resources/application-integrationtest.properties index 5a0e419..cdbe740 100644 --- a/starcoin-indexer/src/test/resources/application-integrationtest.properties +++ b/starcoin-indexer/src/test/resources/application-integrationtest.properties @@ -1,24 +1,27 @@ # server.port=8300 -starcoin.seeds=${HOSTS} -starcoin.network=${NETWORK} -starcoin.bg_task.jobs=${BG_TASK_JOBS} -starcoin.indexer.bulk_size=${BULK_SIZE} -starcoin.indexer.txn_offset=${TXN_OFFSET} +starcoin.seeds=vega.seed.starcoin.org +starcoin.network=vega +starcoin.bg_task.jobs= +starcoin.indexer.bulk_size=100 +starcoin.indexer.txn_offset=0 jasypt.encryptor.bean=encryptorBean -jasypt.encryptor.password=${STARCOIN_ES_PWD} -elasticsearch.host=${STARCOIN_ES_URL} -elasticsearch.protocol=${STARCOIN_ES_PROTOCOL} -elasticsearch.port=${STARCOIN_ES_PORT} -elasticsearch.username=${STARCOIN_ES_USER} +jasypt.encryptor.password= +elasticsearch.host=localhost +elasticsearch.protocol=http +elasticsearch.port=9200 +elasticsearch.username= elasticsearch.connTimeout=10000 elasticsearch.socketTimeout=10000 elasticsearch.connectionRequestTimeout=2000 -swap.contract.address=${SWAP_CONTRACT_ADDR} -starcoin.swap.api.url=${SWAP_API_URL} +swap.contract.address=0x8c109349c6bd91411d6bc962e080c4a3 +starcoin.swap.api.url=https://swap-api.starswap.xyz logging.file.name=logs/indexer.log + +## Unittest config +spring.profiles=unittest spring.datasource.hikari.minimum-idle=1 spring.datasource.hikari.maximum-pool-size=2 -spring.datasource.url=${DS_URL} -spring.datasource.username=${DB_USER_NAME} -spring.datasource.password=${DB_PWD} -spring.jpa.properties.hibernate.default_schema=${DB_SCHEMA} \ No newline at end of file +spring.datasource.url=jdbc:postgresql://localhost/starcoin +spring.datasource.username=starcoin +spring.datasource.password=starcoin +spring.jpa.properties.hibernate.default_schema=vega \ No newline at end of file diff --git a/starcoin-indexer/src/test/resources/unit-test.env b/starcoin-indexer/src/test/resources/unit-test.env deleted file mode 100644 index 53e1d2c..0000000 --- a/starcoin-indexer/src/test/resources/unit-test.env +++ /dev/null @@ -1,18 +0,0 @@ -HOSTS=halley.seed.starcoin.org -NETWORK=halley -BG_TASK_JOBS=dag_inspector -TXN_OFFSET=0 -BULK_SIZE=100 -STARCOIN_ES_PWD= -STARCOIN_ES_URL=localhost -STARCOIN_ES_PROTOCOL=http -STARCOIN_ES_PORT=9200 -STARCOIN_ES_USER= -SWAP_API_URL=https://swap-api.starswap.xyz -SWAP_CONTRACT_ADDR=0x8c109349c6bd91411d6bc962e080c4a3 -DS_URL=jdbc:postgresql://localhost/starcoin -DB_SCHEMA=halley -DB_USER_NAME=kaspa -DB_PWD=kaspa -PROGRAM_ARGS= -# auto_repair 9411700 \ No newline at end of file diff --git a/starcoin-scan-api/local.env b/starcoin-scan-api/local.env new file mode 100644 index 0000000..ac373c2 --- /dev/null +++ b/starcoin-scan-api/local.env @@ -0,0 +1,13 @@ +STARCOIN_ES_URL=localhost +STARCOIN_ES_PROTOCOL=http +STARCOIN_ES_PORT=9200 +STARCOIN_ES_USER= +STARCOIN_ES_INDEX_VERSION= +STARCOIN_ES_PWD= +MAIN_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=main +BARNARD_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=barnard +HALLEY_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=halley +DS_URL=jdbc:postgresql://localhost/starcoin +STARCOIN_USER_DS_URL="jdbc:postgresql://localhost/starcoin?currentSchema=starcoin_user" +DB_USER_NAME=starcoin +DB_PWD=starcoin \ No newline at end of file From 99f28a8a537b7634574a8b603ee58d367302cf2a Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Thu, 27 Jun 2024 18:37:41 +0800 Subject: [PATCH 52/82] [bug fix] upgrade starcoin-java sdk for debug --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6a37694..8201b60 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.starcoin sdk - 1.2.7 + 1.2.8 org.springframework.boot From 102ffaaeeb52708e70120206a8e4cbfdc0b15e8d Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Thu, 27 Jun 2024 20:34:26 +0800 Subject: [PATCH 53/82] [debug] add some configs --- .../starcoin-indexer-vega-deployment.yaml | 4 +- .../resources/application-unittest.properties | 72 +++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 starcoin-scan-api/src/test/resources/application-unittest.properties diff --git a/kube/indexer/starcoin-indexer-vega-deployment.yaml b/kube/indexer/starcoin-indexer-vega-deployment.yaml index a130153..6077b5c 100644 --- a/kube/indexer/starcoin-indexer-vega-deployment.yaml +++ b/kube/indexer/starcoin-indexer-vega-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: starcoin-indexer - image: starcoin/starcoin_indexer:halley_fix_db_2 + image: starcoin/starcoin_indexer:dag_inspector_vega_5 ports: - containerPort: 8300 env: @@ -26,7 +26,7 @@ spec: - name: NETWORK value: "vega.0727" - name: BG_TASK_JOBS - value: "indexer,secondary,market_cap,txn_payload,swap_stats,txn_global_idx_update" + value: "indexer,secondary,market_cap,txn_payload,swap_stats,txn_global_idx_update,dag_inspector" - name: BULK_SIZE value: "80" - name: STARCOIN_ES_PWD diff --git a/starcoin-scan-api/src/test/resources/application-unittest.properties b/starcoin-scan-api/src/test/resources/application-unittest.properties new file mode 100644 index 0000000..c03bbf4 --- /dev/null +++ b/starcoin-scan-api/src/test/resources/application-unittest.properties @@ -0,0 +1,72 @@ +# server.port=8500 +spring.application.name=stccan +indexer.version=${STARCOIN_ES_INDEX_VERSION} +jasypt.encryptor.bean=encryptorBean +jasypt.encryptor.password=${STARCOIN_ES_PWD} +elasticsearch.host=${STARCOIN_ES_URL} +elasticsearch.protocol=${STARCOIN_ES_PROTOCOL} +elasticsearch.port=${STARCOIN_ES_PORT} +elasticsearch.username=${STARCOIN_ES_USER} +elasticsearch.connTimeout=10000 +elasticsearch.socketTimeout=10000 +elasticsearch.connectionRequestTimeout=2000 +logging.file.name=logs/scan.log +logging.level.org.springframework.web.filter.CommonsRequestLoggingFilter=DEBUG + +#zuul config +zuul.ratelimit.enabled = true +zuul.ratelimit.repository= BUCKET4J_HAZELCAST +zuul.ratelimit.response-headers=standard +zuul.ratelimit.deny-request.response-status-code=429 +zuul.ratelimit.default-policy-list[0].limit=5 +zuul.ratelimit.default-policy-list[0].quota=1000 +zuul.ratelimit.default-policy-list[0].refresh-interval=60 +zuul.ratelimit.default-policy-list[0].type[0]=user +zuul.ratelimit.default-policy-list[0].type[1]=origin +zuul.ratelimit.default-policy-list[0].type[2]=url +zuul.ratelimit.default-policy-list[0].type[3]=http_method +zuul.ratelimit.policy-list.userGrade[0].limit=5 +zuul.ratelimit.policy-list.userGrade[0].quota=1000 +zuul.ratelimit.policy-list.userGrade[0].refresh-interval=60 +zuul.ratelimit.policy-list.userGrade[0].type[0]=user +zuul.ratelimit.policy-list.userGrade[0].type[1]=origin +zuul.ratelimit.policy-list.userGrade[0].type[2]=url +zuul.ratelimit.policy-list.userGrade[1].limit=10 +zuul.ratelimit.policy-list.userGrade[1].quota=1000 +zuul.ratelimit.policy-list.userGrade[1].refresh-interval=60 +zuul.ratelimit.policy-list.userGrade[1].type[0]=user +zuul.ratelimit.policy-list.userGrade[1].type[1]=origin +zuul.ratelimit.policy-list.userGrade[1].type[2]=url +zuul.ratelimit.policy-list.userGrade[2].limit=20 +zuul.ratelimit.policy-list.userGrade[2].quota=1000 +zuul.ratelimit.policy-list.userGrade[2].refresh-interval=60 +zuul.ratelimit.policy-list.userGrade[2].type[0]=user +zuul.ratelimit.policy-list.userGrade[2].type[1]=origin +zuul.ratelimit.policy-list.userGrade[2].type[2]=url +#main ds config +spring.datasource.main.jdbc-url=${MAIN_DS_URL} +spring.datasource.main.username=${DB_USER_NAME} +spring.datasource.main.password=${DB_PWD} +#barnard ds config +spring.datasource.barnard.jdbc-url=${BARNARD_DS_URL} +spring.datasource.barnard.username=${DB_USER_NAME} +spring.datasource.barnard.password=${DB_PWD} +#halley ds config +spring.datasource.halley.jdbc-url=${HALLEY_DS_URL} +spring.datasource.halley.username=${DB_USER_NAME} +spring.datasource.halley.password=${DB_PWD} +# vega ds config +spring.datasource.vega.jdbc-url=${HALLEY_DS_URL} +spring.datasource.vega.username=${DB_USER_NAME} +spring.datasource.vega.password=${DB_PWD} + +#default ds config +spring.datasource.hikari.minimum-idle=1 +spring.datasource.hikari.maximum-pool-size=2 +spring.datasource.url=${DS_URL} +spring.datasource.username=${DB_USER_NAME} +spring.datasource.password=${DB_PWD} +#user_info ds config +spring.datasource.userinfo.jdbc-url=${STARCOIN_USER_DS_URL} +spring.datasource.userinfo.username=${DB_USER_NAME} +spring.datasource.userinfo.password=${DB_PWD} \ No newline at end of file From b178572d0e52c205bde8553d1ea4d28a7d450675 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Fri, 28 Jun 2024 13:47:24 +0800 Subject: [PATCH 54/82] [debug] fixed local env file --- starcoin-scan-api/local.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starcoin-scan-api/local.env b/starcoin-scan-api/local.env index ac373c2..fa26630 100644 --- a/starcoin-scan-api/local.env +++ b/starcoin-scan-api/local.env @@ -8,6 +8,6 @@ MAIN_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=main BARNARD_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=barnard HALLEY_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=halley DS_URL=jdbc:postgresql://localhost/starcoin -STARCOIN_USER_DS_URL="jdbc:postgresql://localhost/starcoin?currentSchema=starcoin_user" +STARCOIN_USER_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=starcoin_user DB_USER_NAME=starcoin DB_PWD=starcoin \ No newline at end of file From d8b2cdfae2163a38c7717f03c5601a50d05718fb Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 29 Jun 2024 09:50:32 +0800 Subject: [PATCH 55/82] [debug] fixed parameters error --- .../controller/DagInspectorController.java | 14 +++--- .../scan/service/DagInspectorService.java | 45 ++++++++++++++++--- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java index a14263d..c18b610 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java @@ -7,8 +7,6 @@ import org.starcoin.scan.service.vo.DIAppConfigVo; import org.starcoin.scan.service.vo.DIBlocksAndEdgesAndHeightGroupsVo; -import java.util.List; - @Api(tags = "dag-inspector") @RestController @RequestMapping("v2/dag-inspector") @@ -26,12 +24,12 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlocksBetweenHeights( return dagInspectorService.getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } - @GetMapping("/{network}/header") - public DIBlocksAndEdgesAndHeightGroupsVo header( + @GetMapping("/{network}/head") + public DIBlocksAndEdgesAndHeightGroupsVo head( @PathVariable("network") String network, - @RequestParam Long heightDifferent + @RequestParam Long heightDifference ) throws Exception { - return dagInspectorService.getHead(network, heightDifferent); + return dagInspectorService.getHead(network, heightDifference); } @GetMapping("/{network}/blockHash") @@ -46,10 +44,10 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash( @GetMapping("/{network}/blockDAAScore") public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore( @PathVariable("network") String network, - @RequestParam Long daaScore, + @RequestParam Long blockDAAScore, @RequestParam Long heightDifference ) throws Exception { - return dagInspectorService.getBlockDAAScore(network, daaScore, heightDifference); + return dagInspectorService.getBlockDAAScore(network, blockDAAScore, heightDifference); } @GetMapping("/{network}/appConfig") diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index e01e6e4..85fdf05 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -28,6 +28,7 @@ import org.starcoin.scan.service.vo.*; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -53,12 +54,26 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String } groups.setBlocks(blockList); - groups.setEdges(getEdgeList(network, startHeight, endHeight)); + + List blockEdges = getEdgeList(network, startHeight, endHeight); + groups.setEdges(blockEdges); Set heightList = blockList.stream() .map(DagInspectorBlock::getHeight) .collect(Collectors.toSet()); + + // Add edges FromHeight and ToHeight + heightList.addAll( + blockEdges.stream() + .map(DagInspectorEdge::getFromHeight) + .collect(Collectors.toSet()) + ); + heightList.addAll( + blockEdges.stream() + .map(DagInspectorEdge::getToHeight) + .collect(Collectors.toSet()) + ); groups.setHeightGroups(getHeightGroup(network, heightList)); return groups; @@ -111,11 +126,20 @@ private List getBlockList(String network, Long startHeight, L SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery("height").gte(startHeight).lte(endHeight))); sourceBuilder.sort("height", SortOrder.ASC); - sourceBuilder.size(endHeight.intValue() - startHeight.intValue()); + //sourceBuilder.size(endHeight.intValue() - startHeight.intValue()); + sourceBuilder.size(1000); searchRequest.source(sourceBuilder); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); - return result.getContents(); + return result.getContents().stream().peek(block -> { + if (block.getMergeSetBlueIds() == null) { + block.setMergeSetBlueIds(new ArrayList<>()); + } + + if (block.getMergeSetRedIds() == null) { + block.setMergeSetRedIds(new ArrayList<>()); + } + }).collect(Collectors.toList()); } /** @@ -132,7 +156,9 @@ List getEdgeList(String network, Long startHeight, Long endHei RangeQueryBuilder toHeightQuery = QueryBuilders.rangeQuery("to_height").lte(endHeight); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() .query(QueryBuilders.boolQuery().must(fromHeightQuery).must(toHeightQuery)) - .sort("to_height", SortOrder.ASC); + .sort("to_height", SortOrder.ASC) + .size(100); + searchRequest.source(sourceBuilder); SearchResponse searchResponse; try { @@ -156,7 +182,8 @@ protected List getHeightGroup(String network, Set SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECT_HEIGHT_GROUP)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() .query(QueryBuilders.termsQuery("height", heights)); - sourceBuilder.size(heights.size()); + //sourceBuilder.size(heights.size()); + sourceBuilder.size(1000); searchRequest.source(sourceBuilder); SearchResponse searchResponse; try { @@ -170,7 +197,13 @@ protected List getHeightGroup(String network, Set } public DIAppConfigVo getAppConfig(String network) { - return new DIAppConfigVo(); + DIAppConfigVo configVo = new DIAppConfigVo(); + configVo.setApiVersion("2.0.0"); + configVo.setNetwork(network); + configVo.setProcessingVersion("0.0.1"); + configVo.setWebVersion("0.0.1"); + configVo.setStarcoinVersion("2.0.0"); + return configVo; } private Long getMaxHeightFromStorage(String network) throws IOException { From 45cbed24fbb3fbb4f66784b9997064361fe58f2a Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Mon, 1 Jul 2024 12:17:08 +0800 Subject: [PATCH 56/82] [bug fix] upgrade starcoin-java sdk for fixed parser error of some RPC calling --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8201b60..60122ae 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.starcoin sdk - 1.2.8 + 1.2.9 org.springframework.boot From e5a3fbe2150f48c4670463cec264454b44711526 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Mon, 1 Jul 2024 12:37:17 +0800 Subject: [PATCH 57/82] [bug fix] fixed catch exception error of type error --- .../starcoin/indexer/handler/DagInspectorIndexer.java | 2 +- .../indexer/handler/DagInspectorIndexerHandler.java | 2 +- .../starcoin/indexer/handler/ElasticSearchHandler.java | 4 ++-- .../org/starcoin/indexer/handler/IndexerHandle.java | 4 ++-- .../org/starcoin/indexer/handler/MarketCapHandle.java | 2 +- .../org/starcoin/indexer/handler/RepairHandle.java | 4 ++-- .../org/starcoin/indexer/handler/ServiceUtils.java | 10 +++++++--- .../org/starcoin/indexer/handler/SubscribeHandler.java | 2 +- .../java/org/starcoin/indexer/handler/SwapHandle.java | 2 +- .../indexer/handler/TransactionInfoHandle.java | 2 +- .../indexer/handler/TransactionPayloadHandle.java | 2 +- .../java/org/starcoin/indexer/handler/TvlService.java | 2 +- .../org/starcoin/indexer/handler/UpdateTVLHandle.java | 2 +- .../org/starcoin/indexer/handler/SwapIndexerTest.java | 2 +- .../java/org/starcoin/indexer/handler/TvlTest.java | 2 +- 15 files changed, 24 insertions(+), 20 deletions(-) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java index 4b3cc17..e61e103 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java @@ -10,7 +10,7 @@ import org.starcoin.bean.Block; import org.starcoin.bean.BlockHeader; import org.starcoin.bean.BlockOffset; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import javax.annotation.PostConstruct; import java.io.IOException; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index 40a5596..a070c28 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -23,7 +23,7 @@ import org.starcoin.api.Result; import org.starcoin.bean.*; import org.starcoin.constant.Constant; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.starcoin.utils.ExceptionWrap; import javax.annotation.PostConstruct; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java index 29227fa..0c8672f 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java @@ -54,7 +54,7 @@ import org.starcoin.constant.Constant; import org.starcoin.indexer.service.AddressHolderService; import org.starcoin.indexer.service.TransactionPayloadService; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.starcoin.types.AccountAddress; import org.starcoin.types.StructTag; import org.starcoin.types.TokenCode; @@ -379,7 +379,7 @@ public void bulk(List blockList) { addTokenInfo(tokenInfo, codeStr); //add to cache tokenCache.put(codeStr, tokenInfo); - } catch (JSONRPC2SessionException | JsonProcessingException e) { + } catch (JSONRPC2SessionException e) { logger.error("flush token error:", e); } } diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java index b9a6f5b..98451c2 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java @@ -11,7 +11,7 @@ import org.starcoin.bean.Block; import org.starcoin.bean.BlockHeader; import org.starcoin.bean.BlockOffset; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import javax.annotation.PostConstruct; import java.util.ArrayList; @@ -169,7 +169,7 @@ protected void executeInternal(JobExecutionContext jobExecutionContext) { localBlockOffset.setBlockHash(currentHandleHeader.getBlockHash()); elasticSearchHandler.setRemoteOffset(localBlockOffset); logger.info("indexer update success: {}", localBlockOffset); - } catch (JSONRPC2SessionException | JsonProcessingException e) { + } catch (JSONRPC2SessionException e) { logger.error("chain header error:", e); } } diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java index 761bd13..1cfbe6f 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java @@ -19,7 +19,7 @@ import org.starcoin.api.TokenContractRPCClient; import org.starcoin.bean.*; import org.starcoin.constant.Constant; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import javax.annotation.PostConstruct; import java.io.IOException; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java index fdf466e..17bcdad 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java @@ -10,7 +10,7 @@ import org.starcoin.api.TransactionRPCClient; import org.starcoin.bean.Block; import org.starcoin.bean.BlockHeader; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import java.util.ArrayList; import java.util.HashMap; @@ -66,7 +66,7 @@ public boolean autoRepair(long startNumber, int count) { logger.info("repair too fast: {}", startNumber); return false; } - } catch (JSONRPC2SessionException | JsonProcessingException e) { + } catch (JSONRPC2SessionException e) { logger.error("get master error: ", e); return false; } diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java index d355971..2a053ed 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java @@ -7,6 +7,7 @@ import com.novi.bcs.BcsDeserializer; import com.novi.serde.Bytes; import com.novi.serde.DeserializationError; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; @@ -22,7 +23,6 @@ import org.starcoin.api.TransactionRPCClient; import org.starcoin.bean.*; import org.starcoin.bean.TransferOffset; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import org.starcoin.utils.ResultWithId; import org.starcoin.types.ModuleId; import org.starcoin.types.ScriptFunction; @@ -180,7 +180,7 @@ static TokenInfo getTokenInfo(StateRPCClient stateRPCClient, String tokenCode) { if (tokenInfo != null) { tokenCache.put(tokenCode, tokenInfo); } - } catch (JSONRPC2SessionException | JsonProcessingException e) { + } catch (JSONRPC2SessionException e) { logger.error("get token info error:", e); } } @@ -198,7 +198,11 @@ public static BigDecimal divideScalingFactor(StateRPCClient stateRPCClient, Stri return actualValue; } - public static Map getTokenReserveFromState(StateRPCClient stateRPCClient, String contractAddress, String stateRoot) throws JSONRPC2SessionException { + public static Map getTokenReserveFromState( + StateRPCClient stateRPCClient, + String contractAddress, + String stateRoot + ) throws JSONRPC2SessionException { ListResource resource = stateRPCClient.getState(contractAddress, true, stateRoot); // System.out.println(resource); Map poolReserves = new HashMap<>(); diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SubscribeHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SubscribeHandler.java index 14c61ee..b813260 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SubscribeHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SubscribeHandler.java @@ -7,7 +7,7 @@ import org.starcoin.api.TransactionRPCClient; import org.starcoin.bean.PendingTransaction; import org.starcoin.bean.PendingTransactionNotification; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.web3j.protocol.websocket.WebSocketService; import java.net.ConnectException; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SwapHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SwapHandle.java index 34519e0..52d974e 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SwapHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/SwapHandle.java @@ -13,7 +13,7 @@ import org.starcoin.indexer.service.SwapStatService; import org.starcoin.indexer.service.SwapTxnService; import org.starcoin.indexer.service.TokenStatService; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.starcoin.utils.NumberUtils; import org.starcoin.utils.StructTagUtil; import org.starcoin.utils.SwapApiClient; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionInfoHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionInfoHandle.java index c38d676..7b7124f 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionInfoHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionInfoHandle.java @@ -21,7 +21,7 @@ import org.starcoin.api.TransactionRPCClient; import org.starcoin.bean.Transaction; import org.starcoin.constant.Constant; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import java.io.IOException; import java.util.List; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionPayloadHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionPayloadHandle.java index 1a8504f..6a487f9 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionPayloadHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TransactionPayloadHandle.java @@ -16,7 +16,7 @@ import org.starcoin.constant.Constant; import org.starcoin.constant.StarcoinNetwork; import org.starcoin.indexer.service.SwapTxnService; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.starcoin.types.*; import org.starcoin.types.StructTag; import org.starcoin.types.TransactionPayload; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TvlService.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TvlService.java index 3d6aff4..ecbcca2 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TvlService.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/TvlService.java @@ -8,7 +8,7 @@ import org.starcoin.bean.ContractCall; import org.starcoin.bean.TokenPairTvl; import org.starcoin.bean.TokenTvlAmount; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import java.math.BigInteger; import java.util.ArrayList; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/UpdateTVLHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/UpdateTVLHandle.java index 14adcc8..44a8f5e 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/UpdateTVLHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/UpdateTVLHandle.java @@ -13,7 +13,7 @@ import org.starcoin.indexer.service.SwapPoolStatService; import org.starcoin.indexer.service.SwapStatService; import org.starcoin.indexer.service.TokenStatService; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.starcoin.utils.SwapApiClient; import javax.annotation.PostConstruct; diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java index b1a343d..723bb32 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java @@ -14,7 +14,7 @@ import org.starcoin.bean.SwapTransaction; import org.starcoin.bean.SwapType; import org.starcoin.indexer.service.SwapTxnService; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.starcoin.utils.SwapApiClient; import java.io.IOException; diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java index 2985792..c88d23c 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java @@ -1,10 +1,10 @@ package org.starcoin.indexer.handler; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.junit.Test; import org.starcoin.api.ContractRPCClient; import org.starcoin.api.StateRPCClient; import org.starcoin.bean.ContractCall; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import java.net.MalformedURLException; import java.net.URL; From 3ea25048e96447508a460f91be6f5612ae2ea029 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Mon, 1 Jul 2024 14:30:08 +0800 Subject: [PATCH 58/82] [dag indexer] Solve the problem of missing blocks --- .../java/org/starcoin/constant/Constant.java | 6 +- .../indexer/handler/BlockIndexerOffset.java | 136 ++++++++++++++++++ .../indexer/handler/DagInspectorIndexer.java | 118 +++++++-------- .../handler/DagInspectorIndexerHandler.java | 11 +- .../scan/service/DagInspectorService.java | 14 +- 5 files changed, 212 insertions(+), 73 deletions(-) create mode 100644 starcoin-indexer/src/main/java/org/starcoin/indexer/handler/BlockIndexerOffset.java diff --git a/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java b/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java index fb404ae..a298c54 100644 --- a/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java +++ b/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java @@ -15,9 +15,9 @@ public class Constant { public static final String MARKET_CAP_INDEX = "market_cap"; public static final String PAYLOAD_INDEX = "txn_payloads"; public static final String TOKEN_INFO_INDEX = "token_info"; - public static final String DAG_INSPECTOR_BLOCK = "dag_inspector_block"; - public static final String DAG_INSPECTOR_EDGE = "dag_inspector_edge"; - public static final String DAG_INSPECT_HEIGHT_GROUP = "dag_inspector_height_group"; + public static final String DAG_INSPECTOR_BLOCK_INDEX = "dag_inspector_block"; + public static final String DAG_INSPECTOR_EDGE_INDEX = "dag_inspector_edge"; + public static final String DAG_INSPECT_HEIGHT_GROUP_INDEX = "dag_inspector_height_group"; public static final String EVENT_FILTER_ADDRESS = "0x00000000000000000000000000000001"; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/BlockIndexerOffset.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/BlockIndexerOffset.java new file mode 100644 index 0000000..e378304 --- /dev/null +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/BlockIndexerOffset.java @@ -0,0 +1,136 @@ +package org.starcoin.indexer.handler; + +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.GetMappingsRequest; +import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.client.indices.PutMappingRequest; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.starcoin.api.BlockRPCClient; +import org.starcoin.bean.Block; +import org.starcoin.bean.BlockHeader; +import org.starcoin.bean.BlockOffset; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class BlockIndexerOffset { + + private static final Logger logger = LoggerFactory.getLogger(BlockIndexerOffset.class); + + private String offsetIndexer; + + private BlockOffset localBlockOffset; + + private BlockHeader currentHandleHeader; + + private RestHighLevelClient esClient; + + private BlockRPCClient blockRPCClient; + + + public BlockIndexerOffset(String offsetIndexer, BlockRPCClient blockRPCClient, RestHighLevelClient client) { + this.offsetIndexer = offsetIndexer; + this.esClient = client; + this.blockRPCClient = blockRPCClient; + } + + public void initRemoteOffset() { + //update current handle header + try { + localBlockOffset = getRemoteOffset(); + if (localBlockOffset != null) { + Block block = blockRPCClient.getBlockByHeight(localBlockOffset.getBlockHeight()); + if (block != null) { + currentHandleHeader = block.getHeader(); + } else { + logger.error("init offset block not exist on chain: {}", localBlockOffset); + } + } else { + logger.warn("offset is null,init reset to genesis"); + currentHandleHeader = blockRPCClient.getBlockByHeight(0).getHeader(); + updateBlockOffset(0L, currentHandleHeader.getBlockHash()); + logger.info("init offset ok: {}", localBlockOffset); + } + } catch (JSONRPC2SessionException e) { + logger.error("set current header error:", e); + } + } + + public void updateBlockOffset(Long blockHeight, String blockHash) { + if (localBlockOffset == null) { + localBlockOffset = new BlockOffset(blockHeight, blockHash); + } else { + localBlockOffset.setBlockHeight(blockHeight); + localBlockOffset.setBlockHash(blockHash); + } + setRemoteOffset(localBlockOffset); + } + + public Long getLocalBlockOffsetHeight() { + return localBlockOffset.getBlockHeight(); + } + + public String getLocalOffsetBlockHash() { + return localBlockOffset.getBlockHash(); + } + + public BlockOffset getRemoteOffset() { + GetMappingsRequest request = new GetMappingsRequest(); + try { + request.indices(offsetIndexer); + GetMappingsResponse response = esClient.indices().getMapping(request, RequestOptions.DEFAULT); + MappingMetadata data = response.mappings().get(offsetIndexer); + Object meta = data.getSourceAsMap().get("_meta"); + if (meta != null) { + Map tip = (Map) ((LinkedHashMap) meta).get("tip"); + String blockHash = tip.get("block_hash").toString(); + Integer blockHeight = (Integer) tip.get("block_number"); + return new BlockOffset(blockHeight.longValue(), blockHash); + } + } catch (Exception e) { + logger.error("get remote offset error:", e); + } + return null; + } + + public void setRemoteOffset(BlockOffset blockOffset) { + PutMappingRequest request = new PutMappingRequest(offsetIndexer); + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.startObject("_meta"); + { + builder.startObject("tip"); + { + builder.field("block_hash", blockOffset.getBlockHash()); + builder.field("block_number", blockOffset.getBlockHeight()); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + request.source(builder); + esClient.indices().putMapping(request, RequestOptions.DEFAULT); + logger.info("remote offset update ok : {}", blockOffset); + } catch (Exception e) { + logger.error("get remote offset error:", e); + } + } + + @Override + public String toString() { + return "BlockIndexerOffset{" + + "offsetIndexer='" + offsetIndexer + '\'' + + ", localBlockOffset=" + localBlockOffset + + ", currentHandleHeader=" + currentHandleHeader + + '}'; + } +} diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java index e61e103..53ec3b8 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java @@ -1,5 +1,6 @@ package org.starcoin.indexer.handler; +import org.elasticsearch.client.RestHighLevelClient; import org.quartz.JobExecutionContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,6 +19,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.LongStream; +import static org.starcoin.constant.Constant.DAG_INSPECTOR_BLOCK_INDEX; + public class DagInspectorIndexer extends QuartzJobBean { private static final Logger logger = LoggerFactory.getLogger(DagInspectorIndexer.class); @@ -25,58 +28,44 @@ public class DagInspectorIndexer extends QuartzJobBean { @Autowired private DagInspectorIndexerHandler inspectorHandler; - @Autowired - private ElasticSearchHandler elasticSearchHandler; + private BlockIndexerOffset blockIndexerOffset; @Autowired private BlockRPCClient blockRPCClient; + @Autowired + private RestHighLevelClient esClient; + @Value("${starcoin.indexer.bulk_size}") private long bulkSize; - private BlockOffset localBlockOffset; - - private BlockHeader currentHandleHeader; + @Value("${starcoin.network}") + private String network; @PostConstruct public void initOffset() { - localBlockOffset = elasticSearchHandler.getRemoteOffset(); - //update current handle header - try { - if (localBlockOffset != null) { - Block block = blockRPCClient.getBlockByHeight(localBlockOffset.getBlockHeight()); - if (block != null) { - currentHandleHeader = block.getHeader(); - } else { - logger.error("init offset block not exist on chain: {}", localBlockOffset); - } - - } else { - logger.warn("offset is null,init reset to genesis"); - currentHandleHeader = blockRPCClient.getBlockByHeight(0).getHeader(); - localBlockOffset = new BlockOffset(0, currentHandleHeader.getBlockHash()); - elasticSearchHandler.setRemoteOffset(localBlockOffset); - logger.info("init offset ok: {}", localBlockOffset); - } - } catch (JSONRPC2SessionException e) { - logger.error("set current header error:", e); - } + blockIndexerOffset = new BlockIndexerOffset( + ServiceUtils.getIndex(network, DAG_INSPECTOR_BLOCK_INDEX), + blockRPCClient, + esClient + ); + blockIndexerOffset.initRemoteOffset(); } @Override protected void executeInternal(JobExecutionContext jobExecutionContext) { //read current offset - if (localBlockOffset == null || currentHandleHeader == null) { + if (blockIndexerOffset == null) { initOffset(); } - BlockOffset remoteBlockOffset = elasticSearchHandler.getRemoteOffset(); + BlockOffset remoteBlockOffset = blockIndexerOffset.getRemoteOffset(); logger.info("current remote offset: {}", remoteBlockOffset); if (remoteBlockOffset == null) { logger.warn("offset must not null, please check blocks.mapping!!"); return; } - if (remoteBlockOffset.getBlockHeight() > localBlockOffset.getBlockHeight()) { + if (remoteBlockOffset.getBlockHeight() > blockIndexerOffset.getLocalBlockOffsetHeight()) { logger.info("indexer equalize chain blocks."); return; } @@ -93,12 +82,12 @@ public void fetchAndProcessBlocksParallel() { // Calculate bulk size long headHeight = chainHeader.getHeight(); - long bulkNumber = Math.min(headHeight - localBlockOffset.getBlockHeight(), bulkSize); + long bulkNumber = Math.min(headHeight - blockIndexerOffset.getLocalBlockOffsetHeight(), bulkSize); ConcurrentHashMap blockMap = new ConcurrentHashMap<>(); LongStream.rangeClosed(1, bulkNumber).parallel().forEach(index -> { - long currentBlockHeight = localBlockOffset.getBlockHeight() + index; + long currentBlockHeight = blockIndexerOffset.getLocalBlockOffsetHeight() + index; logger.info("Start Get block number: {}, currentBlockHeight: {}", index, currentBlockHeight); Block block; @@ -125,10 +114,10 @@ public void fetchAndProcessBlocksParallel() { .map(Block::getHeader) .max(Comparator.comparingLong(BlockHeader::getHeight)).orElseThrow(); - localBlockOffset.setBlockHeight(lastBlockHeader.getHeight()); - localBlockOffset.setBlockHash(lastBlockHeader.getBlockHash()); - elasticSearchHandler.setRemoteOffset(localBlockOffset); - logger.info("indexer update success: {}", localBlockOffset); + blockIndexerOffset.updateBlockOffset(lastBlockHeader.getHeight(), lastBlockHeader.getBlockHash()); + + logger.info("Index update success: {}", blockIndexerOffset); + } catch (JSONRPC2SessionException | IOException e) { logger.error("chain header error:", e); } finally { @@ -140,18 +129,18 @@ public void fetchAndProcessBlocksParallel() { public void fetchAndProcessBlockSequel() { // Read chain header try { - BlockHeader chainHeader = blockRPCClient.getChainHeader(); - - // Calculate bulk size - long headHeight = chainHeader.getHeight(); - long bulkNumber = Math.min(headHeight - localBlockOffset.getBlockHeight(), bulkSize); + long bulkNumber = Math.min(blockRPCClient.getChainHeader().getHeight() - blockIndexerOffset.getLocalBlockOffsetHeight(), bulkSize); int index = 1; List blockList = new ArrayList<>(); - long minHeight = localBlockOffset.getBlockHeight(); + long minHeight = blockIndexerOffset.getLocalBlockOffsetHeight(); + String currentBlockHash = blockIndexerOffset.getLocalOffsetBlockHash(); Set visit = new HashSet<>(); Deque deque = new ArrayDeque<>(); + + long currentBlockHeight = minHeight; + while (index <= bulkNumber) { - long currentBlockHeight = localBlockOffset.getBlockHeight() + index; + currentBlockHeight = minHeight + index; logger.info("Start Get block number: {}, currentBlockHeight: {}", index, currentBlockHeight); Block block = blockRPCClient.getBlockByHeight(currentBlockHeight); @@ -169,18 +158,18 @@ public void fetchAndProcessBlockSequel() { } } blockList.add(block); - //update current header - currentHandleHeader = block.getHeader(); index++; + + currentBlockHash = block.getHeader().getBlockHash(); + logger.info("add block: {}", block.getHeader()); } inspectorHandler.upsertDagInfoFromBlocks(blockList); // Update offset - localBlockOffset.setBlockHeight(currentHandleHeader.getHeight()); - localBlockOffset.setBlockHash(currentHandleHeader.getBlockHash()); - elasticSearchHandler.setRemoteOffset(localBlockOffset); - logger.info("indexer update success: {}", localBlockOffset); + blockIndexerOffset.updateBlockOffset(currentBlockHeight, currentBlockHash); + + logger.info("indexer update success: {}", blockIndexerOffset); } catch (JSONRPC2SessionException | IOException e) { logger.error("chain header error:", e); } @@ -193,17 +182,30 @@ void fetchParentsBlock( List blockList, long minHeight ) throws JSONRPC2SessionException { + List parents = block.getHeader().getParentsHash(); + if (parents == null || parents.isEmpty()) { + return ; + } + for (String parent : block.getHeader().getParentsHash()) { - if (!visit.contains(parent)) { - visit.add(parent); - Block block_parent = elasticSearchHandler.getBlockContent(parent); - if (block_parent == null) { - block_parent = blockRPCClient.getBlockByHash(parent); - if (block_parent.getHeader().getHeight() >= minHeight) { - deque.addLast(block_parent); - blockList.add(block_parent); - } - } + if (visit.contains(parent)) { + continue; + } + visit.add(parent); + Block block_parent = blockList + .stream() + .filter(b -> b.getHeader().getBlockHash().compareToIgnoreCase(parent) == 0) + .findAny() + .orElse(null); + + if (block_parent != null) { + continue; + } + + block_parent = blockRPCClient.getBlockByHash(parent); + if (block_parent.getHeader().getHeight() >= minHeight) { + deque.addLast(block_parent); + blockList.add(block_parent); } } } diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index a070c28..447790b 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -55,9 +55,9 @@ public class DagInspectorIndexerHandler { @PostConstruct public void initIndexs() throws IOException { - dagInspectNodeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_BLOCK); - dagInspectEdgeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_EDGE); - dagInspectHeightGroupIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECT_HEIGHT_GROUP); + dagInspectNodeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_BLOCK_INDEX); + dagInspectEdgeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_EDGE_INDEX); + dagInspectHeightGroupIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECT_HEIGHT_GROUP_INDEX); } public void upsertDagInfoFromBlocks(List blockList) throws IOException, JSONRPC2SessionException { @@ -139,11 +139,12 @@ List buildEdgeDataFromDagBlockDataMaybeUpate( List newDagBlocks = new ArrayList<>(); for (DagInspectorBlock dagBlock : dagBlockMap.values()) { - if (dagBlock.getParentIds().isEmpty()) { + List parentIds = dagBlock.getParentIds(); + if (parentIds == null || parentIds.isEmpty()) { continue; } - for (String parentHash : dagBlock.getParentIds()) { + for (String parentHash : parentIds) { DagInspectorBlock parentDagBlock = dagBlockMap.get(parentHash); if (parentDagBlock == null) { logger.info("Parent block not found: {} ", parentHash); diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index 85fdf05..a94e04d 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -122,7 +122,7 @@ public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDiff * @return */ private List getBlockList(String network, Long startHeight, Long endHeight) throws ElasticsearchException, IOException { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK_INDEX)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery("height").gte(startHeight).lte(endHeight))); sourceBuilder.sort("height", SortOrder.ASC); @@ -151,7 +151,7 @@ private List getBlockList(String network, Long startHeight, L * @return */ List getEdgeList(String network, Long startHeight, Long endHeight) { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_EDGE)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_EDGE_INDEX)); RangeQueryBuilder fromHeightQuery = QueryBuilders.rangeQuery("from_height").gte(startHeight); RangeQueryBuilder toHeightQuery = QueryBuilders.rangeQuery("to_height").lte(endHeight); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() @@ -179,7 +179,7 @@ List getEdgeList(String network, Long startHeight, Long endHei * @return */ protected List getHeightGroup(String network, Set heights) { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECT_HEIGHT_GROUP)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECT_HEIGHT_GROUP_INDEX)); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() .query(QueryBuilders.termsQuery("height", heights)); //sourceBuilder.size(heights.size()); @@ -207,7 +207,7 @@ public DIAppConfigVo getAppConfig(String network) { } private Long getMaxHeightFromStorage(String network) throws IOException { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK_INDEX)); // Build the SearchSourceBuilder with max aggregation SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -240,7 +240,7 @@ private Long getMaxHeightFromStorage(String network) throws IOException { } private DagInspectorBlock getBlockWithHashFromStorage(String network, String blockHash) { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK_INDEX)); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.should(QueryBuilders.termQuery("block_hash", blockHash)); @@ -258,7 +258,7 @@ private DagInspectorBlock getBlockWithHashFromStorage(String network, String blo } private DagInspectorBlock getHeightWithDAAScoreFromStorage(String network, Long daaScore) { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK_INDEX)); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.should(QueryBuilders.termQuery("daa_score", daaScore)); @@ -276,7 +276,7 @@ private DagInspectorBlock getHeightWithDAAScoreFromStorage(String network, Long } public List getBlocksByHeight(String network, Long height) { - SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK)); + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK_INDEX)); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); boolQueryBuilder.should(QueryBuilders.termQuery("height", height)); From 809d2253e6352d9420ccb77c20b382c449c67d6f Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Mon, 1 Jul 2024 17:33:18 +0800 Subject: [PATCH 59/82] [dag indexer] Fixed error of get max height api --- .../handler/DagInspectorIndexerHandler.java | 18 ++++++--- .../scan/service/DagInspectorService.java | 37 ++++--------------- 2 files changed, 19 insertions(+), 36 deletions(-) diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java index 447790b..23d8f8d 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -91,11 +91,15 @@ public void upsertDagInfoFromBlocks(List blockList) throws IOException, J dagBlock = new DagInspectorBlock(); dagBlock.setBlockHash(currentBlockHash); dagBlock.setTimestamp(block.getHeader().getTimestamp()); - dagBlock.setColor(NODE_COLOR_GRAY); + // TODO(bobong): Check color + dagBlock.setColor(NODE_COLOR_BLUE); dagBlock.setDaaScore(ghostdagData.getBlueScore()); dagBlock.setHeight(block.getHeader().getHeight()); dagBlock.setSelectedParentHash(ghostdagData.getSelectedParent()); - dagBlock.setParentIds(block.getHeader().getParentsHash()); + dagBlock.setParentIds(block.getHeader().getParentsHash() != null ? block.getHeader().getParentsHash() : new ArrayList<>()); + dagBlock.setMergeSetRedIds(ghostdagData.getMergesetReds() != null ? ghostdagData.getMergesetReds() : new ArrayList<>()); + dagBlock.setMergeSetBlueIds(ghostdagData.getMergesetBlues() != null ? ghostdagData.getMergesetBlues() : new ArrayList<>()); + // Block is the virtual selected parent chain because the list read from block height dagBlock.setInVirtualSelectedParentChain(true); @@ -149,7 +153,6 @@ List buildEdgeDataFromDagBlockDataMaybeUpate( if (parentDagBlock == null) { logger.info("Parent block not found: {} ", parentHash); parentDagBlock = getDagInspectorBlockFromHash(parentHash, heightGroupList, false); - // Put into buffer list newDagBlocks.add(parentDagBlock); } @@ -337,12 +340,15 @@ protected DagInspectorBlock getDagInspectorBlockFromHash( dagBlock.setBlockHash(blockHash); dagBlock.setTimestamp(block.getHeader().getTimestamp()); - dagBlock.setColor(NODE_COLOR_GRAY); - dagBlock.setDaaScore(blockGhostdagData.getBlueScore()); + // TODO: Check color + dagBlock.setColor(NODE_COLOR_BLUE); + dagBlock.setDaaScore(blockGhostdagData.getBlueScore() != null ? blockGhostdagData.getBlueScore() : 0); dagBlock.setHeight(block.getHeader().getHeight()); dagBlock.setSelectedParentHash(blockGhostdagData.getSelectedParent()); - dagBlock.setParentIds(block.getHeader().getParentsHash()); + dagBlock.setParentIds(block.getHeader().getParentsHash() != null ? block.getHeader().getParentsHash() : new ArrayList<>()); dagBlock.setInVirtualSelectedParentChain(isSelectedParentChain); + dagBlock.setMergeSetBlueIds(blockGhostdagData.getMergesetBlues() != null ? blockGhostdagData.getMergesetBlues() : new ArrayList<>()); + dagBlock.setMergeSetRedIds(blockGhostdagData.getMergesetReds() != null ? blockGhostdagData.getMergesetReds() : new ArrayList<>()); // Height group list index Integer groupSize = getHeightGroupSizeOrDefault(heightGroupList, blockHeight, 0); diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index a94e04d..d8b43d7 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -1,6 +1,4 @@ package org.starcoin.scan.service; - -import com.alibaba.fastjson.JSONObject; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -9,12 +7,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.TopHits; -import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -208,33 +201,17 @@ public DIAppConfigVo getAppConfig(String network) { private Long getMaxHeightFromStorage(String network) throws IOException { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK_INDEX)); - - // Build the SearchSourceBuilder with max aggregation SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - TermsAggregationBuilder maxHeightAgg = AggregationBuilders - .terms("max_height_agg") - .field("height") - .size(1); - TopHitsAggregationBuilder topHitsAgg = AggregationBuilders - .topHits("top_hits") - .size(1); - - maxHeightAgg.subAggregation(topHitsAgg); - searchSourceBuilder.aggregation(maxHeightAgg); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchSourceBuilder.sort("height", SortOrder.DESC); + searchSourceBuilder.size(1); // We only need the top 1 result searchRequest.source(searchSourceBuilder); - // Execute the search request SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - Terms maxHeightTerms = searchResponse.getAggregations().get("max_height_agg"); - if (maxHeightTerms.getBuckets().size() > 0) { - Terms.Bucket bucket = maxHeightTerms.getBuckets().get(0); - TopHits topHits = bucket.getAggregations().get("top_hits"); - if (topHits.getHits().getHits().length > 0) { - String topHitsJson = topHits.getHits().getHits()[0].getSourceAsString(); - logger.info("Object with the max height: " + topHitsJson); - DagInspectorBlock block = JSONObject.parseObject(topHitsJson, DagInspectorBlock.class); - return block.getHeight(); - } + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); + List contents = result.getContents(); + if (!contents.isEmpty()) { + return contents.get(0).getHeight(); } return 0L; } From b13b7e2be8de5eb6935f7aee8167bb01b80afd8f Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Mon, 1 Jul 2024 18:35:49 +0800 Subject: [PATCH 60/82] [dag indexer] Fixed error of get max height api --- .../java/org/starcoin/scan/service/DagInspectorService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index d8b43d7..8ae538d 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -145,8 +145,8 @@ private List getBlockList(String network, Long startHeight, L */ List getEdgeList(String network, Long startHeight, Long endHeight) { SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_EDGE_INDEX)); - RangeQueryBuilder fromHeightQuery = QueryBuilders.rangeQuery("from_height").gte(startHeight); - RangeQueryBuilder toHeightQuery = QueryBuilders.rangeQuery("to_height").lte(endHeight); + RangeQueryBuilder fromHeightQuery = QueryBuilders.rangeQuery("from_height").gte(startHeight).lte(endHeight); + RangeQueryBuilder toHeightQuery = QueryBuilders.rangeQuery("to_height").gte(startHeight).lte(endHeight); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() .query(QueryBuilders.boolQuery().must(fromHeightQuery).must(toHeightQuery)) .sort("to_height", SortOrder.ASC) From 6d86d2810ae0e171cea5b663d6880bc1448ad93a Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Tue, 2 Jul 2024 15:27:12 +0800 Subject: [PATCH 61/82] [dag indexer] add dag indexer for vega --- .../starcoin-dga-indexer-vega-deployment.yaml | 64 +++++++++++++++++++ .../starcoin-indexer-vega-deployment.yaml | 4 +- 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 kube/dag-indexer/starcoin-dga-indexer-vega-deployment.yaml diff --git a/kube/dag-indexer/starcoin-dga-indexer-vega-deployment.yaml b/kube/dag-indexer/starcoin-dga-indexer-vega-deployment.yaml new file mode 100644 index 0000000..d82bc31 --- /dev/null +++ b/kube/dag-indexer/starcoin-dga-indexer-vega-deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: starcoin-dag-indexer-deployment + namespace: starcoin-vega + labels: + app: starcoin-dag-indexer +spec: + replicas: 1 + selector: + matchLabels: + app: starcoin-dag-indexer + template: + metadata: + labels: + app: starcoin-dag-indexer + spec: + containers: + - name: starcoin-dag-indexer + image: starcoin/starcoin_indexer:dag_inspector_vega_6 + ports: + - containerPort: 8300 + env: + - name: HOSTS + value: "vega.seed.starcoin.org" + - name: NETWORK + value: "vega.0727" + - name: BG_TASK_JOBS + value: "dag_inspector" + - name: BULK_SIZE + value: "10" + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch-pw-elastic + key: password + - name: STARCOIN_ES_URL + value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + - name: TXN_OFFSET + value: "0" + - name: STARCOIN_ES_PROTOCOL + value: "https" + - name: STARCOIN_ES_PORT + value: "443" + - name: STARCOIN_ES_USER + value: elastic + - name: SWAP_API_URL + value: "https://swap-api.starswap.xyz" + - name: SWAP_CONTRACT_ADDR + value: "0xbd7e8be8fae9f60f2f5136433e36a091" + - name: DS_URL + value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + - name: DB_USER_NAME + value: "starcoin" + - name: DB_PWD + valueFrom: + secretKeyRef: + name: postgresql-starcoin-yu + key: password + - name: DB_SCHEMA + value: "vega" + - name: PROGRAM_ARGS + value: "" + #valume: diff --git a/kube/indexer/starcoin-indexer-vega-deployment.yaml b/kube/indexer/starcoin-indexer-vega-deployment.yaml index 6077b5c..154d187 100644 --- a/kube/indexer/starcoin-indexer-vega-deployment.yaml +++ b/kube/indexer/starcoin-indexer-vega-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: starcoin-indexer - image: starcoin/starcoin_indexer:dag_inspector_vega_5 + image: starcoin/starcoin_indexer:dag_inspector_vega_6 ports: - containerPort: 8300 env: @@ -26,7 +26,7 @@ spec: - name: NETWORK value: "vega.0727" - name: BG_TASK_JOBS - value: "indexer,secondary,market_cap,txn_payload,swap_stats,txn_global_idx_update,dag_inspector" + value: "indexer,secondary,market_cap,txn_payload,swap_stats,txn_global_idx_update" - name: BULK_SIZE value: "80" - name: STARCOIN_ES_PWD From ba505fef84091bc48bec9f510e3683b9ebaa7d2f Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Tue, 2 Jul 2024 17:39:03 +0800 Subject: [PATCH 62/82] [dag scan-api] Solve the problem that when searching for a specific block or daascore, the following blocks are not fully displayed --- .../starcoin/scan/service/DagInspectorService.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java index 8ae538d..8afe770 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -81,8 +81,9 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash( if (block == null) { throw new RuntimeException("Cannot find block by block hash"); } - Long endHeight = block.getHeight(); - long startHeight = Math.max(block.getHeight() - heightDifference, 0L); + long blockHeight = block.getHeight(); + long startHeight = Math.max(blockHeight - heightDifference, 0L); + Long endHeight = blockHeight + heightDifference; return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } @@ -95,8 +96,10 @@ public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore( if (block == null) { throw new RuntimeException("Cannot find block by block hash"); } - Long endHeight = block.getHeight(); - long startHeight = Math.max(block.getHeight() - heightDifference, 0L); + + long blockHeight = block.getHeight(); + long startHeight = Math.max(blockHeight - heightDifference, 0L); + Long endHeight = blockHeight + heightDifference; return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); } From a96df94d42ad0e63908a0bdc9ca996db9775a724 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Fri, 5 Jul 2024 13:55:19 +0800 Subject: [PATCH 63/82] [deployment] add Digitalocean k8s Deploy Config --- .../elasticsearch-deployment.yaml | 111 ++++++++++++++++++ kube/elasticsearch/kibana-deployment.yaml | 74 ++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 kube/elasticsearch/elasticsearch-deployment.yaml create mode 100644 kube/elasticsearch/kibana-deployment.yaml diff --git a/kube/elasticsearch/elasticsearch-deployment.yaml b/kube/elasticsearch/elasticsearch-deployment.yaml new file mode 100644 index 0000000..30c1add --- /dev/null +++ b/kube/elasticsearch/elasticsearch-deployment.yaml @@ -0,0 +1,111 @@ +# elasticsearch-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch +spec: + replicas: 1 + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + spec: + initContainers: + - name: init-permissions + image: busybox + command: [ "sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data" ] + volumeMounts: + - name: es-data + mountPath: /usr/share/elasticsearch/data + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 + resources: + requests: + cpu: "1" + memory: "4Gi" + limits: + cpu: "2" + memory: "6Gi" + ports: + - containerPort: 9200 + name: http + - containerPort: 9300 + name: transport + volumeMounts: + - name: es-data + mountPath: /usr/share/elasticsearch/data + - name: elasticsearch-config + mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + subPath: elasticsearch.yml + env: + - name: discovery.type + value: single-node + - name: ELASTIC_USERNAME + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: ELASTIC_PASSWORD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password + volumes: + - name: es-data + persistentVolumeClaim: + claimName: es-pvc + - name: elasticsearch-config + configMap: + name: elasticsearch-config +--- +# Elasticsearch Configuration +apiVersion: v1 +kind: ConfigMap +metadata: + name: elasticsearch-config +data: + elasticsearch.yml: | + xpack.security.enabled: true + xpack.license.self_generated.type: basic + network.host: 0.0.0.0 +--- +# Elasticsearch Persistent Volume +apiVersion: v1 +kind: PersistentVolume +metadata: + name: es-pv +spec: + capacity: + storage: 80Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /mnt/data/es +--- +# Elasticsearch Persistent Volume Claim +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: es-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 80Gi + +--- +# Elasticsearch Service +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch +spec: + ports: + - port: 9200 + selector: + app: elasticsearch diff --git a/kube/elasticsearch/kibana-deployment.yaml b/kube/elasticsearch/kibana-deployment.yaml new file mode 100644 index 0000000..314fc7c --- /dev/null +++ b/kube/elasticsearch/kibana-deployment.yaml @@ -0,0 +1,74 @@ +# kibana-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana +spec: + replicas: 1 + selector: + matchLabels: + app: kibana + template: + metadata: + labels: + app: kibana + spec: + containers: + - name: kibana + image: docker.elastic.co/kibana/kibana:7.17.0 + resources: + requests: + cpu: "0.5" + memory: "2Gi" + limits: + cpu: "1" + memory: "2Gi" + ports: + - containerPort: 5601 + volumeMounts: + - name: kibana-config + mountPath: /usr/share/kibana/config/kibana.yml + subPath: kibana.yml + env: + - name: ELASTICSEARCH_HOSTS + value: "http://elasticsearch:9200" + - name: ES_USERNAME + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: ES_PASSWORD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password + volumes: + - name: kibana-config + configMap: + name: kibana-config +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kibana-config +data: + kibana.yml: | + server.name: kibana + server.host: "0.0.0.0" + elasticsearch.hosts: ["http://elasticsearch:9200"] + elasticsearch.username: "${ES_USERNAME}" + elasticsearch.password: "${ES_PASSWORD}" +--- +apiVersion: v1 +kind: Service +metadata: + name: kibana +spec: + type: NodePort + selector: + app: kibana + ports: + - protocol: TCP + port: 5601 + targetPort: 5601 + nodePort: 31000 \ No newline at end of file From 57aa50683f32f0e860fa3357456b279becdeeb98 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Fri, 5 Jul 2024 19:54:28 +0800 Subject: [PATCH 64/82] [deployment] add Digitalocean k8s Deploy Config for postgresql --- .../elasticsearch-deployment.yaml | 3 +- .../kibana-deployment.yaml | 0 .../postgresql-deployment.yaml | 90 +++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) rename kube/{elasticsearch => base-components}/elasticsearch-deployment.yaml (98%) rename kube/{elasticsearch => base-components}/kibana-deployment.yaml (100%) create mode 100644 kube/base-components/postgresql-deployment.yaml diff --git a/kube/elasticsearch/elasticsearch-deployment.yaml b/kube/base-components/elasticsearch-deployment.yaml similarity index 98% rename from kube/elasticsearch/elasticsearch-deployment.yaml rename to kube/base-components/elasticsearch-deployment.yaml index 30c1add..25bd918 100644 --- a/kube/elasticsearch/elasticsearch-deployment.yaml +++ b/kube/base-components/elasticsearch-deployment.yaml @@ -58,7 +58,7 @@ spec: - name: es-data persistentVolumeClaim: claimName: es-pvc - - name: elasticsearch-config + - name: elasticsearch-,.ujv config configMap: name: elasticsearch-config --- @@ -97,7 +97,6 @@ spec: resources: requests: storage: 80Gi - --- # Elasticsearch Service apiVersion: v1 diff --git a/kube/elasticsearch/kibana-deployment.yaml b/kube/base-components/kibana-deployment.yaml similarity index 100% rename from kube/elasticsearch/kibana-deployment.yaml rename to kube/base-components/kibana-deployment.yaml diff --git a/kube/base-components/postgresql-deployment.yaml b/kube/base-components/postgresql-deployment.yaml new file mode 100644 index 0000000..25c9e9f --- /dev/null +++ b/kube/base-components/postgresql-deployment.yaml @@ -0,0 +1,90 @@ +# Persistent Volume +apiVersion: v1 +kind: PersistentVolume +metadata: + name: postgres-pv +spec: + capacity: + storage: 80Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: do-block-storage + hostPath: + path: /mnt/data/postgres +--- +# Persistent Volume Claim +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 80Gi + volumeName: postgres-pv + storageClassName: do-block-storage +--- +# Postgresql deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:13 + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: starcoin + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgresql + key: username + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql + key: password + resources: + requests: + memory: "4Gi" + cpu: "2" + limits: + memory: "4Gi" + cpu: "2" + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: postgres-storage + volumes: + - name: postgres-storage + persistentVolumeClaim: + claimName: postgres-pvc +--- +# Service +apiVersion: v1 +kind: Service +metadata: + name: postgres-service +spec: + type: ClusterIP + ports: + - port: 5432 + targetPort: 5432 + selector: + app: postgres + From 16060d1ffa8120394f50c994c3b377db694fb805 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 6 Jul 2024 13:25:54 +0800 Subject: [PATCH 65/82] [Deployment] DigtalOcean, Add network cross namespace access config from starcoin-vega namespace to default --- .../allowaccess-network-policy.yaml | 33 +++++++++++++++++++ .../starcoin-dga-indexer-vega-deployment.yaml | 16 ++++----- kube/mappings/es_pipeline.scripts | 2 +- 3 files changed, 42 insertions(+), 9 deletions(-) create mode 100644 kube/base-components/allowaccess-network-policy.yaml diff --git a/kube/base-components/allowaccess-network-policy.yaml b/kube/base-components/allowaccess-network-policy.yaml new file mode 100644 index 0000000..68440c0 --- /dev/null +++ b/kube/base-components/allowaccess-network-policy.yaml @@ -0,0 +1,33 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-pg-from-vega + namespace: default +spec: + podSelector: + matchLabels: + app: postgres-service + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-vega +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-es-from-vega + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-vega diff --git a/kube/dag-indexer/starcoin-dga-indexer-vega-deployment.yaml b/kube/dag-indexer/starcoin-dga-indexer-vega-deployment.yaml index d82bc31..30fae5e 100644 --- a/kube/dag-indexer/starcoin-dga-indexer-vega-deployment.yaml +++ b/kube/dag-indexer/starcoin-dga-indexer-vega-deployment.yaml @@ -29,33 +29,33 @@ spec: value: "dag_inspector" - name: BULK_SIZE value: "10" + - name: STARCOIN_ES_USER + value: elastic - name: STARCOIN_ES_PWD valueFrom: secretKeyRef: - name: elasticsearch-pw-elastic + name: elasticsearch key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: TXN_OFFSET value: "0" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" - - name: STARCOIN_ES_USER - value: elastic + value: "9200" - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "vega" diff --git a/kube/mappings/es_pipeline.scripts b/kube/mappings/es_pipeline.scripts index fdae8cc..6d17eb2 100644 --- a/kube/mappings/es_pipeline.scripts +++ b/kube/mappings/es_pipeline.scripts @@ -33,7 +33,7 @@ PUT _component_template/block-mappings PUT _component_template/block_ids-mappings { "template": { - # copy block.mapping.json to here. + # copy block.mapping_ids.json to here. } } From fc6373a4b610acebb3639fe5b78a22c1a547b486 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 6 Jul 2024 14:30:03 +0800 Subject: [PATCH 66/82] [Deployment] DigtalOcean, Add api policy and namespace --- .../allowaccess-network-policy.yaml | 39 +++++++++++++++++++ kube/scan-api/scan-api-deployment.yaml | 26 ++++++------- kube/scan-api/service-scan-api.yaml | 1 + kube/scan-api/service-scan-hazelcast.yaml | 1 + 4 files changed, 52 insertions(+), 15 deletions(-) diff --git a/kube/base-components/allowaccess-network-policy.yaml b/kube/base-components/allowaccess-network-policy.yaml index 68440c0..896610d 100644 --- a/kube/base-components/allowaccess-network-policy.yaml +++ b/kube/base-components/allowaccess-network-policy.yaml @@ -31,3 +31,42 @@ spec: - namespaceSelector: matchLabels: name: starcoin-vega + + +--- +# Postgres service for starcoin-api +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-pg-from-api + namespace: default +spec: + podSelector: + matchLabels: + app: postgres-service + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-api + +--- +# Elasticsearch service for starcoin-api +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-es-from-api + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-api \ No newline at end of file diff --git a/kube/scan-api/scan-api-deployment.yaml b/kube/scan-api/scan-api-deployment.yaml index 436cc20..c48fc99 100644 --- a/kube/scan-api/scan-api-deployment.yaml +++ b/kube/scan-api/scan-api-deployment.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: scan-api-deployment + namespace: starcoin-api labels: app: scan-api spec: @@ -24,11 +25,11 @@ spec: containerPort: 5701 env: - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER value: elastic - name: STARCOIN_ES_INDEX_VERSION @@ -36,23 +37,18 @@ spec: - name: STARCOIN_ES_PWD valueFrom: secretKeyRef: - name: elasticsearch-pw-elastic + name: elasticsearch key: password - - name: MAIN_DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=main" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - - name: BARNARD_DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=barnard" - - name: HALLEY_DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=halley" - - name: STARCOIN_USER_DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=starcoin_user" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password diff --git a/kube/scan-api/service-scan-api.yaml b/kube/scan-api/service-scan-api.yaml index 9b0dac9..0c31a23 100644 --- a/kube/scan-api/service-scan-api.yaml +++ b/kube/scan-api/service-scan-api.yaml @@ -4,6 +4,7 @@ metadata: labels: app: scan-api name: scan-api + namespace: starcoin-api spec: ports: - port: 80 diff --git a/kube/scan-api/service-scan-hazelcast.yaml b/kube/scan-api/service-scan-hazelcast.yaml index fe76c6b..7ab72bd 100644 --- a/kube/scan-api/service-scan-hazelcast.yaml +++ b/kube/scan-api/service-scan-hazelcast.yaml @@ -4,6 +4,7 @@ metadata: labels: app: scan-api name: hazelcast-service + namespace: starcoin-api spec: selector: app: scan-api From 7ef85fbf6672ae6869e565b38a56ab90b80cc8d7 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 6 Jul 2024 14:31:19 +0800 Subject: [PATCH 67/82] [Deployment] reformat files --- ...oyment.yaml => starcoin-indexer-swapinfo-main-deployment.yaml} | 0 ...oyment.yaml => starcoin-indexer-swaptxns-main-deployment.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename kube/indexer/{starswap-info-main-deployment.yaml => starcoin-indexer-swapinfo-main-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-swap-main-deployment.yaml => starcoin-indexer-swaptxns-main-deployment.yaml} (100%) diff --git a/kube/indexer/starswap-info-main-deployment.yaml b/kube/indexer/starcoin-indexer-swapinfo-main-deployment.yaml similarity index 100% rename from kube/indexer/starswap-info-main-deployment.yaml rename to kube/indexer/starcoin-indexer-swapinfo-main-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-swap-main-deployment.yaml b/kube/indexer/starcoin-indexer-swaptxns-main-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-swap-main-deployment.yaml rename to kube/indexer/starcoin-indexer-swaptxns-main-deployment.yaml From 7ec7a2b01ff40917e620d37b448ab4a89e201573 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 6 Jul 2024 14:39:15 +0800 Subject: [PATCH 68/82] [Deployment] reformat files --- .../stcscan-barnard-deployment.yaml} | 0 .../stcscan-cmd-handle-main-deployment.yaml} | 0 .../stcscan-halley-deployment.yaml} | 0 .../stcscan-main-deployment.yaml} | 0 .../stcscan-proxima-deployment.yaml} | 0 .../stcscan-repair-barnard-deployment.yaml} | 0 .../stcscan-repair-halley-deployment.yaml} | 0 .../stcscan-repair-main-deployment.yaml} | 0 .../stcscan-txn-main-deployment.yaml} | 0 .../stcscan-vega-deployment.yaml} | 0 .../swap-info-main-deployment.yaml} | 0 .../swap-stat-main-deployment.yaml} | 0 .../swap-txns-main-deployment.yaml} | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename kube/indexer/{starcoin-indexer-barnard-deployment.yaml => stc-scan/stcscan-barnard-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-cmd-handle-main-deployment.yaml => stc-scan/stcscan-cmd-handle-main-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-halley-deployment.yaml => stc-scan/stcscan-halley-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-main-deployment.yaml => stc-scan/stcscan-main-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-proxima-deployment.yaml => stc-scan/stcscan-proxima-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-repair-barnard-deployment.yaml => stc-scan/stcscan-repair-barnard-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-repair-halley-deployment.yaml => stc-scan/stcscan-repair-halley-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-repair-main-deployment.yaml => stc-scan/stcscan-repair-main-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-txn-main-deployment.yaml => stc-scan/stcscan-txn-main-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-vega-deployment.yaml => stc-scan/stcscan-vega-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-swapinfo-main-deployment.yaml => swap/swap-info-main-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-stat-main-deployment.yaml => swap/swap-stat-main-deployment.yaml} (100%) rename kube/indexer/{starcoin-indexer-swaptxns-main-deployment.yaml => swap/swap-txns-main-deployment.yaml} (100%) diff --git a/kube/indexer/starcoin-indexer-barnard-deployment.yaml b/kube/indexer/stc-scan/stcscan-barnard-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-barnard-deployment.yaml rename to kube/indexer/stc-scan/stcscan-barnard-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-cmd-handle-main-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml rename to kube/indexer/stc-scan/stcscan-cmd-handle-main-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-halley-deployment.yaml b/kube/indexer/stc-scan/stcscan-halley-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-halley-deployment.yaml rename to kube/indexer/stc-scan/stcscan-halley-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-main-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-main-deployment.yaml rename to kube/indexer/stc-scan/stcscan-main-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-proxima-deployment.yaml b/kube/indexer/stc-scan/stcscan-proxima-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-proxima-deployment.yaml rename to kube/indexer/stc-scan/stcscan-proxima-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml b/kube/indexer/stc-scan/stcscan-repair-barnard-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml rename to kube/indexer/stc-scan/stcscan-repair-barnard-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-repair-halley-deployment.yaml b/kube/indexer/stc-scan/stcscan-repair-halley-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-repair-halley-deployment.yaml rename to kube/indexer/stc-scan/stcscan-repair-halley-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-repair-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-repair-main-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-repair-main-deployment.yaml rename to kube/indexer/stc-scan/stcscan-repair-main-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-txn-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-txn-main-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-txn-main-deployment.yaml rename to kube/indexer/stc-scan/stcscan-txn-main-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-vega-deployment.yaml b/kube/indexer/stc-scan/stcscan-vega-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-vega-deployment.yaml rename to kube/indexer/stc-scan/stcscan-vega-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-swapinfo-main-deployment.yaml b/kube/indexer/swap/swap-info-main-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-swapinfo-main-deployment.yaml rename to kube/indexer/swap/swap-info-main-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-stat-main-deployment.yaml b/kube/indexer/swap/swap-stat-main-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-stat-main-deployment.yaml rename to kube/indexer/swap/swap-stat-main-deployment.yaml diff --git a/kube/indexer/starcoin-indexer-swaptxns-main-deployment.yaml b/kube/indexer/swap/swap-txns-main-deployment.yaml similarity index 100% rename from kube/indexer/starcoin-indexer-swaptxns-main-deployment.yaml rename to kube/indexer/swap/swap-txns-main-deployment.yaml From 0f18c960fde7451741327d9ddc74d0ceec5cc168 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 6 Jul 2024 14:39:38 +0800 Subject: [PATCH 69/82] [Deployment] reformat files --- .../dag/dag-vega-deployment.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename kube/{dag-indexer/starcoin-dga-indexer-vega-deployment.yaml => indexer/dag/dag-vega-deployment.yaml} (100%) diff --git a/kube/dag-indexer/starcoin-dga-indexer-vega-deployment.yaml b/kube/indexer/dag/dag-vega-deployment.yaml similarity index 100% rename from kube/dag-indexer/starcoin-dga-indexer-vega-deployment.yaml rename to kube/indexer/dag/dag-vega-deployment.yaml From 42e7009bb2bdf44d9a4f792e67fd1ec8ea7dbe82 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 6 Jul 2024 15:43:09 +0800 Subject: [PATCH 70/82] [Deployment] Fixed DigitalOcean deploy error --- kube/indexer/dag/dag-vega-deployment.yaml | 10 +++++----- kube/scan-api/scan-api-deployment.yaml | 18 +++++++++++++++--- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/kube/indexer/dag/dag-vega-deployment.yaml b/kube/indexer/dag/dag-vega-deployment.yaml index 30fae5e..e74a184 100644 --- a/kube/indexer/dag/dag-vega-deployment.yaml +++ b/kube/indexer/dag/dag-vega-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-dag-indexer-deployment + name: starcoin-indexer-dag-deployment namespace: starcoin-vega labels: - app: starcoin-dag-indexer + app: starcoin-indexer-dag spec: replicas: 1 selector: matchLabels: - app: starcoin-dag-indexer + app: starcoin-indexer-dag template: metadata: labels: - app: starcoin-dag-indexer + app: starcoin-indexer-dag spec: containers: - - name: starcoin-dag-indexer + - name: starcoin-indexer-dag image: starcoin/starcoin_indexer:dag_inspector_vega_6 ports: - containerPort: 8300 diff --git a/kube/scan-api/scan-api-deployment.yaml b/kube/scan-api/scan-api-deployment.yaml index c48fc99..950589e 100644 --- a/kube/scan-api/scan-api-deployment.yaml +++ b/kube/scan-api/scan-api-deployment.yaml @@ -31,14 +31,17 @@ spec: - name: STARCOIN_ES_PORT value: "9200" - name: STARCOIN_ES_USER - value: elastic - - name: STARCOIN_ES_INDEX_VERSION - value: ".0727" + valueFrom: + secretKeyRef: + name: elasticsearch + key: username - name: STARCOIN_ES_PWD valueFrom: secretKeyRef: name: elasticsearch key: password + - name: STARCOIN_ES_INDEX_VERSION + value: ".0727" - name: DS_URL value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME @@ -51,4 +54,13 @@ spec: secretKeyRef: name: postgresql key: password + # Maybe useless variables + - name: MAIN_DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin?currentSchema=main" + - name: BARNARD_DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin?currentSchema=barnard" + - name: HALLEY_DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin?currentSchema=halley" + - name: STARCOIN_USER_DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin?currentSchema=starcoin_user" From 4fe719dae08695a0d2e2fe00c82cf0f3b2ad4c65 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Mon, 8 Jul 2024 11:08:24 +0800 Subject: [PATCH 71/82] [Deployment] Fixed elasticsearch k8s access policy --- .../allowaccess-network-policy.yaml | 22 +++++++++++++- kube/base-components/kibana-deployment.yaml | 4 +-- .../stc-scan/stcscan-vega-deployment.yaml | 29 ++++++++++--------- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/kube/base-components/allowaccess-network-policy.yaml b/kube/base-components/allowaccess-network-policy.yaml index 896610d..dd4c0f2 100644 --- a/kube/base-components/allowaccess-network-policy.yaml +++ b/kube/base-components/allowaccess-network-policy.yaml @@ -69,4 +69,24 @@ spec: - from: - namespaceSelector: matchLabels: - name: starcoin-api \ No newline at end of file + name: starcoin-api + +--- +# Default namespace access for elasticsearch +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-internal-elasticsearch + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - podSelector: {} + ports: + - protocol: TCP + port: 9200 diff --git a/kube/base-components/kibana-deployment.yaml b/kube/base-components/kibana-deployment.yaml index 314fc7c..20f52c7 100644 --- a/kube/base-components/kibana-deployment.yaml +++ b/kube/base-components/kibana-deployment.yaml @@ -31,7 +31,7 @@ spec: subPath: kibana.yml env: - name: ELASTICSEARCH_HOSTS - value: "http://elasticsearch:9200" + value: "http://elasticsearch.default.svc.cluster.local:9200" - name: ES_USERNAME valueFrom: secretKeyRef: @@ -55,7 +55,7 @@ data: kibana.yml: | server.name: kibana server.host: "0.0.0.0" - elasticsearch.hosts: ["http://elasticsearch:9200"] + elasticsearch.hosts: ["http://elasticsearch.default.svc.cluster.local:9200"] elasticsearch.username: "${ES_USERNAME}" elasticsearch.password: "${ES_PASSWORD}" --- diff --git a/kube/indexer/stc-scan/stcscan-vega-deployment.yaml b/kube/indexer/stc-scan/stcscan-vega-deployment.yaml index 154d187..afe6dcd 100644 --- a/kube/indexer/stc-scan/stcscan-vega-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-vega-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-deployment + name: starcoin-indexer-stcscan-deployment namespace: starcoin-vega labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer + app: starcoin-indexer-stcscan template: metadata: labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: containers: - - name: starcoin-indexer + - name: starcoin-indexer-stcscan image: starcoin/starcoin_indexer:dag_inspector_vega_6 ports: - containerPort: 8300 @@ -28,20 +28,20 @@ spec: - name: BG_TASK_JOBS value: "indexer,secondary,market_cap,txn_payload,swap_stats,txn_global_idx_update" - name: BULK_SIZE - value: "80" + value: "100" - name: STARCOIN_ES_PWD valueFrom: secretKeyRef: - name: elasticsearch-pw-elastic + name: elasticsearch key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: TXN_OFFSET value: "0" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER value: elastic - name: SWAP_API_URL @@ -49,13 +49,16 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "vega" From 5c044329600dfb8388cad5b80b2434deaf1d4453 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Mon, 8 Jul 2024 12:29:30 +0800 Subject: [PATCH 72/82] [Deployment] Fixed some configs --- .../stc-scan/stcscan-barnard-deployment.yaml | 40 ++++++++++-------- .../stcscan-cmd-handle-main-deployment.yaml | 40 ++++++++++-------- .../stc-scan/stcscan-halley-deployment.yaml | 42 +++++++++++-------- .../stc-scan/stcscan-main-deployment.yaml | 40 ++++++++++-------- .../stc-scan/stcscan-proxima-deployment.yaml | 40 ++++++++++-------- .../stcscan-repair-barnard-deployment.yaml | 42 +++++++++++-------- .../stcscan-repair-halley-deployment.yaml | 40 ++++++++++-------- .../stcscan-repair-main-deployment.yaml | 40 ++++++++++-------- .../stc-scan/stcscan-txn-main-deployment.yaml | 40 ++++++++++-------- .../swap/swap-info-main-deployment.yaml | 28 ++++++++----- .../swap/swap-stat-main-deployment.yaml | 8 ++-- .../swap/swap-txns-main-deployment.yaml | 8 ++-- .../starscan-sub-barnard-deployment-ali.yaml | 26 +++++++----- .../starscan-sub-barnard-deployment.yaml | 30 +++++++------ .../starscan-sub-halley-deployment.yaml | 27 ++++++------ .../starscan-sub-main-deployment-ali.yaml | 26 +++++++----- .../starscan-sub-main-deployment.yaml | 15 ++++--- .../starscan-sub-proxima-deployment.yaml | 30 +++++++------ 18 files changed, 326 insertions(+), 236 deletions(-) diff --git a/kube/indexer/stc-scan/stcscan-barnard-deployment.yaml b/kube/indexer/stc-scan/stcscan-barnard-deployment.yaml index b666aba..756cd38 100644 --- a/kube/indexer/stc-scan/stcscan-barnard-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-barnard-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-deployment + name: starcoin-indexer-stcscan-deployment namespace: starcoin-barnard labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer + app: starcoin-indexer-stcscan template: metadata: labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: containers: - - name: starcoin-indexer + - name: starcoin-indexer-stcscan image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 @@ -31,31 +31,37 @@ spec: value: "0" - name: BULK_SIZE value: "80" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "barnard" diff --git a/kube/indexer/stc-scan/stcscan-cmd-handle-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-cmd-handle-main-deployment.yaml index 6ea7a43..c4824fb 100644 --- a/kube/indexer/stc-scan/stcscan-cmd-handle-main-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-cmd-handle-main-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-cmd-handle-deployment + name: starcoin-indexer-stcscan-cmd-handle-deployment namespace: starcoin-main labels: - app: starcoin-indexer-cmd-handle + app: starcoin-indexer-stcscan-cmd-handle spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer-cmd-handle + app: starcoin-indexer-stcscan-cmd-handle template: metadata: labels: - app: starcoin-indexer-cmd-handle + app: starcoin-indexer-stcscan-cmd-handle spec: containers: - - name: starcoin-indexer-cmd-handle + - name: starcoin-indexer-stcscan-cmd-handle image: starcoin/starcoin_indexer:sha-ba00b10 ports: - containerPort: 8300 @@ -31,31 +31,37 @@ spec: value: "0" - name: BULK_SIZE value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" diff --git a/kube/indexer/stc-scan/stcscan-halley-deployment.yaml b/kube/indexer/stc-scan/stcscan-halley-deployment.yaml index 904fab7..7a035d6 100644 --- a/kube/indexer/stc-scan/stcscan-halley-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-halley-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-deployment + name: starcoin-indexer-stcscan-deployment namespace: starcoin-halley labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer + app: starcoin-indexer-stcscan template: metadata: labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: containers: - - name: starcoin-indexer + - name: starcoin-indexer-stcscan image: starcoin/starcoin_indexer:halley_fix_db_1 ports: - containerPort: 8300 @@ -29,33 +29,39 @@ spec: value: "indexer,secondary,market_cap,txn_payload,txn_global_idx_update" - name: BULK_SIZE value: "80" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" - name: TXN_OFFSET value: "0" + - name: STARCOIN_ES_URL + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "halley" diff --git a/kube/indexer/stc-scan/stcscan-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-main-deployment.yaml index 5da5b23..c31d1ea 100644 --- a/kube/indexer/stc-scan/stcscan-main-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-main-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-deployment + name: starcoin-indexer-stcscan-deployment namespace: starcoin-main labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer + app: starcoin-indexer-stcscan template: metadata: labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: containers: - - name: starcoin-indexer + - name: starcoin-indexer-stcscan image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 @@ -31,31 +31,37 @@ spec: value: "0" - name: BULK_SIZE value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" diff --git a/kube/indexer/stc-scan/stcscan-proxima-deployment.yaml b/kube/indexer/stc-scan/stcscan-proxima-deployment.yaml index bf5911c..13efaf4 100644 --- a/kube/indexer/stc-scan/stcscan-proxima-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-proxima-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-deployment + name: starcoin-indexer-stcscan-deployment namespace: starcoin-proxima labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer + app: starcoin-indexer-stcscan template: metadata: labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: containers: - - name: starcoin-indexer + - name: starcoin-indexer-stcscan image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 @@ -29,33 +29,39 @@ spec: value: "indexer,secondary,market_cap,txn_payload" - name: BULK_SIZE value: "80" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: TXN_OFFSET value: "0" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "proxima" diff --git a/kube/indexer/stc-scan/stcscan-repair-barnard-deployment.yaml b/kube/indexer/stc-scan/stcscan-repair-barnard-deployment.yaml index cd2e587..ab7975c 100644 --- a/kube/indexer/stc-scan/stcscan-repair-barnard-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-repair-barnard-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-repair-deployment + name: starcoin-indexer-stcscan-repair-deployment namespace: starcoin-barnard labels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair template: metadata: labels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair spec: containers: - - name: starcoin-indexer-repair + - name: starcoin-indexer-stcscan-repair image: starcoin/starcoin_indexer:sha-6eb4945 ports: - containerPort: 8300 @@ -31,33 +31,39 @@ spec: value: "5000000" - name: BULK_SIZE value: "80" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "barnard" - name: PROGRAM_ARGS - value: "auto_repair 16057420" + value: "auto_repair 16057420" \ No newline at end of file diff --git a/kube/indexer/stc-scan/stcscan-repair-halley-deployment.yaml b/kube/indexer/stc-scan/stcscan-repair-halley-deployment.yaml index f00aa76..f60d911 100644 --- a/kube/indexer/stc-scan/stcscan-repair-halley-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-repair-halley-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-repair-deployment + name: starcoin-indexer-stcscan-repair-deployment namespace: starcoin-halley labels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair template: metadata: labels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair spec: containers: - - name: starcoin-indexer-repair + - name: starcoin-indexer-stcscan-repair image: starcoin/starcoin_indexer:halley_fix_db_1 ports: - containerPort: 8300 @@ -29,33 +29,39 @@ spec: value: "indexer,secondary,market_cap,txn_payload" - name: BULK_SIZE value: "80" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: TXN_OFFSET value: "0" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "halley" diff --git a/kube/indexer/stc-scan/stcscan-repair-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-repair-main-deployment.yaml index b687e50..dbc44a4 100644 --- a/kube/indexer/stc-scan/stcscan-repair-main-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-repair-main-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-repair-deployment + name: starcoin-indexer-stcscan-repair-deployment namespace: starcoin-main labels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair template: metadata: labels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair spec: containers: - - name: starcoin-indexer-repair + - name: starcoin-indexer-stcscan-repair image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 @@ -31,31 +31,37 @@ spec: value: "0" - name: BULK_SIZE value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" diff --git a/kube/indexer/stc-scan/stcscan-txn-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-txn-main-deployment.yaml index 48fd2ba..dd4bc9c 100644 --- a/kube/indexer/stc-scan/stcscan-txn-main-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-txn-main-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-txn-deployment + name: starcoin-indexer-stcscan-txn-deployment namespace: starcoin-main labels: - app: starcoin-indexer-txn + app: starcoin-indexer-stcscan-txn spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer-txn + app: starcoin-indexer-stcscan-txn template: metadata: labels: - app: starcoin-indexer-txn + app: starcoin-indexer-stcscan-txn spec: containers: - - name: starcoin-indexer-txn + - name: starcoin-indexer-stcscan-txn image: starcoin/starcoin_indexer:sha-ba00b10 ports: - containerPort: 8300 @@ -31,31 +31,37 @@ spec: value: "5217755" - name: BULK_SIZE value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" diff --git a/kube/indexer/swap/swap-info-main-deployment.yaml b/kube/indexer/swap/swap-info-main-deployment.yaml index 320d8f4..f1df85e 100644 --- a/kube/indexer/swap/swap-info-main-deployment.yaml +++ b/kube/indexer/swap/swap-info-main-deployment.yaml @@ -29,17 +29,22 @@ spec: value: "swap_stats,price_hour,price_stat,swap_pool_fee_stat" - name: TXN_OFFSET value: "0" + - name: STARCOIN_ES_URL + value: "elasticsearch.default.svc.cluster.local" + - name: STARCOIN_ES_PROTOCOL + value: "http" + - name: STARCOIN_ES_PORT + value: "9200" + - name: STARCOIN_ES_USER + valueFrom: + secretKeyRef: + name: elasticsearch + key: username - name: STARCOIN_ES_PWD valueFrom: secretKeyRef: - name: elasticsearch-pw-elastic + name: elasticsearch key: password - - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" - - name: STARCOIN_ES_PROTOCOL - value: "https" - - name: STARCOIN_ES_PORT - value: "443" - name: BULK_SIZE value: "100" - name: SWAP_API_URL @@ -47,13 +52,16 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" diff --git a/kube/indexer/swap/swap-stat-main-deployment.yaml b/kube/indexer/swap/swap-stat-main-deployment.yaml index 3d920b8..85f7a2a 100644 --- a/kube/indexer/swap/swap-stat-main-deployment.yaml +++ b/kube/indexer/swap/swap-stat-main-deployment.yaml @@ -37,11 +37,11 @@ spec: name: elasticsearch-pw-elastic key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER value: elastic - name: SWAP_API_URL @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/indexer/swap/swap-txns-main-deployment.yaml b/kube/indexer/swap/swap-txns-main-deployment.yaml index dde80d6..67c4346 100644 --- a/kube/indexer/swap/swap-txns-main-deployment.yaml +++ b/kube/indexer/swap/swap-txns-main-deployment.yaml @@ -37,11 +37,11 @@ spec: name: elasticsearch-pw-elastic key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER value: elastic - name: SWAP_API_URL @@ -49,7 +49,7 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME value: "starcoin" - name: DB_PWD diff --git a/kube/subscribe/starscan-sub-barnard-deployment-ali.yaml b/kube/subscribe/starscan-sub-barnard-deployment-ali.yaml index 9a91da8..7a9c1d9 100644 --- a/kube/subscribe/starscan-sub-barnard-deployment-ali.yaml +++ b/kube/subscribe/starscan-sub-barnard-deployment-ali.yaml @@ -27,31 +27,37 @@ spec: value: "barnard" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "123.57.40.36" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL value: "http" - name: STARCOIN_ES_PORT value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://123.57.40.36:8432/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-pw-starcoin + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" diff --git a/kube/subscribe/starscan-sub-barnard-deployment.yaml b/kube/subscribe/starscan-sub-barnard-deployment.yaml index 9625dd4..a68d943 100644 --- a/kube/subscribe/starscan-sub-barnard-deployment.yaml +++ b/kube/subscribe/starscan-sub-barnard-deployment.yaml @@ -27,31 +27,37 @@ spec: value: "barnard.0727" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" diff --git a/kube/subscribe/starscan-sub-halley-deployment.yaml b/kube/subscribe/starscan-sub-halley-deployment.yaml index 672ef7f..8ad3221 100644 --- a/kube/subscribe/starscan-sub-halley-deployment.yaml +++ b/kube/subscribe/starscan-sub-halley-deployment.yaml @@ -27,36 +27,37 @@ spec: value: "halley.0727" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username - name: STARCOIN_ES_PWD valueFrom: secretKeyRef: - name: elasticsearch-pw-elastic + name: elasticsearch key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" diff --git a/kube/subscribe/starscan-sub-main-deployment-ali.yaml b/kube/subscribe/starscan-sub-main-deployment-ali.yaml index b6c83ba..b74044f 100644 --- a/kube/subscribe/starscan-sub-main-deployment-ali.yaml +++ b/kube/subscribe/starscan-sub-main-deployment-ali.yaml @@ -27,31 +27,37 @@ spec: value: "main" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "123.57.40.36" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL value: "http" - name: STARCOIN_ES_PORT value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://123.57.40.36:8432/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-pw-starcoin + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" diff --git a/kube/subscribe/starscan-sub-main-deployment.yaml b/kube/subscribe/starscan-sub-main-deployment.yaml index 6745106..0a7c7b2 100644 --- a/kube/subscribe/starscan-sub-main-deployment.yaml +++ b/kube/subscribe/starscan-sub-main-deployment.yaml @@ -33,11 +33,11 @@ spec: name: elasticsearch-pw-elastic key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER value: elastic - name: SWAP_API_URL @@ -45,13 +45,16 @@ spec: - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" diff --git a/kube/subscribe/starscan-sub-proxima-deployment.yaml b/kube/subscribe/starscan-sub-proxima-deployment.yaml index e8e71f5..b060b60 100644 --- a/kube/subscribe/starscan-sub-proxima-deployment.yaml +++ b/kube/subscribe/starscan-sub-proxima-deployment.yaml @@ -27,31 +27,37 @@ spec: value: "proxima.0727" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://scan-db.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" From 82b0dbd1a7ea529731a8bf2f615378c9a2fd89d3 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Wed, 10 Jul 2024 15:46:08 +0800 Subject: [PATCH 73/82] [Deployment] Set elasticsearch pv class to do-block-storage-retain --- .../elasticsearch-deployment.yaml | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/kube/base-components/elasticsearch-deployment.yaml b/kube/base-components/elasticsearch-deployment.yaml index 25bd918..738126f 100644 --- a/kube/base-components/elasticsearch-deployment.yaml +++ b/kube/base-components/elasticsearch-deployment.yaml @@ -58,7 +58,7 @@ spec: - name: es-data persistentVolumeClaim: claimName: es-pvc - - name: elasticsearch-,.ujv config + - name: elasticsearch-config configMap: name: elasticsearch-config --- @@ -72,19 +72,7 @@ data: xpack.security.enabled: true xpack.license.self_generated.type: basic network.host: 0.0.0.0 ---- -# Elasticsearch Persistent Volume -apiVersion: v1 -kind: PersistentVolume -metadata: - name: es-pv -spec: - capacity: - storage: 80Gi - accessModes: - - ReadWriteOnce - hostPath: - path: /mnt/data/es + --- # Elasticsearch Persistent Volume Claim apiVersion: v1 @@ -96,7 +84,9 @@ spec: - ReadWriteOnce resources: requests: - storage: 80Gi + storage: 100Gi + storageClassName: do-block-storage-retain + --- # Elasticsearch Service apiVersion: v1 From feb95553945c704a26fbd3f74325fcfc50ed5f74 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Fri, 12 Jul 2024 23:40:16 +0800 Subject: [PATCH 74/82] [Deployment] Path namespace and network configuration for other Nets --- .../allowaccess-network-policy.yaml | 104 ++++++++++++++++++ kube/base-components/namespaces.yaml | 55 +++++++++ .../stc-scan/stcscan-main-deployment.yaml | 2 +- kube/scan-api/scan-api-deployment.yaml | 2 +- 4 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 kube/base-components/namespaces.yaml diff --git a/kube/base-components/allowaccess-network-policy.yaml b/kube/base-components/allowaccess-network-policy.yaml index dd4c0f2..9bfdd10 100644 --- a/kube/base-components/allowaccess-network-policy.yaml +++ b/kube/base-components/allowaccess-network-policy.yaml @@ -32,6 +32,110 @@ spec: matchLabels: name: starcoin-vega +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-pg-from-main + namespace: default +spec: + podSelector: + matchLabels: + app: postgres-service + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-main +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-es-from-main + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-main + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-pg-from-barnard + namespace: default +spec: + podSelector: + matchLabels: + app: postgres-service + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-barnard +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-es-from-barnard + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-barnard + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-pg-from-proxima + namespace: default +spec: + podSelector: + matchLabels: + app: postgres-service + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-proxima +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-es-from-proxima + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-proxima --- # Postgres service for starcoin-api diff --git a/kube/base-components/namespaces.yaml b/kube/base-components/namespaces.yaml new file mode 100644 index 0000000..fa61572 --- /dev/null +++ b/kube/base-components/namespaces.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: starcoin-main + labels: + app: starcoin + kubernetes.io/metadata.name: starcoin-main + name: starcoin-main + network: main + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: starcoin-barnard + labels: + app: starcoin + kubernetes.io/metadata.name: starcoin-barnard + name: starcoin-barnard + network: barnard + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: starcoin-proxima + labels: + app: starcoin + kubernetes.io/metadata.name: starcoin-proxima + name: starcoin-proxima + network: proxima + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: starcoin-halley + labels: + app: starcoin + kubernetes.io/metadata.name: starcoin-halley + name: starcoin-halley + network: halley + + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: starcoin-vega + labels: + app: starcoin + kubernetes.io/metadata.name: starcoin-vega + name: starcoin-vega + network: vega \ No newline at end of file diff --git a/kube/indexer/stc-scan/stcscan-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-main-deployment.yaml index c31d1ea..229eaab 100644 --- a/kube/indexer/stc-scan/stcscan-main-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-main-deployment.yaml @@ -26,7 +26,7 @@ spec: - name: NETWORK value: "main.0727" - name: BG_TASK_JOBS - value: "indexer,secondary,market_cap,txn_payload" + value: "indexer,secondary,market_cap,txn_payload,txn_global_idx_update" - name: TXN_OFFSET value: "0" - name: BULK_SIZE diff --git a/kube/scan-api/scan-api-deployment.yaml b/kube/scan-api/scan-api-deployment.yaml index 950589e..0659cbe 100644 --- a/kube/scan-api/scan-api-deployment.yaml +++ b/kube/scan-api/scan-api-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: scan-api - image: starcoin/starcoin_scan:sha-1223fc3 + image: starcoin/starcoin_scan:scan_api_dag_2 ports: - name: http containerPort: 8500 From 4ba90d0fde70993b09e6c8d52e2140aea4c42939 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Wed, 14 Aug 2024 12:05:51 +0800 Subject: [PATCH 75/82] [Deployment] for Digital ocean --- kube/indexer/dag/dag-vega-deployment.yaml | 2 +- kube/indexer/stc-scan/stcscan-main-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kube/indexer/dag/dag-vega-deployment.yaml b/kube/indexer/dag/dag-vega-deployment.yaml index e74a184..2bbabc1 100644 --- a/kube/indexer/dag/dag-vega-deployment.yaml +++ b/kube/indexer/dag/dag-vega-deployment.yaml @@ -28,7 +28,7 @@ spec: - name: BG_TASK_JOBS value: "dag_inspector" - name: BULK_SIZE - value: "10" + value: "500" - name: STARCOIN_ES_USER value: elastic - name: STARCOIN_ES_PWD diff --git a/kube/indexer/stc-scan/stcscan-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-main-deployment.yaml index 229eaab..0daa112 100644 --- a/kube/indexer/stc-scan/stcscan-main-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-main-deployment.yaml @@ -30,7 +30,7 @@ spec: - name: TXN_OFFSET value: "0" - name: BULK_SIZE - value: "100" + value: "1000" - name: STARCOIN_ES_URL value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL From d8bbc67e24700948bc11bcd2c1ad20b27d42bc70 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Wed, 28 Aug 2024 18:44:51 +0800 Subject: [PATCH 76/82] [test] 1. add index test; 2. change index job into Handler class --- .../starcoin/indexer/config/QuartzConfig.java | 3 +- .../indexer/handler/IndexerHandleJob.java | 181 ++++++++++++++++++ ...andle.java => LegacyMainIndexHandler.java} | 55 +++--- .../indexer/handler/MarketCapIndexer.java | 2 + .../indexer/test/IndexHandlerJobTest.java | 31 +++ 5 files changed, 246 insertions(+), 26 deletions(-) create mode 100644 starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandleJob.java rename starcoin-indexer/src/main/java/org/starcoin/indexer/handler/{IndexerHandle.java => LegacyMainIndexHandler.java} (88%) create mode 100644 starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexHandlerJobTest.java diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java index 8ca2b3d..9048169 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java @@ -5,7 +5,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; import org.springframework.scheduling.quartz.SchedulerFactoryBean; import org.starcoin.config.SearchJobFactory; import org.starcoin.indexer.handler.*; @@ -25,7 +24,7 @@ public class QuartzConfig { @Bean public JobDetail indexerJob() { - return JobBuilder.newJob(IndexerHandle.class).withIdentity("indexer").storeDurably().build(); + return JobBuilder.newJob(IndexerHandleJob.class).withIdentity("indexer").storeDurably().build(); } @Bean diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandleJob.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandleJob.java new file mode 100644 index 0000000..dfb33cd --- /dev/null +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandleJob.java @@ -0,0 +1,181 @@ +package org.starcoin.indexer.handler; +import org.quartz.JobExecutionContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.quartz.QuartzJobBean; +import org.starcoin.api.BlockRPCClient; +import org.starcoin.api.TransactionRPCClient; + +import javax.annotation.PostConstruct; + + +public class IndexerHandleJob extends QuartzJobBean { + // private static final Logger logger = LoggerFactory.getLogger(IndexerHandleJob.class); + + @Value("${starcoin.indexer.bulk_size}") + private long bulkSize; + + @Autowired + private ElasticSearchHandler elasticSearchHandler; + + @Autowired + private TransactionRPCClient transactionRPCClient; + + @Autowired + private BlockRPCClient blockRPCClient; + + private LegacyMainIndexHandler legacyIndexHandler; + +// @PostConstruct +// public void initOffset() { +// localBlockOffset = elasticSearchHandler.getRemoteOffset(); +// //update current handle header +// try { +// if (localBlockOffset != null) { +// Block block = blockRPCClient.getBlockByHeight(localBlockOffset.getBlockHeight()); +// if (block != null) { +// currentHandleHeader = block.getHeader(); +// } else { +// logger.error("init offset block not exist on chain: {}", localBlockOffset); +// } +// +// } else { +// logger.warn("offset is null,init reset to genesis"); +// currentHandleHeader = blockRPCClient.getBlockByHeight(0).getHeader(); +// localBlockOffset = new BlockOffset(0, currentHandleHeader.getBlockHash()); +// elasticSearchHandler.setRemoteOffset(localBlockOffset); +// logger.info("init offset ok: {}", localBlockOffset); +// } +// } catch (JSONRPC2SessionException e) { +// logger.error("set current header error:", e); +// } +// } +// +// @Override +// protected void executeInternal(JobExecutionContext jobExecutionContext) { +// //read current offset +// if (localBlockOffset == null || currentHandleHeader == null) { +//// logger.warn("local offset error, reset it: {}, {}", localOffset, currentHandleHeader); +// initOffset(); +// } +// BlockOffset remoteBlockOffset = elasticSearchHandler.getRemoteOffset(); +// logger.info("current remote offset: {}", remoteBlockOffset); +// if (remoteBlockOffset == null) { +// logger.warn("offset must not null, please check blocks.mapping!!"); +// return; +// } +// if (remoteBlockOffset.getBlockHeight() > localBlockOffset.getBlockHeight()) { +// logger.info("indexer equalize chain blocks."); +// return; +// } +// //read head +// try { +// BlockHeader chainHeader = blockRPCClient.getChainHeader(); +// //calculate bulk size +// long headHeight = chainHeader.getHeight(); +// long bulkNumber = Math.min(headHeight - localBlockOffset.getBlockHeight(), bulkSize); +// int index = 1; +// List blockList = new ArrayList<>(); +// while (index <= bulkNumber) { +// long readNumber = localBlockOffset.getBlockHeight() + index; +// Block block = blockRPCClient.getBlockByHeight(readNumber); +// if (!block.getHeader().getParentHash().equals(currentHandleHeader.getBlockHash())) { +// //fork handle until reach forked point block +// logger.warn("Fork detected, roll back: {}, {}, {}", readNumber, block.getHeader().getParentHash(), currentHandleHeader.getBlockHash()); +// Block lastForkBlock, lastMasterBlock; +// BlockHeader forkHeader = currentHandleHeader; +// long lastMasterNumber = readNumber - 1; +// String forkHeaderParentHash = null; +// do { +// //获取分叉的block +// if (forkHeaderParentHash == null) { +// //第一次先回滚当前最高的分叉块 +// forkHeaderParentHash = forkHeader.getBlockHash(); +// } else { +// forkHeaderParentHash = forkHeader.getParentHash(); +// } +// lastForkBlock = elasticSearchHandler.getBlockContent(forkHeaderParentHash); +// if (lastForkBlock == null) { +// logger.warn("get fork block null: {}", forkHeaderParentHash); +// //read from node +// lastForkBlock = blockRPCClient.getBlockByHash(forkHeaderParentHash); +// } +// if (lastForkBlock != null) { +// elasticSearchHandler.bulkForkedUpdate(lastForkBlock); +// logger.info("rollback forked block ok: {}, {}", lastForkBlock.getHeader().getHeight(), forkHeaderParentHash); +// } else { +// //如果块取不到，先退出当前任务，下一个轮询周期再执行 +// logger.warn("get forked block is null: {}", forkHeaderParentHash); +// return; +// } +// +// //获取上一个高度主块 +// lastMasterBlock = blockRPCClient.getBlockByHeight(lastMasterNumber); +// if (lastMasterBlock != null) { +// long forkNumber = forkHeader.getHeight(); +// logger.info("fork number: {}", forkNumber); +// forkHeader = lastForkBlock.getHeader(); +// //reset offset to handled fork block +// currentHandleHeader = forkHeader; +// localBlockOffset.setBlockHeight(currentHandleHeader.getHeight()); +// localBlockOffset.setBlockHash(currentHandleHeader.getBlockHash()); +// elasticSearchHandler.setRemoteOffset(localBlockOffset); +// if (lastMasterNumber == forkNumber && lastMasterBlock.getHeader().getBlockHash().equals(forkHeaderParentHash)) { +// //find fork point +// logger.info("find fork height: {}", lastMasterNumber); +// break; +// } +// //继续找下一个分叉 +// lastMasterNumber--; +// logger.info("continue last forked block: {}", lastMasterNumber); +// } else { +// logger.warn("get last master Block null: {}", lastMasterNumber); +// } +// } while (true); +// +// logger.info("rollback handle ok: {}", localBlockOffset); +// return; //退出当前任务，重新添加从分叉点之后的block +// } +// +// //set event +// ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block); +// blockList.add(block); +// +// //update current header +// currentHandleHeader = block.getHeader(); +// index++; +// logger.debug("add block: {}", block.getHeader()); +// } +// //bulk execute +// elasticSearchHandler.bulk(blockList); +// +// //update offset +// localBlockOffset.setBlockHeight(currentHandleHeader.getHeight()); +// localBlockOffset.setBlockHash(currentHandleHeader.getBlockHash()); +// elasticSearchHandler.setRemoteOffset(localBlockOffset); +// logger.info("indexer update success: {}", localBlockOffset); +// } catch (JSONRPC2SessionException e) { +// logger.error("chain header error:", e); +// } +// } + + @PostConstruct + public void initOffset() { + if (legacyIndexHandler == null) { + legacyIndexHandler = new LegacyMainIndexHandler( + elasticSearchHandler, + transactionRPCClient, + blockRPCClient, + bulkSize + ); + legacyIndexHandler.initOffset(); + } + } + + @Override + protected void executeInternal(JobExecutionContext jobExecutionContext) { + legacyIndexHandler.execute(); + } +} \ No newline at end of file diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/LegacyMainIndexHandler.java similarity index 88% rename from starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java rename to starcoin-indexer/src/main/java/org/starcoin/indexer/handler/LegacyMainIndexHandler.java index 98451c2..f70b8e5 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/LegacyMainIndexHandler.java @@ -1,47 +1,54 @@ package org.starcoin.indexer.handler; -import com.fasterxml.jackson.core.JsonProcessingException; -import org.quartz.JobExecutionContext; + +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.quartz.QuartzJobBean; import org.starcoin.api.BlockRPCClient; import org.starcoin.api.TransactionRPCClient; import org.starcoin.bean.Block; import org.starcoin.bean.BlockHeader; import org.starcoin.bean.BlockOffset; -import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; -import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.List; - -public class IndexerHandle extends QuartzJobBean { - private static final Logger logger = LoggerFactory.getLogger(IndexerHandle.class); +public class LegacyMainIndexHandler { + private static final Logger logger = LoggerFactory.getLogger(LegacyMainIndexHandler.class); private BlockOffset localBlockOffset; - private BlockHeader currentHandleHeader; - - @Value("${starcoin.network}") - private String network; - @Value("${starcoin.indexer.bulk_size}") - private long bulkSize; + private BlockHeader currentHandleHeader; - @Autowired private ElasticSearchHandler elasticSearchHandler; - - @Autowired private TransactionRPCClient transactionRPCClient; - @Autowired private BlockRPCClient blockRPCClient; + private Long bulkSize; + + public LegacyMainIndexHandler( + ElasticSearchHandler elasticSearchHandler, + TransactionRPCClient transactionRPCClient, + BlockRPCClient blockRPCClient, + Long bulkSize + ) { + this.elasticSearchHandler = elasticSearchHandler; + this.transactionRPCClient = transactionRPCClient; + this.blockRPCClient = blockRPCClient; + this.bulkSize = bulkSize; + } + + public void initOffsetWith(Long height, String blockHash) throws JSONRPC2SessionException { + localBlockOffset = new BlockOffset(height, blockHash); + Block block = blockRPCClient.getBlockByHeight(height); + if (block != null) { + currentHandleHeader = block.getHeader(); + } else { + logger.error("init offset block not exist on chain: {}", localBlockOffset); + } + } - @PostConstruct public void initOffset() { localBlockOffset = elasticSearchHandler.getRemoteOffset(); //update current handle header @@ -66,8 +73,8 @@ public void initOffset() { } } - @Override - protected void executeInternal(JobExecutionContext jobExecutionContext) { + + public void execute() { //read current offset if (localBlockOffset == null || currentHandleHeader == null) { // logger.warn("local offset error, reset it: {}, {}", localOffset, currentHandleHeader); @@ -173,4 +180,4 @@ protected void executeInternal(JobExecutionContext jobExecutionContext) { logger.error("chain header error:", e); } } -} \ No newline at end of file +} diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapIndexer.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapIndexer.java index ab5a4cd..a706927 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapIndexer.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapIndexer.java @@ -13,8 +13,10 @@ public class MarketCapIndexer extends QuartzJobBean { private static final Logger logger = LoggerFactory.getLogger(MarketCapIndexer.class); + @Autowired private MarketCapHandle handle; + @Autowired private AddressHolderService addressHolderService; diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexHandlerJobTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexHandlerJobTest.java new file mode 100644 index 0000000..228d85b --- /dev/null +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexHandlerJobTest.java @@ -0,0 +1,31 @@ +package org.starcoin.indexer.test; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.starcoin.api.BlockRPCClient; +import org.starcoin.api.TransactionRPCClient; +import org.starcoin.indexer.handler.ElasticSearchHandler; +import org.starcoin.indexer.handler.LegacyMainIndexHandler; + +public class IndexHandlerJobTest extends IndexerLogicBaseTest { + + @Value("${starcoin.indexer.bulk_size}") + private long bulkSize; + + @Autowired + private ElasticSearchHandler elasticSearchHandler; + + @Autowired + private TransactionRPCClient transactionRPCClient; + + @Autowired + private BlockRPCClient blockRPCClient; + + @Test + public void testIndexerHandle() throws Exception { + LegacyMainIndexHandler legacyMainIndexHandler = new LegacyMainIndexHandler(elasticSearchHandler, transactionRPCClient, blockRPCClient, bulkSize); + legacyMainIndexHandler.initOffsetWith(2713240L, "0x4d58d276809bd061ba422a4699c90c790efc5dd1b6d40e8c2adb0b1cb98dfafd"); + legacyMainIndexHandler.execute(); + } +} From b20fb4dc20b734a18f3d36888d6b45105c233dc9 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Thu, 19 Sep 2024 10:32:19 +0800 Subject: [PATCH 77/82] [digital ocean] Add backup pvc --- .../elasticsearch-deployment.yaml | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/kube/base-components/elasticsearch-deployment.yaml b/kube/base-components/elasticsearch-deployment.yaml index 738126f..ff4b653 100644 --- a/kube/base-components/elasticsearch-deployment.yaml +++ b/kube/base-components/elasticsearch-deployment.yaml @@ -38,6 +38,8 @@ spec: volumeMounts: - name: es-data mountPath: /usr/share/elasticsearch/data + - name: es-backup + mountPath: /data/es_snapshot_repository - name: elasticsearch-config mountPath: /usr/share/elasticsearch/config/elasticsearch.yml subPath: elasticsearch.yml @@ -58,6 +60,9 @@ spec: - name: es-data persistentVolumeClaim: claimName: es-pvc + - name: es-backup + persistentVolumeClaim: + claimName: es-backup-pvc - name: elasticsearch-config configMap: name: elasticsearch-config @@ -72,6 +77,7 @@ data: xpack.security.enabled: true xpack.license.self_generated.type: basic network.host: 0.0.0.0 + path.repo: ["/data/es_snapshot_repository"] --- # Elasticsearch Persistent Volume Claim @@ -84,7 +90,21 @@ spec: - ReadWriteOnce resources: requests: - storage: 100Gi + storage: 500Gi + storageClassName: do-block-storage-retain + +--- +# Elasticsearch Snapshot Repository Persistent Volume Claim +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: es-backup-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500Gi storageClassName: do-block-storage-retain --- From 258f43ca450d26c0d2803e2aa693b23bbb3d40ad Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Thu, 19 Sep 2024 11:48:42 +0800 Subject: [PATCH 78/82] [digital ocean] Add privileged for elasticsearch-deployment.yaml --- kube/base-components/elasticsearch-deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kube/base-components/elasticsearch-deployment.yaml b/kube/base-components/elasticsearch-deployment.yaml index ff4b653..1c5a8be 100644 --- a/kube/base-components/elasticsearch-deployment.yaml +++ b/kube/base-components/elasticsearch-deployment.yaml @@ -22,6 +22,8 @@ spec: mountPath: /usr/share/elasticsearch/data containers: - name: elasticsearch + securityContext: + privileged: true image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 resources: requests: From 9a804b6f3b301b7b0cf2cbed78439fe79c52c792 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Thu, 19 Sep 2024 19:51:10 +0800 Subject: [PATCH 79/82] [digital ocean] Add some config for elasticsearch-deployment.yaml --- .../elasticsearch-deployment.yaml | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/kube/base-components/elasticsearch-deployment.yaml b/kube/base-components/elasticsearch-deployment.yaml index 1c5a8be..9539e05 100644 --- a/kube/base-components/elasticsearch-deployment.yaml +++ b/kube/base-components/elasticsearch-deployment.yaml @@ -45,6 +45,8 @@ spec: - name: elasticsearch-config mountPath: /usr/share/elasticsearch/config/elasticsearch.yml subPath: elasticsearch.yml +# - name: s3-mount-point +# mountPath: /mnt/s3_mount env: - name: discovery.type value: single-node @@ -58,6 +60,29 @@ spec: secretKeyRef: name: elasticsearch key: password + # S3 配置 + - name: S3_CLIENT_DEFAULT_REGION + value: "ap-northeast-1" + - name: S3_CLIENT_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-credentials + key: access-key + - name: S3_CLIENT_SECRET_KEY + valueFrom: + secretKeyRef: + name: aws-credentials + key: secret-key + lifecycle: + postStart: + exec: + command: + - "sh" + - "-c" + - | + echo ${S3_CLIENT_ACCESS_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add s3.client.default.access_key --stdin + echo ${S3_CLIENT_SECRET_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add s3.client.default.secret_key --stdin + /usr/share/elasticsearch/bin/elasticsearch-plugin list | grep -q repository-s3 || /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch repository-s3 volumes: - name: es-data persistentVolumeClaim: @@ -68,6 +93,9 @@ spec: - name: elasticsearch-config configMap: name: elasticsearch-config +# - name: s3-mount-point +# emptyDir: {} + --- # Elasticsearch Configuration apiVersion: v1 @@ -80,7 +108,11 @@ data: xpack.license.self_generated.type: basic network.host: 0.0.0.0 path.repo: ["/data/es_snapshot_repository"] - + s3.client.default.endpoint: "s3.ap-northeast-1.amazonaws.com" + s3.client.default.protocol: https + s3.client.default.read_timeout: 50s + s3.client.default.max_retries: 3 + s3.client.default.use_throttle_retries: true --- # Elasticsearch Persistent Volume Claim apiVersion: v1 From b1633d2681ea9c1b7805f2b8ad85fd67b55102af Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Thu, 26 Sep 2024 15:20:49 +0800 Subject: [PATCH 80/82] [digital ocean] Add some config for elasticsearch-deployment.yaml --- kube/base-components/elasticsearch-deployment.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/kube/base-components/elasticsearch-deployment.yaml b/kube/base-components/elasticsearch-deployment.yaml index 9539e05..b4bb176 100644 --- a/kube/base-components/elasticsearch-deployment.yaml +++ b/kube/base-components/elasticsearch-deployment.yaml @@ -76,13 +76,9 @@ spec: lifecycle: postStart: exec: - command: - - "sh" - - "-c" - - | - echo ${S3_CLIENT_ACCESS_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add s3.client.default.access_key --stdin - echo ${S3_CLIENT_SECRET_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add s3.client.default.secret_key --stdin - /usr/share/elasticsearch/bin/elasticsearch-plugin list | grep -q repository-s3 || /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch repository-s3 + command: ["/bin/bash", "-c", "/usr/share/elasticsearch/bin/elasticsearch-plugin list | grep -q repository-s3 || /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch repository-s3 && \ + echo ${S3_CLIENT_ACCESS_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add s3.client.default.access_key --stdin &&\ + echo ${S3_CLIENT_SECRET_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add s3.client.default.secret_key --stdin"] volumes: - name: es-data persistentVolumeClaim: From 942bf878e1f0c8afc3e077bfb1df3450c7cc0cd9 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Thu, 26 Sep 2024 19:56:43 +0800 Subject: [PATCH 81/82] [digital ocean] Fixed swap indexer configuration --- .../swap/swap-stat-main-deployment.yaml | 22 ++++++++++++------- .../swap/swap-txns-main-deployment.yaml | 7 ++++-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/kube/indexer/swap/swap-stat-main-deployment.yaml b/kube/indexer/swap/swap-stat-main-deployment.yaml index 85f7a2a..89913c3 100644 --- a/kube/indexer/swap/swap-stat-main-deployment.yaml +++ b/kube/indexer/swap/swap-stat-main-deployment.yaml @@ -31,11 +31,6 @@ spec: value: "0" - name: BULK_SIZE value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL @@ -43,7 +38,15 @@ spec: - name: STARCOIN_ES_PORT value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR @@ -51,11 +54,14 @@ spec: - name: DS_URL value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" diff --git a/kube/indexer/swap/swap-txns-main-deployment.yaml b/kube/indexer/swap/swap-txns-main-deployment.yaml index 67c4346..6b29008 100644 --- a/kube/indexer/swap/swap-txns-main-deployment.yaml +++ b/kube/indexer/swap/swap-txns-main-deployment.yaml @@ -51,11 +51,14 @@ spec: - name: DS_URL value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" From 0df6e41113818a87c51d1b993e663a11b1504318 Mon Sep 17 00:00:00 2001 From: Bob Ong Date: Sat, 5 Oct 2024 18:35:03 +0800 Subject: [PATCH 82/82] [digital ocean] Fixed swap indexer configuration --- kube/indexer/swap/swap-txns-main-deployment.yaml | 15 +++++++++------ kube/subscribe/starscan-sub-main-deployment.yaml | 15 +++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/kube/indexer/swap/swap-txns-main-deployment.yaml b/kube/indexer/swap/swap-txns-main-deployment.yaml index 6b29008..14af86a 100644 --- a/kube/indexer/swap/swap-txns-main-deployment.yaml +++ b/kube/indexer/swap/swap-txns-main-deployment.yaml @@ -31,11 +31,6 @@ spec: value: "0" - name: BULK_SIZE value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL @@ -43,7 +38,15 @@ spec: - name: STARCOIN_ES_PORT value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR diff --git a/kube/subscribe/starscan-sub-main-deployment.yaml b/kube/subscribe/starscan-sub-main-deployment.yaml index 0a7c7b2..fed8735 100644 --- a/kube/subscribe/starscan-sub-main-deployment.yaml +++ b/kube/subscribe/starscan-sub-main-deployment.yaml @@ -27,11 +27,6 @@ spec: value: "main.0727" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL @@ -39,7 +34,15 @@ spec: - name: STARCOIN_ES_PORT value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR