diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3ebf37d..09776f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,11 +20,6 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 - - name: maven-settings-xml-action - uses: whelk-io/maven-settings-xml-action@v18 - with: - repositories: '[{ "id": "github", "url": "https://maven.pkg.github.com/starcoinorg/*" }]' - servers: '[{ "id": "github", "username": "${{ github.actor }}", "password": "${{ secrets.GIT_PACKAGE_TOKEN }}" }]' - name: Cache Maven packages uses: actions/cache@v1 with: diff --git a/.github/workflows/docker_build_indexer.yml b/.github/workflows/docker_build_indexer.yml index 45a7822..032f5f9 100644 --- a/.github/workflows/docker_build_indexer.yml +++ b/.github/workflows/docker_build_indexer.yml @@ -22,7 +22,7 @@ jobs: id: docker_meta uses: crazy-max/ghaction-docker-meta@v1 with: - images: starcoin/starcoin_indexer,ghcr.io/starcoinorg/starcoin_indexer + images: starcoin/starcoin_indexer tag-sha: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1.6.0 @@ -33,12 +33,6 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GIT_PACKAGE_TOKEN }} - name: maven-settings-xml-action uses: whelk-io/maven-settings-xml-action@v18 with: diff --git a/.github/workflows/docker_build_scanapi.yml b/.github/workflows/docker_build_scanapi.yml index 23012e6..0c8c258 100644 --- a/.github/workflows/docker_build_scanapi.yml +++ b/.github/workflows/docker_build_scanapi.yml @@ -22,7 +22,7 @@ jobs: id: docker_meta uses: crazy-max/ghaction-docker-meta@v1 with: - images: starcoin/starcoin_scan,ghcr.io/starcoinorg/starcoin_scan + images: starcoin/starcoin_scan tag-sha: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1.6.0 diff --git a/README.md b/README.md index 2f49545..f22b4a3 100644 --- a/README.md +++ b/README.md @@ -1 +1,3 @@ -# starcoin-search +# stc-scan +Established some independent handles for indexing data with scan to customize statistics data for swap, used in conjunction with the [swap-stat-api](https://github.com/Elements-Studio/swap-stat-api) repository. + diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 0000000..527d1b5 --- /dev/null +++ b/docker-compose/README.md @@ -0,0 +1,63 @@ +# How to build local environment + +## 1. Install "Docker" and "Docker Compose" +## 2. Run the command to start the database and components +```bash +docker-compose up +``` +## 3. Start starcoin-index project + +### Config the startup environment variable +```dotenv +HOSTS=localhost +NETWORK=halley # select which network to scan +BG_TASK_JOBS=dag_inspector +TXN_OFFSET=0 +BULK_SIZE=100 +STARCOIN_ES_PWD= +STARCOIN_ES_URL=localhost +STARCOIN_ES_PROTOCOL=http +STARCOIN_ES_PORT=9200 +STARCOIN_ES_USER= +SWAP_API_URL=https://swap-api.starswap.xyz +SWAP_CONTRACT_ADDR=0x8c109349c6bd91411d6bc962e080c4a3 +DS_URL=jdbc:postgresql://localhost/starcoin +DB_SCHEMA=halley +DB_USER_NAME=starcoin +DB_PWD=starcoin +PROGRAM_ARGS= +# auto_repair 9411700 +``` + +### Configuration Elasticsearch template +[IMPORTANT!!] Make sure your template has added to Elastic search service before add data, including component template and index template to ES. +Following file: [[es_pipeline.scripts](..%2Fkube%2Fmappings%2Fes_pipeline.scripts)] + +1. Open the 'Kibana' site has been started in the docker-compose environment, usually the url is http://localhost:5601 +2. Navigate to 'Dev Tools' +3. Follow the instructions in the file of giving above to add the template to ES + +### Add SQL tables for network +[IMPORTANT!!] Add the [tables](../starcoin-indexer/deploy/create_table.sql) for the network you want to scan, including main, barnard, halley, etc. + +## 4. Start starcoin-scan-api project + +### Config the startup enviroment variable +```dotenv +STARCOIN_ES_URL=localhost +STARCOIN_ES_PROTOCOL=http +STARCOIN_ES_PORT=9200 +STARCOIN_ES_USER= +STARCOIN_ES_INDEX_VERSION= +STARCOIN_ES_PWD= +MAIN_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=main +BARNARD_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=barnard +HALLEY_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=halley +DS_URL=jdbc:postgresql://localhost/starcoin +STARCOIN_USER_DS_URL="jdbc:postgresql://localhost/starcoin?currentSchema=starcoin_user" +DB_USER_NAME=starcoin +DB_PWD=starcoin +``` + +### Add SQL tables for network +[IMPORTANT!!] Add the [tables](../starcoin-scan-api/deploy/create_table.sql) for the network you want to scan, including main, barnard, halley, etc. diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml new file mode 100644 index 0000000..160a848 --- /dev/null +++ b/docker-compose/docker-compose.yaml @@ -0,0 +1,67 @@ +# This composer file is used to configure the local environment for debugging +version: '3.8' + +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.2 + container_name: elasticsearch + environment: + - node.name=elasticsearch + - cluster.name=docker-cluster + - discovery.type=single-node + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata:/usr/share/elasticsearch/data + ports: + - "9200:9200" + + hazelcast: + image: hazelcast/hazelcast:latest + container_name: hazelcast + ports: + - "5701:5701" + environment: + - HZ_CLUSTERNAME=stcscan-hazelcast-cluster + - HZ_NETWORK_JOIN_MULTICAST_ENABLED=false + - HZ_NETWORK_JOIN_TCPIP_ENABLED=true + - HZ_NETWORK_JOIN_TCPIP_MEMBERS=hazelcast + - HZ_CACHE_CODE_CACHE_STATISTICS_ENABLED=true + - HZ_CACHE_CODE_CACHE_EVICTION_SIZE=10000 + - HZ_CACHE_CODE_CACHE_EVICTION_MAX_SIZE_POLICY=ENTRY_COUNT + - HZ_CACHE_CODE_CACHE_EVICTION_EVICTION_POLICY=LFU + - HZ_CACHE_SESSION_STATISTICS_ENABLED=true + - HZ_CACHE_SESSION_EVICTION_SIZE=50000 + - HZ_CACHE_SESSION_EVICTION_MAX_SIZE_POLICY=ENTRY_COUNT + - HZ_CACHE_SESSION_EVICTION_EVICTION_POLICY=LRU + + kibana: + image: docker.elastic.co/kibana/kibana:7.17.2 + container_name: kibana + environment: + ELASTICSEARCH_URL: http://elasticsearch:9200 + ELASTICSEARCH_HOSTS: http://elasticsearch:9200 + ports: + - 5601:5601 + depends_on: + - elasticsearch + + postgresql: + image: postgres:13.2 + container_name: postgres_db + environment: + POSTGRES_USER: starcoin + POSTGRES_PASSWORD: starcoin + POSTGRES_DB: starcoin + volumes: + - db_data:/var/lib/postgresql/data + ports: + - 5432:5432 + +volumes: + esdata: + driver: local diff --git a/kube/base-components/allowaccess-network-policy.yaml b/kube/base-components/allowaccess-network-policy.yaml new file mode 100644 index 0000000..9bfdd10 --- /dev/null +++ b/kube/base-components/allowaccess-network-policy.yaml @@ -0,0 +1,196 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-pg-from-vega + namespace: default +spec: + podSelector: + matchLabels: + app: postgres-service + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-vega +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-es-from-vega + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-vega + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-pg-from-main + namespace: default +spec: + podSelector: + matchLabels: + app: postgres-service + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-main +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-es-from-main + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-main + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-pg-from-barnard + namespace: default +spec: + podSelector: + matchLabels: + app: postgres-service + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-barnard +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-es-from-barnard + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-barnard + +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-pg-from-proxima + namespace: default +spec: + podSelector: + matchLabels: + app: postgres-service + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-proxima +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-es-from-proxima + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-proxima + +--- +# Postgres service for starcoin-api +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-pg-from-api + namespace: default +spec: + podSelector: + matchLabels: + app: postgres-service + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-api + +--- +# Elasticsearch service for starcoin-api +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-access-es-from-api + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + name: starcoin-api + +--- +# Default namespace access for elasticsearch +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-internal-elasticsearch + namespace: default +spec: + podSelector: + matchLabels: + app: elasticsearch + policyTypes: + - Ingress + ingress: + - from: + - podSelector: {} + ports: + - protocol: TCP + port: 9200 diff --git a/kube/base-components/elasticsearch-deployment.yaml b/kube/base-components/elasticsearch-deployment.yaml new file mode 100644 index 0000000..b4bb176 --- /dev/null +++ b/kube/base-components/elasticsearch-deployment.yaml @@ -0,0 +1,150 @@ +# elasticsearch-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch +spec: + replicas: 1 + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + spec: + initContainers: + - name: init-permissions + image: busybox + command: [ "sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data" ] + volumeMounts: + - name: es-data + mountPath: /usr/share/elasticsearch/data + containers: + - name: elasticsearch + securityContext: + privileged: true + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 + resources: + requests: + cpu: "1" + memory: "4Gi" + limits: + cpu: "2" + memory: "6Gi" + ports: + - containerPort: 9200 + name: http + - containerPort: 9300 + name: transport + volumeMounts: + - name: es-data + mountPath: /usr/share/elasticsearch/data + - name: es-backup + mountPath: /data/es_snapshot_repository + - name: elasticsearch-config + mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + subPath: elasticsearch.yml +# - name: s3-mount-point +# mountPath: /mnt/s3_mount + env: + - name: discovery.type + value: single-node + - name: ELASTIC_USERNAME + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: ELASTIC_PASSWORD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password + # S3 配置 + - name: S3_CLIENT_DEFAULT_REGION + value: "ap-northeast-1" + - name: S3_CLIENT_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-credentials + key: access-key + - name: S3_CLIENT_SECRET_KEY + valueFrom: + secretKeyRef: + name: aws-credentials + key: secret-key + lifecycle: + postStart: + exec: + command: ["/bin/bash", "-c", "/usr/share/elasticsearch/bin/elasticsearch-plugin list | grep -q repository-s3 || /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch repository-s3 && \ + echo ${S3_CLIENT_ACCESS_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add s3.client.default.access_key --stdin &&\ + echo ${S3_CLIENT_SECRET_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add s3.client.default.secret_key --stdin"] + volumes: + - name: es-data + persistentVolumeClaim: + claimName: es-pvc + - name: es-backup + persistentVolumeClaim: + claimName: es-backup-pvc + - name: elasticsearch-config + configMap: + name: elasticsearch-config +# - name: s3-mount-point +# emptyDir: {} + +--- +# Elasticsearch Configuration +apiVersion: v1 +kind: ConfigMap +metadata: + name: elasticsearch-config +data: + elasticsearch.yml: | + xpack.security.enabled: true + xpack.license.self_generated.type: basic + network.host: 0.0.0.0 + path.repo: ["/data/es_snapshot_repository"] + s3.client.default.endpoint: "s3.ap-northeast-1.amazonaws.com" + s3.client.default.protocol: https + s3.client.default.read_timeout: 50s + s3.client.default.max_retries: 3 + s3.client.default.use_throttle_retries: true +--- +# Elasticsearch Persistent Volume Claim +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: es-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500Gi + storageClassName: do-block-storage-retain + +--- +# Elasticsearch Snapshot Repository Persistent Volume Claim +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: es-backup-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500Gi + storageClassName: do-block-storage-retain + +--- +# Elasticsearch Service +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch +spec: + ports: + - port: 9200 + selector: + app: elasticsearch diff --git a/kube/base-components/kibana-deployment.yaml b/kube/base-components/kibana-deployment.yaml new file mode 100644 index 0000000..20f52c7 --- /dev/null +++ b/kube/base-components/kibana-deployment.yaml @@ -0,0 +1,74 @@ +# kibana-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana +spec: + replicas: 1 + selector: + matchLabels: + app: kibana + template: + metadata: + labels: + app: kibana + spec: + containers: + - name: kibana + image: docker.elastic.co/kibana/kibana:7.17.0 + resources: + requests: + cpu: "0.5" + memory: "2Gi" + limits: + cpu: "1" + memory: "2Gi" + ports: + - containerPort: 5601 + volumeMounts: + - name: kibana-config + mountPath: /usr/share/kibana/config/kibana.yml + subPath: kibana.yml + env: + - name: ELASTICSEARCH_HOSTS + value: "http://elasticsearch.default.svc.cluster.local:9200" + - name: ES_USERNAME + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: ES_PASSWORD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password + volumes: + - name: kibana-config + configMap: + name: kibana-config +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kibana-config +data: + kibana.yml: | + server.name: kibana + server.host: "0.0.0.0" + elasticsearch.hosts: ["http://elasticsearch.default.svc.cluster.local:9200"] + elasticsearch.username: "${ES_USERNAME}" + elasticsearch.password: "${ES_PASSWORD}" +--- +apiVersion: v1 +kind: Service +metadata: + name: kibana +spec: + type: NodePort + selector: + app: kibana + ports: + - protocol: TCP + port: 5601 + targetPort: 5601 + nodePort: 31000 \ No newline at end of file diff --git a/kube/base-components/namespaces.yaml b/kube/base-components/namespaces.yaml new file mode 100644 index 0000000..fa61572 --- /dev/null +++ b/kube/base-components/namespaces.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: starcoin-main + labels: + app: starcoin + kubernetes.io/metadata.name: starcoin-main + name: starcoin-main + network: main + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: starcoin-barnard + labels: + app: starcoin + kubernetes.io/metadata.name: starcoin-barnard + name: starcoin-barnard + network: barnard + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: starcoin-proxima + labels: + app: starcoin + kubernetes.io/metadata.name: starcoin-proxima + name: starcoin-proxima + network: proxima + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: starcoin-halley + labels: + app: starcoin + kubernetes.io/metadata.name: starcoin-halley + name: starcoin-halley + network: halley + + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: starcoin-vega + labels: + app: starcoin + kubernetes.io/metadata.name: starcoin-vega + name: starcoin-vega + network: vega \ No newline at end of file diff --git a/kube/base-components/postgresql-deployment.yaml b/kube/base-components/postgresql-deployment.yaml new file mode 100644 index 0000000..25c9e9f --- /dev/null +++ b/kube/base-components/postgresql-deployment.yaml @@ -0,0 +1,90 @@ +# Persistent Volume +apiVersion: v1 +kind: PersistentVolume +metadata: + name: postgres-pv +spec: + capacity: + storage: 80Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: do-block-storage + hostPath: + path: /mnt/data/postgres +--- +# Persistent Volume Claim +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 80Gi + volumeName: postgres-pv + storageClassName: do-block-storage +--- +# Postgresql deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:13 + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: starcoin + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: postgresql + key: username + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql + key: password + resources: + requests: + memory: "4Gi" + cpu: "2" + limits: + memory: "4Gi" + cpu: "2" + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: postgres-storage + volumes: + - name: postgres-storage + persistentVolumeClaim: + claimName: postgres-pvc +--- +# Service +apiVersion: v1 +kind: Service +metadata: + name: postgres-service +spec: + type: ClusterIP + ports: + - port: 5432 + targetPort: 5432 + selector: + app: postgres + diff --git a/kube/indexer/dag/dag-vega-deployment.yaml b/kube/indexer/dag/dag-vega-deployment.yaml new file mode 100644 index 0000000..2bbabc1 --- /dev/null +++ b/kube/indexer/dag/dag-vega-deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: starcoin-indexer-dag-deployment + namespace: starcoin-vega + labels: + app: starcoin-indexer-dag +spec: + replicas: 1 + selector: + matchLabels: + app: starcoin-indexer-dag + template: + metadata: + labels: + app: starcoin-indexer-dag + spec: + containers: + - name: starcoin-indexer-dag + image: starcoin/starcoin_indexer:dag_inspector_vega_6 + ports: + - containerPort: 8300 + env: + - name: HOSTS + value: "vega.seed.starcoin.org" + - name: NETWORK + value: "vega.0727" + - name: BG_TASK_JOBS + value: "dag_inspector" + - name: BULK_SIZE + value: "500" + - name: STARCOIN_ES_USER + value: elastic + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password + - name: STARCOIN_ES_URL + value: "elasticsearch.default.svc.cluster.local" + - name: TXN_OFFSET + value: "0" + - name: STARCOIN_ES_PROTOCOL + value: "http" + - name: STARCOIN_ES_PORT + value: "9200" + - name: SWAP_API_URL + value: "https://swap-api.starswap.xyz" + - name: SWAP_CONTRACT_ADDR + value: "0xbd7e8be8fae9f60f2f5136433e36a091" + - name: DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" + - name: DB_USER_NAME + value: "starcoin" + - name: DB_PWD + valueFrom: + secretKeyRef: + name: postgresql + key: password + - name: DB_SCHEMA + value: "vega" + - name: PROGRAM_ARGS + value: "" + #valume: diff --git a/kube/indexer/starcoin-indexer-cmd-handle-barnard-deployment.yaml b/kube/indexer/starcoin-indexer-cmd-handle-barnard-deployment.yaml deleted file mode 100644 index 52cb0a6..0000000 --- a/kube/indexer/starcoin-indexer-cmd-handle-barnard-deployment.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: starcoin-indexer-cmd-handle-deployment - namespace: starcoin-barnard - labels: - app: starcoin-indexer-cmd-handle -spec: - replicas: 1 - selector: - matchLabels: - app: starcoin-indexer-cmd-handle - template: - metadata: - labels: - app: starcoin-indexer-cmd-handle - spec: - containers: - - name: starcoin-indexer-cmd-handle - image: starcoin/starcoin_indexer:sha-ba00b10 - ports: - - containerPort: 8300 - env: - - name: HOSTS - value: "barnard.seed.starcoin.org" - - name: NETWORK - value: "barnard.0727" - - name: BG_TASK_JOBS - value: "" - - name: TXN_OFFSET - value: "0" - - name: BULK_SIZE - value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" - - name: STARCOIN_ES_PROTOCOL - value: "https" - - name: STARCOIN_ES_PORT - value: "443" - - name: STARCOIN_ES_USER - value: elastic - - name: SWAP_API_URL - value: "https://swap-api.starswap.xyz" - - name: SWAP_CONTRACT_ADDR - value: "0x8c109349c6bd91411d6bc962e080c4a3" - - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - - name: DB_USER_NAME - value: "starcoin" - - name: DB_PWD - valueFrom: - secretKeyRef: - name: postgresql-starcoin-yu - key: password - - name: DB_SCHEMA - value: "barnard" - - name: PROGRAM_ARGS - value: "migrate_holder" diff --git a/kube/indexer/starcoin-indexer-repair-main-deployment.yaml b/kube/indexer/starcoin-indexer-repair-main-deployment.yaml deleted file mode 100644 index adef2e0..0000000 --- a/kube/indexer/starcoin-indexer-repair-main-deployment.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: starcoin-indexer-repair-deployment - namespace: starcoin-main - labels: - app: starcoin-indexer-repair -spec: - replicas: 1 - selector: - matchLabels: - app: starcoin-indexer-repair - template: - metadata: - labels: - app: starcoin-indexer-repair - spec: - containers: - - name: starcoin-indexer-repair - image: starcoin/starcoin_indexer:sha-6b3888e - ports: - - containerPort: 8300 - env: - - name: HOSTS - value: "main.seed.starcoin.org" - - name: NETWORK - value: "main.0727" - - name: BG_TASK_JOBS - value: "swap_event_handle,swap_pool_fee_stat" - - name: TXN_OFFSET - value: "0" - - name: BULK_SIZE - value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" - - name: STARCOIN_ES_PROTOCOL - value: "https" - - name: STARCOIN_ES_PORT - value: "443" - - name: STARCOIN_ES_USER - value: elastic - - name: SWAP_API_URL - value: "https://swap-api.starswap.xyz" - - name: SWAP_CONTRACT_ADDR - value: "0x8c109349c6bd91411d6bc962e080c4a3" - - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - - name: DB_USER_NAME - value: "starcoin" - - name: DB_PWD - valueFrom: - secretKeyRef: - name: postgresql-starcoin-yu - key: password - - name: DB_SCHEMA - value: "main" - - name: PROGRAM_ARGS - value: "auto_repair 5356700" diff --git a/kube/indexer/starcoin-indexer-barnard-deployment.yaml b/kube/indexer/stc-scan/stcscan-barnard-deployment.yaml similarity index 63% rename from kube/indexer/starcoin-indexer-barnard-deployment.yaml rename to kube/indexer/stc-scan/stcscan-barnard-deployment.yaml index e1454ab..756cd38 100644 --- a/kube/indexer/starcoin-indexer-barnard-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-barnard-deployment.yaml @@ -1,23 +1,23 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-deployment + name: starcoin-indexer-stcscan-deployment namespace: starcoin-barnard labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer + app: starcoin-indexer-stcscan template: metadata: labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: containers: - - name: starcoin-indexer - image: starcoin/starcoin_indexer:sha-9e2763d + - name: starcoin-indexer-stcscan + image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 env: @@ -31,31 +31,37 @@ spec: value: "0" - name: BULK_SIZE value: "80" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "barnard" diff --git a/kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-cmd-handle-main-deployment.yaml similarity index 64% rename from kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml rename to kube/indexer/stc-scan/stcscan-cmd-handle-main-deployment.yaml index 6ea7a43..c4824fb 100644 --- a/kube/indexer/starcoin-indexer-cmd-handle-main-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-cmd-handle-main-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-cmd-handle-deployment + name: starcoin-indexer-stcscan-cmd-handle-deployment namespace: starcoin-main labels: - app: starcoin-indexer-cmd-handle + app: starcoin-indexer-stcscan-cmd-handle spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer-cmd-handle + app: starcoin-indexer-stcscan-cmd-handle template: metadata: labels: - app: starcoin-indexer-cmd-handle + app: starcoin-indexer-stcscan-cmd-handle spec: containers: - - name: starcoin-indexer-cmd-handle + - name: starcoin-indexer-stcscan-cmd-handle image: starcoin/starcoin_indexer:sha-ba00b10 ports: - containerPort: 8300 @@ -31,31 +31,37 @@ spec: value: "0" - name: BULK_SIZE value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" diff --git a/kube/indexer/stc-scan/stcscan-halley-deployment.yaml b/kube/indexer/stc-scan/stcscan-halley-deployment.yaml new file mode 100644 index 0000000..7a035d6 --- /dev/null +++ b/kube/indexer/stc-scan/stcscan-halley-deployment.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: starcoin-indexer-stcscan-deployment + namespace: starcoin-halley + labels: + app: starcoin-indexer-stcscan +spec: + replicas: 1 + selector: + matchLabels: + app: starcoin-indexer-stcscan + template: + metadata: + labels: + app: starcoin-indexer-stcscan + spec: + containers: + - name: starcoin-indexer-stcscan + image: starcoin/starcoin_indexer:halley_fix_db_1 + ports: + - containerPort: 8300 + env: + - name: HOSTS + value: "halley.seed.starcoin.org" + - name: NETWORK + value: "halley.0727" + - name: BG_TASK_JOBS + value: "indexer,secondary,market_cap,txn_payload,txn_global_idx_update" + - name: BULK_SIZE + value: "80" + - name: TXN_OFFSET + value: "0" + - name: STARCOIN_ES_URL + value: "elasticsearch.default.svc.cluster.local" + - name: STARCOIN_ES_PROTOCOL + value: "http" + - name: STARCOIN_ES_PORT + value: "9200" + - name: STARCOIN_ES_USER + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password + - name: SWAP_API_URL + value: "https://swap-api.starswap.xyz" + - name: SWAP_CONTRACT_ADDR + value: "0xbd7e8be8fae9f60f2f5136433e36a091" + - name: DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" + - name: DB_USER_NAME + valueFrom: + secretKeyRef: + name: postgresql + key: username + - name: DB_PWD + valueFrom: + secretKeyRef: + name: postgresql + key: password + - name: DB_SCHEMA + value: "halley" + - name: PROGRAM_ARGS + value: "" + #valume: diff --git a/kube/indexer/stc-scan/stcscan-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-main-deployment.yaml new file mode 100644 index 0000000..0daa112 --- /dev/null +++ b/kube/indexer/stc-scan/stcscan-main-deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: starcoin-indexer-stcscan-deployment + namespace: starcoin-main + labels: + app: starcoin-indexer-stcscan +spec: + replicas: 1 + selector: + matchLabels: + app: starcoin-indexer-stcscan + template: + metadata: + labels: + app: starcoin-indexer-stcscan + spec: + containers: + - name: starcoin-indexer-stcscan + image: starcoin/starcoin_indexer:sha-1223fc3 + ports: + - containerPort: 8300 + env: + - name: HOSTS + value: "main.seed.starcoin.org" + - name: NETWORK + value: "main.0727" + - name: BG_TASK_JOBS + value: "indexer,secondary,market_cap,txn_payload,txn_global_idx_update" + - name: TXN_OFFSET + value: "0" + - name: BULK_SIZE + value: "1000" + - name: STARCOIN_ES_URL + value: "elasticsearch.default.svc.cluster.local" + - name: STARCOIN_ES_PROTOCOL + value: "http" + - name: STARCOIN_ES_PORT + value: "9200" + - name: STARCOIN_ES_USER + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password + - name: SWAP_API_URL + value: "https://swap-api.starswap.xyz" + - name: SWAP_CONTRACT_ADDR + value: "0x8c109349c6bd91411d6bc962e080c4a3" + - name: DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" + - name: DB_USER_NAME + valueFrom: + secretKeyRef: + name: postgresql + key: username + - name: DB_PWD + valueFrom: + secretKeyRef: + name: postgresql + key: password + - name: DB_SCHEMA + value: "main" + - name: PROGRAM_ARGS + value: "" diff --git a/kube/indexer/starcoin-indexer-proxima-deployment.yaml b/kube/indexer/stc-scan/stcscan-proxima-deployment.yaml similarity index 60% rename from kube/indexer/starcoin-indexer-proxima-deployment.yaml rename to kube/indexer/stc-scan/stcscan-proxima-deployment.yaml index 9dde210..13efaf4 100644 --- a/kube/indexer/starcoin-indexer-proxima-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-proxima-deployment.yaml @@ -1,23 +1,23 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-deployment + name: starcoin-indexer-stcscan-deployment namespace: starcoin-proxima labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer + app: starcoin-indexer-stcscan template: metadata: labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: containers: - - name: starcoin-indexer - image: starcoin/starcoin_indexer:sha-3af1c2a + - name: starcoin-indexer-stcscan + image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 env: @@ -26,36 +26,42 @@ spec: - name: NETWORK value: "proxima.0727" - name: BG_TASK_JOBS - value: "indexer,secondary,market_cap,txn_payload,swap_stats" + value: "indexer,secondary,market_cap,txn_payload" - name: BULK_SIZE value: "80" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: TXN_OFFSET value: "0" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "proxima" diff --git a/kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml b/kube/indexer/stc-scan/stcscan-repair-barnard-deployment.yaml similarity index 63% rename from kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml rename to kube/indexer/stc-scan/stcscan-repair-barnard-deployment.yaml index cfa860f..ab7975c 100644 --- a/kube/indexer/starcoin-indexer-repair-barnard-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-repair-barnard-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-repair-deployment + name: starcoin-indexer-stcscan-repair-deployment namespace: starcoin-barnard labels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair template: metadata: labels: - app: starcoin-indexer-repair + app: starcoin-indexer-stcscan-repair spec: containers: - - name: starcoin-indexer-repair + - name: starcoin-indexer-stcscan-repair image: starcoin/starcoin_indexer:sha-6eb4945 ports: - containerPort: 8300 @@ -31,33 +31,39 @@ spec: value: "5000000" - name: BULK_SIZE value: "80" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "barnard" - name: PROGRAM_ARGS - value: "auto_repair 16057420" + value: "auto_repair 16057420" \ No newline at end of file diff --git a/kube/indexer/stc-scan/stcscan-repair-halley-deployment.yaml b/kube/indexer/stc-scan/stcscan-repair-halley-deployment.yaml new file mode 100644 index 0000000..f60d911 --- /dev/null +++ b/kube/indexer/stc-scan/stcscan-repair-halley-deployment.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: starcoin-indexer-stcscan-repair-deployment + namespace: starcoin-halley + labels: + app: starcoin-indexer-stcscan-repair +spec: + replicas: 1 + selector: + matchLabels: + app: starcoin-indexer-stcscan-repair + template: + metadata: + labels: + app: starcoin-indexer-stcscan-repair + spec: + containers: + - name: starcoin-indexer-stcscan-repair + image: starcoin/starcoin_indexer:halley_fix_db_1 + ports: + - containerPort: 8300 + env: + - name: HOSTS + value: "halley.seed.starcoin.org" + - name: NETWORK + value: "halley.0727" + - name: BG_TASK_JOBS + value: "indexer,secondary,market_cap,txn_payload" + - name: BULK_SIZE + value: "80" + - name: STARCOIN_ES_URL + value: "elasticsearch.default.svc.cluster.local" + - name: TXN_OFFSET + value: "0" + - name: STARCOIN_ES_PROTOCOL + value: "http" + - name: STARCOIN_ES_PORT + value: "9200" + - name: STARCOIN_ES_USER + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password + - name: SWAP_API_URL + value: "https://swap-api.starswap.xyz" + - name: SWAP_CONTRACT_ADDR + value: "0xbd7e8be8fae9f60f2f5136433e36a091" + - name: DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" + - name: DB_USER_NAME + valueFrom: + secretKeyRef: + name: postgresql + key: username + - name: DB_PWD + valueFrom: + secretKeyRef: + name: postgresql + key: password + - name: DB_SCHEMA + value: "halley" + - name: PROGRAM_ARGS + value: "auto_repair 1" + #valume: diff --git a/kube/indexer/stc-scan/stcscan-repair-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-repair-main-deployment.yaml new file mode 100644 index 0000000..dbc44a4 --- /dev/null +++ b/kube/indexer/stc-scan/stcscan-repair-main-deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: starcoin-indexer-stcscan-repair-deployment + namespace: starcoin-main + labels: + app: starcoin-indexer-stcscan-repair +spec: + replicas: 1 + selector: + matchLabels: + app: starcoin-indexer-stcscan-repair + template: + metadata: + labels: + app: starcoin-indexer-stcscan-repair + spec: + containers: + - name: starcoin-indexer-stcscan-repair + image: starcoin/starcoin_indexer:sha-1223fc3 + ports: + - containerPort: 8300 + env: + - name: HOSTS + value: "main.seed.starcoin.org" + - name: NETWORK + value: "main.0727" + - name: BG_TASK_JOBS + value: "" + - name: TXN_OFFSET + value: "0" + - name: BULK_SIZE + value: "100" + - name: STARCOIN_ES_URL + value: "elasticsearch.default.svc.cluster.local" + - name: STARCOIN_ES_PROTOCOL + value: "http" + - name: STARCOIN_ES_PORT + value: "9200" + - name: STARCOIN_ES_USER + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password + - name: SWAP_API_URL + value: "https://swap-api.starswap.xyz" + - name: SWAP_CONTRACT_ADDR + value: "0x8c109349c6bd91411d6bc962e080c4a3" + - name: DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" + - name: DB_USER_NAME + valueFrom: + secretKeyRef: + name: postgresql + key: username + - name: DB_PWD + valueFrom: + secretKeyRef: + name: postgresql + key: password + - name: DB_SCHEMA + value: "main" + - name: PROGRAM_ARGS + value: "auto_repair 9411700" diff --git a/kube/indexer/starcoin-indexer-txn-main-deployment.yaml b/kube/indexer/stc-scan/stcscan-txn-main-deployment.yaml similarity index 65% rename from kube/indexer/starcoin-indexer-txn-main-deployment.yaml rename to kube/indexer/stc-scan/stcscan-txn-main-deployment.yaml index 48fd2ba..dd4bc9c 100644 --- a/kube/indexer/starcoin-indexer-txn-main-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-txn-main-deployment.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-txn-deployment + name: starcoin-indexer-stcscan-txn-deployment namespace: starcoin-main labels: - app: starcoin-indexer-txn + app: starcoin-indexer-stcscan-txn spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer-txn + app: starcoin-indexer-stcscan-txn template: metadata: labels: - app: starcoin-indexer-txn + app: starcoin-indexer-stcscan-txn spec: containers: - - name: starcoin-indexer-txn + - name: starcoin-indexer-stcscan-txn image: starcoin/starcoin_indexer:sha-ba00b10 ports: - containerPort: 8300 @@ -31,31 +31,37 @@ spec: value: "5217755" - name: BULK_SIZE value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" diff --git a/kube/indexer/starcoin-indexer-halley-deployment.yaml b/kube/indexer/stc-scan/stcscan-vega-deployment.yaml similarity index 59% rename from kube/indexer/starcoin-indexer-halley-deployment.yaml rename to kube/indexer/stc-scan/stcscan-vega-deployment.yaml index b4f24d3..afe6dcd 100644 --- a/kube/indexer/starcoin-indexer-halley-deployment.yaml +++ b/kube/indexer/stc-scan/stcscan-vega-deployment.yaml @@ -1,47 +1,47 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-deployment - namespace: starcoin-halley + name: starcoin-indexer-stcscan-deployment + namespace: starcoin-vega labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer + app: starcoin-indexer-stcscan template: metadata: labels: - app: starcoin-indexer + app: starcoin-indexer-stcscan spec: containers: - - name: starcoin-indexer - image: starcoin/starcoin_indexer:sha-3af1c2a + - name: starcoin-indexer-stcscan + image: starcoin/starcoin_indexer:dag_inspector_vega_6 ports: - containerPort: 8300 env: - name: HOSTS - value: "halley.seed.starcoin.org" + value: "vega.seed.starcoin.org" - name: NETWORK - value: "halley.0727" + value: "vega.0727" - name: BG_TASK_JOBS value: "indexer,secondary,market_cap,txn_payload,swap_stats,txn_global_idx_update" - name: BULK_SIZE - value: "80" + value: "100" - name: STARCOIN_ES_PWD valueFrom: secretKeyRef: - name: elasticsearch-pw-elastic + name: elasticsearch key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: TXN_OFFSET - value: "451993" + value: "0" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER value: elastic - name: SWAP_API_URL @@ -49,16 +49,19 @@ spec: - name: SWAP_CONTRACT_ADDR value: "0xbd7e8be8fae9f60f2f5136433e36a091" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA - value: "halley" + value: "vega" - name: PROGRAM_ARGS value: "" #valume: diff --git a/kube/indexer/starcoin-indexer-main-deployment.yaml b/kube/indexer/swap/swap-info-main-deployment.yaml similarity index 61% rename from kube/indexer/starcoin-indexer-main-deployment.yaml rename to kube/indexer/swap/swap-info-main-deployment.yaml index ced9a0e..f1df85e 100644 --- a/kube/indexer/starcoin-indexer-main-deployment.yaml +++ b/kube/indexer/swap/swap-info-main-deployment.yaml @@ -1,23 +1,23 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: starcoin-indexer-deployment + name: starswap-info-deployment namespace: starcoin-main labels: - app: starcoin-indexer + app: starswap-info spec: replicas: 1 selector: matchLabels: - app: starcoin-indexer + app: starswap-info template: metadata: labels: - app: starcoin-indexer + app: starswap-info spec: containers: - - name: starcoin-indexer - image: starcoin/starcoin_indexer:sha-9e2763d + - name: starswap-info + image: starcoin/starcoin_indexer:sha-a66e9ab ports: - containerPort: 8300 env: @@ -26,36 +26,42 @@ spec: - name: NETWORK value: "main.0727" - name: BG_TASK_JOBS - value: "indexer,secondary,market_cap,txn_payload" + value: "swap_stats,price_hour,price_stat,swap_pool_fee_stat" - name: TXN_OFFSET value: "0" - - name: BULK_SIZE - value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password + - name: BULK_SIZE + value: "100" - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" diff --git a/kube/indexer/starcoin-indexer-stat-main-deployment.yaml b/kube/indexer/swap/swap-stat-main-deployment.yaml similarity index 72% rename from kube/indexer/starcoin-indexer-stat-main-deployment.yaml rename to kube/indexer/swap/swap-stat-main-deployment.yaml index 1d007cc..89913c3 100644 --- a/kube/indexer/starcoin-indexer-stat-main-deployment.yaml +++ b/kube/indexer/swap/swap-stat-main-deployment.yaml @@ -26,36 +26,42 @@ spec: - name: NETWORK value: "main.0727" - name: BG_TASK_JOBS - value: "swap_stats" + value: "swap_stats,price_hour,price_stat" - name: TXN_OFFSET value: "0" - name: BULK_SIZE value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" diff --git a/kube/indexer/starcoin-indexer-swap-main-deployment.yaml b/kube/indexer/swap/swap-txns-main-deployment.yaml similarity index 68% rename from kube/indexer/starcoin-indexer-swap-main-deployment.yaml rename to kube/indexer/swap/swap-txns-main-deployment.yaml index 0c5179e..14af86a 100644 --- a/kube/indexer/starcoin-indexer-swap-main-deployment.yaml +++ b/kube/indexer/swap/swap-txns-main-deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: starcoin-indexer-swap - image: starcoin/starcoin_indexer:sha-a66e9ab + image: starcoin/starcoin_indexer:sha-1223fc3 ports: - containerPort: 8300 env: @@ -26,36 +26,42 @@ spec: - name: NETWORK value: "main.0727" - name: BG_TASK_JOBS - value: "price_hour,price_stat" + value: "swap_transaction,swap_stats,price_hour,price_stat,swap_pool_fee_stat" - name: TXN_OFFSET value: "0" - name: BULK_SIZE value: "100" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starswap.xyz" - name: SWAP_CONTRACT_ADDR value: "0x8c109349c6bd91411d6bc962e080c4a3" - name: DS_URL - value: "jdbc:postgresql://database-1.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: DB_SCHEMA value: "main" diff --git a/kube/mappings/dag_inspector_block.mapping.json b/kube/mappings/dag_inspector_block.mapping.json new file mode 100644 index 0000000..0345422 --- /dev/null +++ b/kube/mappings/dag_inspector_block.mapping.json @@ -0,0 +1,57 @@ +{ + "mappings" : { + "properties" : { + "block_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "color" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "daa_score" : { + "type" : "long" + }, + "height" : { + "type" : "long" + }, + "height_group_index" : { + "type" : "long" + }, + "in_virtual_selected_parent_chain" : { + "type" : "boolean" + }, + "parent_ids" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "selected_parent_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "timestamp" : { + "type" : "long" + } + } + } +} \ No newline at end of file diff --git a/kube/mappings/dag_inspector_edge.mapping.json b/kube/mappings/dag_inspector_edge.mapping.json new file mode 100644 index 0000000..00655bf --- /dev/null +++ b/kube/mappings/dag_inspector_edge.mapping.json @@ -0,0 +1,36 @@ +{ + "mappings" : { + "properties" : { + "from_block_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "from_group_index" : { + "type" : "long" + }, + "from_height" : { + "type" : "long" + }, + "to_block_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } + }, + "to_group_index" : { + "type" : "long" + }, + "to_height" : { + "type" : "long" + } + } + } +} \ No newline at end of file diff --git a/kube/mappings/dag_inspector_height_group.mapping.json b/kube/mappings/dag_inspector_height_group.mapping.json new file mode 100644 index 0000000..24f6863 --- /dev/null +++ b/kube/mappings/dag_inspector_height_group.mapping.json @@ -0,0 +1,12 @@ +{ + "mappings" : { + "properties" : { + "height" : { + "type" : "long" + }, + "size" : { + "type" : "long" + } + } + } +} \ No newline at end of file diff --git a/kube/mappings/es_pipeline.scripts b/kube/mappings/es_pipeline.scripts index 6c7775a..6d17eb2 100644 --- a/kube/mappings/es_pipeline.scripts +++ b/kube/mappings/es_pipeline.scripts @@ -33,7 +33,7 @@ PUT _component_template/block-mappings PUT _component_template/block_ids-mappings { "template": { - # copy block.mapping.json to here. + # copy block.mapping_ids.json to here. } } @@ -107,6 +107,27 @@ PUT _component_template/block-mappings # copy market_cap.mapping.json to here. } } + + PUT _component_template/dag_inspector_block-mappings + { + "template": { + # copy dag_inspector_block.mapping.json to here. + } + } + + PUT _component_template/dag_inspector_edge-mappings + { + "template": { + # copy dag_inspector_edge.mapping.json to here. + } + } + + PUT _component_template/dag_inspector_height_group-mappings + { + "template": { + # copy dag_inspector_height_group.mapping.json to here. + } + } 4. PUT _index_template/blocks-template { @@ -189,4 +210,25 @@ PUT _component_template/block-mappings "index_patterns": ["*.market_cap"], "priority": 500, "composed_of": ["market_cap-mappings"] + } + + PUT _index_template/dag_inspector_block-template + { + "index_patterns": ["*.dag_inspector_block"], + "priority": 500, + "composed_of": ["dag_inspector_block-mappings"] + } + + PUT _index_template/dag_inspector_edge-template + { + "index_patterns": ["*.dag_inspector_edge"], + "priority": 500, + "composed_of": ["dag_inspector_edge-mappings"] + } + + PUT _index_template/dag_inspector_height_group-template + { + "index_patterns": ["*.dag_inspector_height_group"], + "priority": 500, + "composed_of": ["dag_inspector_height_group-mappings"] } \ No newline at end of file diff --git a/kube/mappings/txn_info.mapping.json b/kube/mappings/txn_info.mapping.json index e0c47d3..12d0d76 100644 --- a/kube/mappings/txn_info.mapping.json +++ b/kube/mappings/txn_info.mapping.json @@ -1,228 +1,231 @@ { - "mappings": { - "dynamic": "strict", - "properties": { - "timestamp": { - "type": "date" - }, - "deleted": { - "type": "boolean" - }, - "block_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "mappings" : { + "dynamic" : "strict", + "properties" : { + "block_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "transaction_type": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "block_metadata": { - "properties": { - "author": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "block_metadata" : { + "properties" : { + "author" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "author_auth_key": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "author_auth_key" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "chain_id": { - "type": "long" + "chain_id" : { + "type" : "long" }, - "number": { - "type": "long" + "number" : { + "type" : "long" }, - "parent_gas_used": { - "type": "long" + "parent_gas_used" : { + "type" : "long" }, - "parent_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "parent_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "timestamp": { - "type": "date" + "parents_hash" : { + "type" : "text" + }, + "timestamp" : { + "type" : "date" }, - "uncles": { - "type": "long" + "uncles" : { + "type" : "long" } } }, - "block_number": { - "type": "long" + "block_number" : { + "type" : "long" + }, + "deleted" : { + "type" : "boolean" }, - "event_root_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "event_root_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "events": { - "type": "nested", - "properties": { - "block_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "events" : { + "type" : "nested", + "properties" : { + "block_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "block_number": { - "type": "long" + "block_number" : { + "type" : "long" }, - "data": { - "type": "text" + "data" : { + "type" : "text" }, - "event_key": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "event_key" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "event_seq_number": { - "type": "long" + "event_seq_number" : { + "type" : "long" }, - "transaction_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "transaction_global_index" : { + "type" : "long" + }, + "transaction_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "transaction_index": { - "type": "long" - }, - "transaction_global_index": { - "type": "long" + "transaction_index" : { + "type" : "long" }, - "type_tag": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "type_tag" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } } } }, - "gas_used": { - "type": "long" + "gas_used" : { + "type" : "long" }, - "state_root_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "state_root_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "status": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "status" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "status_content": { - "type": "object", - "dynamic": true + "status_content" : { + "type" : "object", + "dynamic" : "true" + }, + "timestamp" : { + "type" : "date" + }, + "transaction_global_index" : { + "type" : "long" }, - "transaction_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "transaction_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "transaction_index": { - "type": "long" + "transaction_index" : { + "type" : "long" }, - "transaction_global_index": { - "type": "long" + "transaction_type" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 + } + } }, - "user_transaction": { - "properties": { - "authenticator": { - "properties": { - "Ed25519": { - "properties": { - "public_key": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "user_transaction" : { + "properties" : { + "authenticator" : { + "properties" : { + "Ed25519" : { + "properties" : { + "public_key" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "signature": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "signature" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } } } }, - "MultiEd25519": { - "properties": { - "public_key": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "MultiEd25519" : { + "properties" : { + "public_key" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "signature": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "signature" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } } @@ -230,53 +233,56 @@ } } }, - "raw_txn": { - "properties": { - "chain_id": { - "type": "long" + "raw_txn" : { + "properties" : { + "chain_id" : { + "type" : "long" }, - "expiration_timestamp_secs": { - "type": "date", - "format": "epoch_second" + "expiration_timestamp_secs" : { + "type" : "date", + "format" : "epoch_second" }, - "gas_token_code": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "gas_token_code" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "gas_unit_price": { - "type": "long" + "gas_unit_price" : { + "type" : "long" }, - "max_gas_amount": { - "type": "long" + "max_gas_amount" : { + "type" : "long" }, - "payload": { - "type": "text" + "payload" : { + "type" : "text" }, - "sender": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "sender" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } }, - "sequence_number": { - "type": "long" + "sequence_number" : { + "type" : "long" + }, + "decoded_payload": { + "type" : "text" } } }, - "transaction_hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "transaction_hash" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above" : 256 } } } @@ -284,5 +290,4 @@ } } } -} - +} \ No newline at end of file diff --git a/kube/scan-api/scan-api-deployment.yaml b/kube/scan-api/scan-api-deployment.yaml index 8b78da5..0659cbe 100644 --- a/kube/scan-api/scan-api-deployment.yaml +++ b/kube/scan-api/scan-api-deployment.yaml @@ -2,10 +2,11 @@ apiVersion: apps/v1 kind: Deployment metadata: name: scan-api-deployment + namespace: starcoin-api labels: app: scan-api spec: - replicas: 3 + replicas: 1 selector: matchLabels: app: scan-api @@ -16,7 +17,7 @@ spec: spec: containers: - name: scan-api - image: starcoin/starcoin_scan:sha-3af1c2a + image: starcoin/starcoin_scan:scan_api_dag_2 ports: - name: http containerPort: 8500 @@ -24,35 +25,42 @@ spec: containerPort: 5701 env: - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic - - name: STARCOIN_ES_INDEX_VERSION - value: ".0727" + valueFrom: + secretKeyRef: + name: elasticsearch + key: username - name: STARCOIN_ES_PWD valueFrom: secretKeyRef: - name: elasticsearch-pw-elastic + name: elasticsearch key: password - - name: MAIN_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=main" + - name: STARCOIN_ES_INDEX_VERSION + value: ".0727" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - - name: BARNARD_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=barnard" - - name: HALLEY_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=halley" - - name: STARCOIN_USER_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=starcoin_user" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password + # Maybe useless variables + - name: MAIN_DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin?currentSchema=main" + - name: BARNARD_DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin?currentSchema=barnard" + - name: HALLEY_DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin?currentSchema=halley" + - name: STARCOIN_USER_DS_URL + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin?currentSchema=starcoin_user" diff --git a/kube/scan-api/scan-test-api-deployment.yaml b/kube/scan-api/scan-test-api-deployment.yaml deleted file mode 100644 index 305eee7..0000000 --- a/kube/scan-api/scan-test-api-deployment.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: scan-test-api-deployment - labels: - app: scan-test-api -spec: - replicas: 1 - selector: - matchLabels: - app: scan-test-api - template: - metadata: - labels: - app: scan-test-api - spec: - containers: - - name: scan-test-api - image: starcoin/starcoin_scan:sha-3af1c2a - ports: - - name: http - containerPort: 8500 - - name: multicast - containerPort: 5701 - env: - - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" - - name: STARCOIN_ES_PROTOCOL - value: "https" - - name: STARCOIN_ES_PORT - value: "443" - - name: STARCOIN_ES_USER - value: elastic - - name: STARCOIN_ES_INDEX_VERSION - value: ".0727" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - - name: MAIN_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=main" - - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" - - name: BARNARD_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=barnard" - - name: HALLEY_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=halley" - - name: STARCOIN_USER_DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin?currentSchema=starcoin_user" - - name: DB_USER_NAME - value: "starcoin" - - name: DB_PWD - valueFrom: - secretKeyRef: - name: postgresql-starcoin-yu - key: password - diff --git a/kube/scan-api/service-scan-api.yaml b/kube/scan-api/service-scan-api.yaml index 9b0dac9..0c31a23 100644 --- a/kube/scan-api/service-scan-api.yaml +++ b/kube/scan-api/service-scan-api.yaml @@ -4,6 +4,7 @@ metadata: labels: app: scan-api name: scan-api + namespace: starcoin-api spec: ports: - port: 80 diff --git a/kube/scan-api/service-scan-hazelcast.yaml b/kube/scan-api/service-scan-hazelcast.yaml index fe76c6b..7ab72bd 100644 --- a/kube/scan-api/service-scan-hazelcast.yaml +++ b/kube/scan-api/service-scan-hazelcast.yaml @@ -4,6 +4,7 @@ metadata: labels: app: scan-api name: hazelcast-service + namespace: starcoin-api spec: selector: app: scan-api diff --git a/kube/scan-api/service-scan-test-api.yaml b/kube/scan-api/service-scan-test-api.yaml deleted file mode 100644 index 1bec2e7..0000000 --- a/kube/scan-api/service-scan-test-api.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: scan-test-api - name: scan-test-api -spec: - ports: - - port: 80 - name: http - protocol: TCP - targetPort: 8500 - selector: - app: scan-test-api - type: LoadBalancer -status: - loadBalancer: { } diff --git a/kube/scan-api/service-scan-test-hazelcast.yaml b/kube/scan-api/service-scan-test-hazelcast.yaml deleted file mode 100644 index 420513f..0000000 --- a/kube/scan-api/service-scan-test-hazelcast.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: scan-test-api - name: hazelcast-service -spec: - selector: - app: scan-test-api - ports: - - name: multicast - port: 5701 - type: LoadBalancer \ No newline at end of file diff --git a/kube/subscribe/starscan-sub-barnard-deployment-ali.yaml b/kube/subscribe/starscan-sub-barnard-deployment-ali.yaml index 9a91da8..7a9c1d9 100644 --- a/kube/subscribe/starscan-sub-barnard-deployment-ali.yaml +++ b/kube/subscribe/starscan-sub-barnard-deployment-ali.yaml @@ -27,31 +27,37 @@ spec: value: "barnard" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "123.57.40.36" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL value: "http" - name: STARCOIN_ES_PORT value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://123.57.40.36:8432/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-pw-starcoin + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" diff --git a/kube/subscribe/starscan-sub-barnard-deployment.yaml b/kube/subscribe/starscan-sub-barnard-deployment.yaml index e93e3ff..a68d943 100644 --- a/kube/subscribe/starscan-sub-barnard-deployment.yaml +++ b/kube/subscribe/starscan-sub-barnard-deployment.yaml @@ -27,31 +27,37 @@ spec: value: "barnard.0727" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" diff --git a/kube/subscribe/starscan-sub-halley-deployment.yaml b/kube/subscribe/starscan-sub-halley-deployment.yaml index 6fa4e86..8ad3221 100644 --- a/kube/subscribe/starscan-sub-halley-deployment.yaml +++ b/kube/subscribe/starscan-sub-halley-deployment.yaml @@ -27,36 +27,37 @@ spec: value: "halley.0727" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username - name: STARCOIN_ES_PWD valueFrom: secretKeyRef: - name: elasticsearch-pw-elastic + name: elasticsearch key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" diff --git a/kube/subscribe/starscan-sub-main-deployment-ali.yaml b/kube/subscribe/starscan-sub-main-deployment-ali.yaml index b6c83ba..b74044f 100644 --- a/kube/subscribe/starscan-sub-main-deployment-ali.yaml +++ b/kube/subscribe/starscan-sub-main-deployment-ali.yaml @@ -27,31 +27,37 @@ spec: value: "main" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "123.57.40.36" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL value: "http" - name: STARCOIN_ES_PORT value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://123.57.40.36:8432/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-pw-starcoin + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" diff --git a/kube/subscribe/starscan-sub-main-deployment.yaml b/kube/subscribe/starscan-sub-main-deployment.yaml index c5870ef..fed8735 100644 --- a/kube/subscribe/starscan-sub-main-deployment.yaml +++ b/kube/subscribe/starscan-sub-main-deployment.yaml @@ -27,31 +27,37 @@ spec: value: "main.0727" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" diff --git a/kube/subscribe/starscan-sub-proxima-deployment.yaml b/kube/subscribe/starscan-sub-proxima-deployment.yaml index 7d3ce5b..b060b60 100644 --- a/kube/subscribe/starscan-sub-proxima-deployment.yaml +++ b/kube/subscribe/starscan-sub-proxima-deployment.yaml @@ -27,31 +27,37 @@ spec: value: "proxima.0727" - name: BG_TASK_JOBS value: "clean_pending_txn" - - name: STARCOIN_ES_PWD - valueFrom: - secretKeyRef: - name: elasticsearch-pw-elastic - key: password - name: STARCOIN_ES_URL - value: "search-starcoin-es2-47avtmhexhbg7qtynzebcnnu64.ap-northeast-1.es.amazonaws.com" + value: "elasticsearch.default.svc.cluster.local" - name: STARCOIN_ES_PROTOCOL - value: "https" + value: "http" - name: STARCOIN_ES_PORT - value: "443" + value: "9200" - name: STARCOIN_ES_USER - value: elastic + valueFrom: + secretKeyRef: + name: elasticsearch + key: username + - name: STARCOIN_ES_PWD + valueFrom: + secretKeyRef: + name: elasticsearch + key: password - name: SWAP_API_URL value: "https://swap-api.starcoin.org" - name: SWAP_CONTRACT_ADDR value: "" - name: DS_URL - value: "jdbc:postgresql://starcoin-pg.c0bz9kehdvnb.ap-northeast-1.rds.amazonaws.com/starcoin" + value: "jdbc:postgresql://postgres-service.default.svc.cluster.local/starcoin" - name: DB_USER_NAME - value: "starcoin" + valueFrom: + secretKeyRef: + name: postgresql + key: username - name: DB_PWD valueFrom: secretKeyRef: - name: postgresql-starcoin-yu + name: postgresql key: password - name: PROGRAM_ARGS value: "subscribe" diff --git a/pom.xml b/pom.xml index deece9e..60122ae 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.starcoin sdk - 1.1.31 + 1.2.9 org.springframework.boot diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java new file mode 100644 index 0000000..7af5287 --- /dev/null +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorBlock.java @@ -0,0 +1,122 @@ +package org.starcoin.bean; + +import java.util.List; + +public class DagInspectorBlock { + String blockHash; + Long timestamp; + List parentIds; + Long height; + Long daaScore; + Integer heightGroupIndex; + String selectedParentHash; + String color; + Boolean isInVirtualSelectedParentChain; + List mergeSetRedIds; + List mergeSetBlueIds; + + public String getBlockHash() { + return blockHash; + } + + public void setBlockHash(String blockHash) { + this.blockHash = blockHash; + } + + public Long getTimestamp() { + return timestamp; + } + + public void setTimestamp(Long timestamp) { + this.timestamp = timestamp; + } + + public List getParentIds() { + return parentIds; + } + + public void setParentIds(List parentIds) { + this.parentIds = parentIds; + } + + public Long getHeight() { + return height; + } + + public void setHeight(Long height) { + this.height = height; + } + + public Long getDaaScore() { + return daaScore; + } + + public void setDaaScore(Long daaScore) { + this.daaScore = daaScore; + } + + public Integer getHeightGroupIndex() { + return heightGroupIndex; + } + + public void setHeightGroupIndex(Integer heightGroupIndex) { + this.heightGroupIndex = heightGroupIndex; + } + + public String getSelectedParentHash() { + return selectedParentHash; + } + + public void setSelectedParentHash(String selectedParentHash) { + this.selectedParentHash = selectedParentHash; + } + + public String getColor() { + return color; + } + + public void setColor(String color) { + this.color = color; + } + + public Boolean getInVirtualSelectedParentChain() { + return isInVirtualSelectedParentChain; + } + + public void setInVirtualSelectedParentChain(Boolean inVirtualSelectedParentChain) { + isInVirtualSelectedParentChain = inVirtualSelectedParentChain; + } + + public List getMergeSetRedIds() { + return mergeSetRedIds; + } + + public void setMergeSetRedIds(List mergeSetRedIds) { + this.mergeSetRedIds = mergeSetRedIds; + } + + public List getMergeSetBlueIds() { + return mergeSetBlueIds; + } + + public void setMergeSetBlueIds(List mergeSetBlueIds) { + this.mergeSetBlueIds = mergeSetBlueIds; + } + + @Override + public String toString() { + return "DagInspectorBlock{" + + "blockHash='" + blockHash + '\'' + + ", timestamp=" + timestamp + + ", parentIds=" + parentIds + + ", height=" + height + + ", daaScore=" + daaScore + + ", heightGroupIndex=" + heightGroupIndex + + ", selectedParentHash='" + selectedParentHash + '\'' + + ", color='" + color + '\'' + + ", isInVirtualSelectedParentChain=" + isInVirtualSelectedParentChain + + ", mergeSetRedIds=" + mergeSetRedIds + + ", mergeSetBlueIds=" + mergeSetBlueIds + + '}'; + } +} diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java new file mode 100644 index 0000000..08f3c0e --- /dev/null +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorEdge.java @@ -0,0 +1,70 @@ +package org.starcoin.bean; + +public class DagInspectorEdge { + String fromBlockHash; + String toBlockHash; + Long fromHeight; + Long toHeight; + Integer fromHeightGroupIndex; + Integer toHeightGroupIndex; + + public String getFromBlockHash() { + return fromBlockHash; + } + + public void setFromBlockHash(String fromBlockHash) { + this.fromBlockHash = fromBlockHash; + } + + public String getToBlockHash() { + return toBlockHash; + } + + public void setToBlockHash(String toBlockHash) { + this.toBlockHash = toBlockHash; + } + + public Long getFromHeight() { + return fromHeight; + } + + public void setFromHeight(Long fromHeight) { + this.fromHeight = fromHeight; + } + + public Long getToHeight() { + return toHeight; + } + + public void setToHeight(Long toHeight) { + this.toHeight = toHeight; + } + + public Integer getFromHeightGroupIndex() { + return fromHeightGroupIndex; + } + + public void setFromHeightGroupIndex(Integer fromHeightGroupIndex) { + this.fromHeightGroupIndex = fromHeightGroupIndex; + } + + public Integer getToHeightGroupIndex() { + return toHeightGroupIndex; + } + + public void setToHeightGroupIndex(Integer toHeightGroupIndex) { + this.toHeightGroupIndex = toHeightGroupIndex; + } + + @Override + public String toString() { + return "DagInspectorEdge{" + + "fromBlockHash='" + fromBlockHash + '\'' + + ", toBlockHash='" + toBlockHash + '\'' + + ", fromHeight=" + fromHeight + + ", toHeight=" + toHeight + + ", fromHeightGroupIndex=" + fromHeightGroupIndex + + ", toHeightGroupIndex=" + toHeightGroupIndex + + '}'; + } +} diff --git a/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java new file mode 100644 index 0000000..fa24b24 --- /dev/null +++ b/starcoin-commons/src/main/java/org/starcoin/bean/DagInspectorHeightGroup.java @@ -0,0 +1,30 @@ +package org.starcoin.bean; + +public class DagInspectorHeightGroup { + Long height; + Integer size; + + public Long getHeight() { + return height; + } + + public void setHeight(Long height) { + this.height = height; + } + + public Integer getSize() { + return size; + } + + public void setSize(Integer size) { + this.size = size; + } + + @Override + public String toString() { + return "DagInspectorHeightGroup{" + + "height=" + height + + ", size=" + size + + '}'; + } +} diff --git a/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java b/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java index c1b57df..a298c54 100644 --- a/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java +++ b/starcoin-commons/src/main/java/org/starcoin/constant/Constant.java @@ -15,6 +15,9 @@ public class Constant { public static final String MARKET_CAP_INDEX = "market_cap"; public static final String PAYLOAD_INDEX = "txn_payloads"; public static final String TOKEN_INFO_INDEX = "token_info"; + public static final String DAG_INSPECTOR_BLOCK_INDEX = "dag_inspector_block"; + public static final String DAG_INSPECTOR_EDGE_INDEX = "dag_inspector_edge"; + public static final String DAG_INSPECT_HEIGHT_GROUP_INDEX = "dag_inspector_height_group"; public static final String EVENT_FILTER_ADDRESS = "0x00000000000000000000000000000001"; diff --git a/starcoin-commons/src/main/java/org/starcoin/utils/ExceptionWrap.java b/starcoin-commons/src/main/java/org/starcoin/utils/ExceptionWrap.java new file mode 100644 index 0000000..6329b80 --- /dev/null +++ b/starcoin-commons/src/main/java/org/starcoin/utils/ExceptionWrap.java @@ -0,0 +1,18 @@ +package org.starcoin.utils; + +import java.util.function.Function; + +@FunctionalInterface +public interface ExceptionWrap { + R apply(T t) throws Exception; + + static Function wrap(ExceptionWrap checkedFunction) { + return t -> { + try { + return checkedFunction.apply(t); + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + } +} diff --git a/starcoin-indexer/local.env b/starcoin-indexer/local.env new file mode 100644 index 0000000..c45dc4d --- /dev/null +++ b/starcoin-indexer/local.env @@ -0,0 +1,19 @@ +# HOSTS=halley.seed.starcoin.org +HOSTS=localhost +NETWORK=halley +BG_TASK_JOBS=dag_inspector +TXN_OFFSET=0 +BULK_SIZE=100 +STARCOIN_ES_PWD= +STARCOIN_ES_URL=localhost +STARCOIN_ES_PROTOCOL=http +STARCOIN_ES_PORT=9200 +STARCOIN_ES_USER= +SWAP_API_URL=https://swap-api.starswap.xyz +SWAP_CONTRACT_ADDR=0x8c109349c6bd91411d6bc962e080c4a3 +DS_URL=jdbc:postgresql://localhost/starcoin +DB_SCHEMA=halley +DB_USER_NAME=starcoin +DB_PWD=starcoin +PROGRAM_ARGS= +# auto_repair 9411700 \ No newline at end of file diff --git a/starcoin-indexer/pom.xml b/starcoin-indexer/pom.xml index 7f815dd..7dd9187 100644 --- a/starcoin-indexer/pom.xml +++ b/starcoin-indexer/pom.xml @@ -14,7 +14,7 @@ github - GitHub starcoin-search Apache Maven Packages + GitHub stcscan Apache Maven Packages https://maven.pkg.github.com/starcoinorg/starcoin-indexer diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java index 5d17a49..9048169 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java @@ -24,7 +24,7 @@ public class QuartzConfig { @Bean public JobDetail indexerJob() { - return JobBuilder.newJob(IndexerHandle.class).withIdentity("indexer").storeDurably().build(); + return JobBuilder.newJob(IndexerHandleJob.class).withIdentity("indexer").storeDurably().build(); } @Bean @@ -203,6 +203,22 @@ public Trigger priceStatTrigger() { .build(); } + @Bean + public JobDetail dagInspectorJob() { + return JobBuilder.newJob(DagInspectorIndexer.class).withIdentity("dag_inspector").storeDurably().build(); + } + + @Bean + public Trigger dagInspectorTrigger() { + SimpleScheduleBuilder scheduleBuilder = SimpleScheduleBuilder.simpleSchedule() + .withIntervalInSeconds(15) + .repeatForever(); + return TriggerBuilder.newTrigger().forJob(dagInspectorJob()) + .withIdentity("dag_inspector") + .withSchedule(scheduleBuilder) + .build(); + } + @Bean public SchedulerFactoryBean schedulerFactoryBean() { SchedulerFactoryBean schedulerFactoryBean = new SchedulerFactoryBean(); @@ -216,6 +232,7 @@ public SchedulerFactoryBean schedulerFactoryBean() { return schedulerFactoryBean; } + private Properties quartzProperties() { Properties prop = new Properties(); prop.put("org.quartz.scheduler.instanceName", "quartzScheduler");// 调度器的实例名 @@ -281,6 +298,10 @@ public Scheduler scheduler() throws SchedulerException { if (jobSet.contains(job.getKey().getName())) { scheduler.scheduleJob(job, priceStatTrigger()); } + job = dagInspectorJob(); + if (jobSet.contains(job.getKey().getName())) { + scheduler.scheduleJob(job, dagInspectorTrigger()); + } scheduler.start(); return scheduler; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/BlockIndexerOffset.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/BlockIndexerOffset.java new file mode 100644 index 0000000..e378304 --- /dev/null +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/BlockIndexerOffset.java @@ -0,0 +1,136 @@ +package org.starcoin.indexer.handler; + +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.GetMappingsRequest; +import org.elasticsearch.client.indices.GetMappingsResponse; +import org.elasticsearch.client.indices.PutMappingRequest; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.starcoin.api.BlockRPCClient; +import org.starcoin.bean.Block; +import org.starcoin.bean.BlockHeader; +import org.starcoin.bean.BlockOffset; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class BlockIndexerOffset { + + private static final Logger logger = LoggerFactory.getLogger(BlockIndexerOffset.class); + + private String offsetIndexer; + + private BlockOffset localBlockOffset; + + private BlockHeader currentHandleHeader; + + private RestHighLevelClient esClient; + + private BlockRPCClient blockRPCClient; + + + public BlockIndexerOffset(String offsetIndexer, BlockRPCClient blockRPCClient, RestHighLevelClient client) { + this.offsetIndexer = offsetIndexer; + this.esClient = client; + this.blockRPCClient = blockRPCClient; + } + + public void initRemoteOffset() { + //update current handle header + try { + localBlockOffset = getRemoteOffset(); + if (localBlockOffset != null) { + Block block = blockRPCClient.getBlockByHeight(localBlockOffset.getBlockHeight()); + if (block != null) { + currentHandleHeader = block.getHeader(); + } else { + logger.error("init offset block not exist on chain: {}", localBlockOffset); + } + } else { + logger.warn("offset is null,init reset to genesis"); + currentHandleHeader = blockRPCClient.getBlockByHeight(0).getHeader(); + updateBlockOffset(0L, currentHandleHeader.getBlockHash()); + logger.info("init offset ok: {}", localBlockOffset); + } + } catch (JSONRPC2SessionException e) { + logger.error("set current header error:", e); + } + } + + public void updateBlockOffset(Long blockHeight, String blockHash) { + if (localBlockOffset == null) { + localBlockOffset = new BlockOffset(blockHeight, blockHash); + } else { + localBlockOffset.setBlockHeight(blockHeight); + localBlockOffset.setBlockHash(blockHash); + } + setRemoteOffset(localBlockOffset); + } + + public Long getLocalBlockOffsetHeight() { + return localBlockOffset.getBlockHeight(); + } + + public String getLocalOffsetBlockHash() { + return localBlockOffset.getBlockHash(); + } + + public BlockOffset getRemoteOffset() { + GetMappingsRequest request = new GetMappingsRequest(); + try { + request.indices(offsetIndexer); + GetMappingsResponse response = esClient.indices().getMapping(request, RequestOptions.DEFAULT); + MappingMetadata data = response.mappings().get(offsetIndexer); + Object meta = data.getSourceAsMap().get("_meta"); + if (meta != null) { + Map tip = (Map) ((LinkedHashMap) meta).get("tip"); + String blockHash = tip.get("block_hash").toString(); + Integer blockHeight = (Integer) tip.get("block_number"); + return new BlockOffset(blockHeight.longValue(), blockHash); + } + } catch (Exception e) { + logger.error("get remote offset error:", e); + } + return null; + } + + public void setRemoteOffset(BlockOffset blockOffset) { + PutMappingRequest request = new PutMappingRequest(offsetIndexer); + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.startObject("_meta"); + { + builder.startObject("tip"); + { + builder.field("block_hash", blockOffset.getBlockHash()); + builder.field("block_number", blockOffset.getBlockHeight()); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + request.source(builder); + esClient.indices().putMapping(request, RequestOptions.DEFAULT); + logger.info("remote offset update ok : {}", blockOffset); + } catch (Exception e) { + logger.error("get remote offset error:", e); + } + } + + @Override + public String toString() { + return "BlockIndexerOffset{" + + "offsetIndexer='" + offsetIndexer + '\'' + + ", localBlockOffset=" + localBlockOffset + + ", currentHandleHeader=" + currentHandleHeader + + '}'; + } +} diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java new file mode 100644 index 0000000..53ec3b8 --- /dev/null +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexer.java @@ -0,0 +1,212 @@ +package org.starcoin.indexer.handler; + +import org.elasticsearch.client.RestHighLevelClient; +import org.quartz.JobExecutionContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.quartz.QuartzJobBean; +import org.starcoin.api.BlockRPCClient; +import org.starcoin.bean.Block; +import org.starcoin.bean.BlockHeader; +import org.starcoin.bean.BlockOffset; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.LongStream; + +import static org.starcoin.constant.Constant.DAG_INSPECTOR_BLOCK_INDEX; + +public class DagInspectorIndexer extends QuartzJobBean { + + private static final Logger logger = LoggerFactory.getLogger(DagInspectorIndexer.class); + + @Autowired + private DagInspectorIndexerHandler inspectorHandler; + + private BlockIndexerOffset blockIndexerOffset; + + @Autowired + private BlockRPCClient blockRPCClient; + + @Autowired + private RestHighLevelClient esClient; + + @Value("${starcoin.indexer.bulk_size}") + private long bulkSize; + + @Value("${starcoin.network}") + private String network; + + @PostConstruct + public void initOffset() { + blockIndexerOffset = new BlockIndexerOffset( + ServiceUtils.getIndex(network, DAG_INSPECTOR_BLOCK_INDEX), + blockRPCClient, + esClient + ); + blockIndexerOffset.initRemoteOffset(); + } + + @Override + protected void executeInternal(JobExecutionContext jobExecutionContext) { + //read current offset + if (blockIndexerOffset == null) { + initOffset(); + } + BlockOffset remoteBlockOffset = blockIndexerOffset.getRemoteOffset(); + logger.info("current remote offset: {}", remoteBlockOffset); + if (remoteBlockOffset == null) { + logger.warn("offset must not null, please check blocks.mapping!!"); + return; + } + + if (remoteBlockOffset.getBlockHeight() > blockIndexerOffset.getLocalBlockOffsetHeight()) { + logger.info("indexer equalize chain blocks."); + return; + } + + fetchAndProcessBlockSequel(); + // fetchAndProcessBlocksParallel(); + } + + public void fetchAndProcessBlocksParallel() { + logger.info("Entered"); + try { + // Read chain header + BlockHeader chainHeader = blockRPCClient.getChainHeader(); + + // Calculate bulk size + long headHeight = chainHeader.getHeight(); + long bulkNumber = Math.min(headHeight - blockIndexerOffset.getLocalBlockOffsetHeight(), bulkSize); + + ConcurrentHashMap blockMap = new ConcurrentHashMap<>(); + + LongStream.rangeClosed(1, bulkNumber).parallel().forEach(index -> { + long currentBlockHeight = blockIndexerOffset.getLocalBlockOffsetHeight() + index; + + logger.info("Start Get block number: {}, currentBlockHeight: {}", index, currentBlockHeight); + Block block; + try { + block = blockRPCClient.getBlockByHeight(currentBlockHeight); + if (block == null) { + logger.warn("get block null: {}", currentBlockHeight); + return; + } + + blockMap.put(block.getHeader().getBlockHash(), block); + + logger.info("add block: {}", block.getHeader()); + } catch (Exception e) { + logger.error("Error getting block at height {}: ", currentBlockHeight, e); + } + }); + + // Process the collected blocks + inspectorHandler.upsertDagInfoFromBlocks(new ArrayList<>(blockMap.values())); + + // Update offset with the last processed block + BlockHeader lastBlockHeader = blockMap.values().stream() + .map(Block::getHeader) + .max(Comparator.comparingLong(BlockHeader::getHeight)).orElseThrow(); + + blockIndexerOffset.updateBlockOffset(lastBlockHeader.getHeight(), lastBlockHeader.getBlockHash()); + + logger.info("Index update success: {}", blockIndexerOffset); + + } catch (JSONRPC2SessionException | IOException e) { + logger.error("chain header error:", e); + } finally { + logger.info("Exited"); + } + + } + + public void fetchAndProcessBlockSequel() { + // Read chain header + try { + long bulkNumber = Math.min(blockRPCClient.getChainHeader().getHeight() - blockIndexerOffset.getLocalBlockOffsetHeight(), bulkSize); + int index = 1; + List blockList = new ArrayList<>(); + long minHeight = blockIndexerOffset.getLocalBlockOffsetHeight(); + String currentBlockHash = blockIndexerOffset.getLocalOffsetBlockHash(); + Set visit = new HashSet<>(); + Deque deque = new ArrayDeque<>(); + + long currentBlockHeight = minHeight; + + while (index <= bulkNumber) { + currentBlockHeight = minHeight + index; + + logger.info("Start Get block number: {}, currentBlockHeight: {}", index, currentBlockHeight); + Block block = blockRPCClient.getBlockByHeight(currentBlockHeight); + if (block == null) { + logger.warn("get block null: {}", currentBlockHeight); + return; + } + visit.add(block.getHeader().getBlockHash()); + fetchParentsBlock(block, visit, deque, blockList, minHeight); + while (!deque.isEmpty()) { + int size = deque.size(); + for (int i = 0; i < size; i++) { + Block block_parent = deque.removeFirst(); + fetchParentsBlock(block_parent, visit, deque, blockList, minHeight); + } + } + blockList.add(block); + index++; + + currentBlockHash = block.getHeader().getBlockHash(); + + logger.info("add block: {}", block.getHeader()); + } + inspectorHandler.upsertDagInfoFromBlocks(blockList); + + // Update offset + blockIndexerOffset.updateBlockOffset(currentBlockHeight, currentBlockHash); + + logger.info("indexer update success: {}", blockIndexerOffset); + } catch (JSONRPC2SessionException | IOException e) { + logger.error("chain header error:", e); + } + } + + void fetchParentsBlock( + Block block, + Set visit, + Deque deque, + List blockList, + long minHeight + ) throws JSONRPC2SessionException { + List parents = block.getHeader().getParentsHash(); + if (parents == null || parents.isEmpty()) { + return ; + } + + for (String parent : block.getHeader().getParentsHash()) { + if (visit.contains(parent)) { + continue; + } + visit.add(parent); + Block block_parent = blockList + .stream() + .filter(b -> b.getHeader().getBlockHash().compareToIgnoreCase(parent) == 0) + .findAny() + .orElse(null); + + if (block_parent != null) { + continue; + } + + block_parent = blockRPCClient.getBlockByHash(parent); + if (block_parent.getHeader().getHeight() >= minHeight) { + deque.addLast(block_parent); + blockList.add(block_parent); + } + } + } +} \ No newline at end of file diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java new file mode 100644 index 0000000..23d8f8d --- /dev/null +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/DagInspectorIndexerHandler.java @@ -0,0 +1,365 @@ +package org.starcoin.indexer.handler; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.starcoin.api.BlockRPCClient; +import org.starcoin.api.Result; +import org.starcoin.bean.*; +import org.starcoin.constant.Constant; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; +import org.starcoin.utils.ExceptionWrap; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +@Service +public class DagInspectorIndexerHandler { + + private static final Logger logger = LoggerFactory.getLogger(DagInspectorIndexerHandler.class); + + private static final String NODE_COLOR_GRAY = "gray"; + private static final String NODE_COLOR_RED = "red"; + private static final String NODE_COLOR_BLUE = "blue"; + + @Value("${starcoin.network}") + private String network; + + String dagInspectNodeIndex; + String dagInspectEdgeIndex; + String dagInspectHeightGroupIndex; + + @Autowired + private RestHighLevelClient client; + + @Autowired + private BlockRPCClient blockRPCClient; + + @PostConstruct + public void initIndexs() throws IOException { + dagInspectNodeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_BLOCK_INDEX); + dagInspectEdgeIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECTOR_EDGE_INDEX); + dagInspectHeightGroupIndex = ServiceUtils.createIndexIfNotExist(client, network, Constant.DAG_INSPECT_HEIGHT_GROUP_INDEX); + } + + public void upsertDagInfoFromBlocks(List blockList) throws IOException, JSONRPC2SessionException { + if (blockList.isEmpty()) { + logger.warn("block list is empty"); + return; + } + BulkRequest bulkRequest = new BulkRequest(); + + // Handle blockGhostdag Data Cache + Map dagBlockMap = bulkLoadDagBlock( + blockList.stream() + .map(block -> block.getHeader().getBlockHash()) + .collect(Collectors.toList()) + ); + + List heightGroupList = getGroupHeightSizeFromStorage( + blockList.stream() + .map(block -> block.getHeader().getHeight()) + .distinct() + .collect(Collectors.toList()) + ); + + // Build block node data + blockList.forEach(block -> { + String currentBlockHash = block.getHeader().getBlockHash(); + DagInspectorBlock dagBlock = dagBlockMap.get(currentBlockHash); + if (dagBlock == null) { + try { + BlockGhostdagData ghostdagData = blockRPCClient.getBlockGhostdagData(currentBlockHash); + dagBlock = new DagInspectorBlock(); + dagBlock.setBlockHash(currentBlockHash); + dagBlock.setTimestamp(block.getHeader().getTimestamp()); + // TODO(bobong): Check color + dagBlock.setColor(NODE_COLOR_BLUE); + dagBlock.setDaaScore(ghostdagData.getBlueScore()); + dagBlock.setHeight(block.getHeader().getHeight()); + dagBlock.setSelectedParentHash(ghostdagData.getSelectedParent()); + dagBlock.setParentIds(block.getHeader().getParentsHash() != null ? block.getHeader().getParentsHash() : new ArrayList<>()); + dagBlock.setMergeSetRedIds(ghostdagData.getMergesetReds() != null ? ghostdagData.getMergesetReds() : new ArrayList<>()); + dagBlock.setMergeSetBlueIds(ghostdagData.getMergesetBlues() != null ? ghostdagData.getMergesetBlues() : new ArrayList<>()); + + // Block is the virtual selected parent chain because the list read from block height + dagBlock.setInVirtualSelectedParentChain(true); + + Integer groupSize = getHeightGroupSizeOrDefault(heightGroupList, dagBlock.getHeight(), 0); + dagBlock.setHeightGroupIndex(groupSize); + updateGroupSize(heightGroupList, dagBlock.getHeight(), groupSize + 1); + + dagBlockMap.put(currentBlockHash, dagBlock); + + } catch (JSONRPC2SessionException e) { + throw new RuntimeException(e); + } + } else { + logger.info("Block {} already exists", currentBlockHash); + } + }); + + buildSaveDagEdgeRequest(buildEdgeDataFromDagBlockDataMaybeUpate(dagBlockMap, heightGroupList)).forEach(bulkRequest::add); + buildSaveDagHeightGroupRequest(heightGroupList).forEach(bulkRequest::add); + + // Save all data into storage + buildSaveDagBlockRequest(new ArrayList<>(dagBlockMap.values())).forEach(bulkRequest::add); + + // Bulk save + try { + BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT); + logger.info("bulk block result: {}", response.buildFailureMessage()); + } catch (IOException e) { + logger.error("bulk block error:", e); + } + } + + List buildEdgeDataFromDagBlockDataMaybeUpate( + Map dagBlockMap, + List heightGroupList + ) throws JSONRPC2SessionException { + List edgeList = new ArrayList<>(); + if (dagBlockMap.isEmpty()) { + return edgeList; + } + + List newDagBlocks = new ArrayList<>(); + for (DagInspectorBlock dagBlock : dagBlockMap.values()) { + List parentIds = dagBlock.getParentIds(); + if (parentIds == null || parentIds.isEmpty()) { + continue; + } + + for (String parentHash : parentIds) { + DagInspectorBlock parentDagBlock = dagBlockMap.get(parentHash); + if (parentDagBlock == null) { + logger.info("Parent block not found: {} ", parentHash); + parentDagBlock = getDagInspectorBlockFromHash(parentHash, heightGroupList, false); + // Put into buffer list + newDagBlocks.add(parentDagBlock); + } + DagInspectorEdge edge = new DagInspectorEdge(); + edge.setFromBlockHash(dagBlock.getBlockHash()); + edge.setToBlockHash(parentDagBlock.getBlockHash()); + edge.setFromHeight(dagBlock.getHeight()); + edge.setToHeight(parentDagBlock.getHeight()); + edge.setFromHeightGroupIndex(dagBlock.getHeightGroupIndex() == null ? 0 : dagBlock.getHeightGroupIndex()); + edge.setToHeightGroupIndex(parentDagBlock.getHeightGroupIndex() == null ? 0 :parentDagBlock.getHeightGroupIndex()); + edgeList.add(edge); + } + } + // Put into map + newDagBlocks.forEach(dagBlock -> dagBlockMap.put(dagBlock.getBlockHash(), dagBlock)); + return edgeList; + } + + public Map bulkLoadDagBlock(List blockHashList) { + Map dagBlockMap = new HashMap<>(); + SearchRequest searchRequest = new SearchRequest(dagInspectNodeIndex); + TermsQueryBuilder termQueryBuilder = QueryBuilders.termsQuery("block_hash", blockHashList); + searchRequest.source(new SearchSourceBuilder().query(termQueryBuilder)); + try { + SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT); + JSONObject obj = JSONObject.parseObject(ServiceUtils.getJsonString(response)); + List hits = obj.getJSONObject("hits").getJSONArray("hits").toJavaList(JSONObject.class); + hits.forEach(hit -> { + JSONObject source = hit.getJSONObject("sourceAsMap"); + DagInspectorBlock block = new DagInspectorBlock(); + block.setBlockHash(source.getString("block_hash")); + block.setTimestamp(source.getLong("timestamp")); + block.setColor(source.getString("color")); + block.setDaaScore(source.getLong("daa_score")); + block.setHeight(source.getLong("height")); + block.setHeightGroupIndex(source.getInteger("height_group_index")); + block.setSelectedParentHash(source.getString("selected_parent_hash")); + + JSONArray parentIds = source.getJSONArray("parent_ids"); + block.setParentIds(parentIds != null ? parentIds.toJavaList(String.class) : new ArrayList<>()); + + Boolean virtual_selected = source.getBoolean("in_virtual_selected_parent_chain"); + block.setInVirtualSelectedParentChain(virtual_selected != null ? virtual_selected : false); + + JSONArray redIds = source.getJSONArray("mergeset_red_ids"); + block.setMergeSetRedIds(redIds != null ? redIds.toJavaList(String.class) : new ArrayList<>()); + + JSONArray blueIds = source.getJSONArray("mergeset_blue_ids"); + block.setMergeSetBlueIds(blueIds != null ? blueIds.toJavaList(String.class) : new ArrayList<>()); + + dagBlockMap.put(block.getBlockHash(), block); + }); + } catch (IOException e) { + logger.error("bulkLoadInspectorBlock error:", e); + } + return dagBlockMap; + } + + List buildSaveDagBlockRequest(List blockList) { + if (blockList.isEmpty()) { + return new ArrayList<>(); + } + return blockList.stream().map(ExceptionWrap.wrap(block -> { + IndexRequest request = new IndexRequest(dagInspectNodeIndex); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.field("block_hash", block.getBlockHash()); + builder.field("height", block.getHeight()); + builder.field("timestamp", block.getTimestamp()); + builder.field("selected_parent_hash", block.getSelectedParentHash()); + builder.field("parent_ids", block.getParentIds()); + builder.field("daa_score", block.getDaaScore()); + builder.field("height_group_index", block.getHeightGroupIndex()); + builder.field("in_virtual_selected_parent_chain", block.getInVirtualSelectedParentChain()); + builder.field("mergeset_blue_ids", block.getMergeSetBlueIds()); + builder.field("mergeset_red_ids", block.getMergeSetRedIds()); + builder.field("color", block.getColor()); + } + builder.endObject(); + request.source(builder).id(block.getBlockHash()); + return request; + })).collect(Collectors.toList()); + } + + private List buildSaveDagEdgeRequest(List edgeList) throws IOException { + if (edgeList.isEmpty()) { + return new ArrayList<>(); + } + return edgeList.stream().map(ExceptionWrap.wrap(edge -> { + IndexRequest request = new IndexRequest(dagInspectEdgeIndex); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.field("from_block_hash", edge.getFromBlockHash()); + builder.field("to_block_hash", edge.getToBlockHash()); + builder.field("from_height", edge.getFromHeight()); + builder.field("to_height", edge.getToHeight()); + builder.field("from_group_index", edge.getFromHeightGroupIndex()); + builder.field("to_group_index", edge.getToHeightGroupIndex()); + } + builder.endObject(); + request.source(builder).id(String.format("%s_%s", edge.getFromBlockHash(), edge.getToBlockHash())); + return request; + })).collect(Collectors.toList()); + } + + List buildSaveDagHeightGroupRequest(List heightGroupList) { + if (heightGroupList.isEmpty()) { + return new ArrayList<>(); + } + return heightGroupList.stream().map(ExceptionWrap.wrap(dagInspectorHeightGroup -> { + IndexRequest request = new IndexRequest(dagInspectHeightGroupIndex); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field("height", dagInspectorHeightGroup.getHeight()); + builder.field("size", dagInspectorHeightGroup.getSize()); + builder.endObject(); + request.id(String.valueOf(dagInspectorHeightGroup.getHeight())).source(builder); + return request; + })).collect(Collectors.toList()); + } + + /** + * Get the size of the group at the specified height, or return the default size if the group does not exist. + * + * @param groupList + * @param height + * @param defaultSize + * @return + */ + Integer getHeightGroupSizeOrDefault(List groupList, Long height, Integer defaultSize) { + if (groupList.isEmpty()) { + return defaultSize; + } + DagInspectorHeightGroup group = groupList + .stream() + .filter(g -> g.getHeight().longValue() == height) + .findFirst() + .orElse(null); + return (group != null) ? group.getSize() : defaultSize; + } + + void updateGroupSize(List groupList, Long height, Integer newSize) { + for (DagInspectorHeightGroup group : groupList) { + if (group.getHeight().longValue() == height) { + group.setSize(newSize); + return; + } + } + DagInspectorHeightGroup group = new DagInspectorHeightGroup(); + group.setHeight(height); + group.setSize(newSize); + groupList.add(group); + } + + + private List getGroupHeightSizeFromStorage(List heights) throws IOException { + List groupList = new ArrayList<>(); + if (heights.isEmpty()) { + return groupList; + } + + SearchRequest searchRequest = new SearchRequest(dagInspectHeightGroupIndex); + TermsQueryBuilder termQueryBuilder = QueryBuilders.termsQuery("height", heights); + searchRequest.source(new SearchSourceBuilder().query(termQueryBuilder)); + SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT); + Result result = ServiceUtils.getSearchResult(response, DagInspectorHeightGroup.class); + return result.getContents(); + } + + + protected DagInspectorBlock getDagInspectorBlockFromHash( + String blockHash, + List heightGroupList, + boolean isSelectedParentChain + ) throws JSONRPC2SessionException { + + Block block = blockRPCClient.getBlockByHash(blockHash); + BlockGhostdagData blockGhostdagData = blockRPCClient.getBlockGhostdagData(blockHash); + DagInspectorBlock dagBlock = new DagInspectorBlock(); + + Long blockHeight = block.getHeader().getHeight(); + + dagBlock.setBlockHash(blockHash); + dagBlock.setTimestamp(block.getHeader().getTimestamp()); + // TODO: Check color + dagBlock.setColor(NODE_COLOR_BLUE); + dagBlock.setDaaScore(blockGhostdagData.getBlueScore() != null ? blockGhostdagData.getBlueScore() : 0); + dagBlock.setHeight(block.getHeader().getHeight()); + dagBlock.setSelectedParentHash(blockGhostdagData.getSelectedParent()); + dagBlock.setParentIds(block.getHeader().getParentsHash() != null ? block.getHeader().getParentsHash() : new ArrayList<>()); + dagBlock.setInVirtualSelectedParentChain(isSelectedParentChain); + dagBlock.setMergeSetBlueIds(blockGhostdagData.getMergesetBlues() != null ? blockGhostdagData.getMergesetBlues() : new ArrayList<>()); + dagBlock.setMergeSetRedIds(blockGhostdagData.getMergesetReds() != null ? blockGhostdagData.getMergesetReds() : new ArrayList<>()); + + // Height group list index + Integer groupSize = getHeightGroupSizeOrDefault(heightGroupList, blockHeight, 0); + dagBlock.setHeightGroupIndex(groupSize); + updateGroupSize(heightGroupList, blockHeight, groupSize + 1); + + logger.info("Get block info from hash: {}, height: {}, heightGroupIndex: {}", + blockHash, + block.getHeader().getHeight(), + dagBlock.getHeightGroupIndex() + ); + return dagBlock; + } +} diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java index f786325..0c8672f 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ElasticSearchHandler.java @@ -54,6 +54,7 @@ import org.starcoin.constant.Constant; import org.starcoin.indexer.service.AddressHolderService; import org.starcoin.indexer.service.TransactionPayloadService; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.starcoin.types.AccountAddress; import org.starcoin.types.StructTag; import org.starcoin.types.TokenCode; @@ -62,7 +63,6 @@ import org.starcoin.types.event.WithdrawEvent; import org.starcoin.utils.Hex; import org.starcoin.utils.StructTagUtil; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import javax.annotation.PostConstruct; import java.io.IOException; @@ -379,7 +379,7 @@ public void bulk(List blockList) { addTokenInfo(tokenInfo, codeStr); //add to cache tokenCache.put(codeStr, tokenInfo); - } catch (JSONRPC2SessionException | JsonProcessingException e) { + } catch (JSONRPC2SessionException e) { logger.error("flush token error:", e); } } diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandleJob.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandleJob.java new file mode 100644 index 0000000..dfb33cd --- /dev/null +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandleJob.java @@ -0,0 +1,181 @@ +package org.starcoin.indexer.handler; +import org.quartz.JobExecutionContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.quartz.QuartzJobBean; +import org.starcoin.api.BlockRPCClient; +import org.starcoin.api.TransactionRPCClient; + +import javax.annotation.PostConstruct; + + +public class IndexerHandleJob extends QuartzJobBean { + // private static final Logger logger = LoggerFactory.getLogger(IndexerHandleJob.class); + + @Value("${starcoin.indexer.bulk_size}") + private long bulkSize; + + @Autowired + private ElasticSearchHandler elasticSearchHandler; + + @Autowired + private TransactionRPCClient transactionRPCClient; + + @Autowired + private BlockRPCClient blockRPCClient; + + private LegacyMainIndexHandler legacyIndexHandler; + +// @PostConstruct +// public void initOffset() { +// localBlockOffset = elasticSearchHandler.getRemoteOffset(); +// //update current handle header +// try { +// if (localBlockOffset != null) { +// Block block = blockRPCClient.getBlockByHeight(localBlockOffset.getBlockHeight()); +// if (block != null) { +// currentHandleHeader = block.getHeader(); +// } else { +// logger.error("init offset block not exist on chain: {}", localBlockOffset); +// } +// +// } else { +// logger.warn("offset is null,init reset to genesis"); +// currentHandleHeader = blockRPCClient.getBlockByHeight(0).getHeader(); +// localBlockOffset = new BlockOffset(0, currentHandleHeader.getBlockHash()); +// elasticSearchHandler.setRemoteOffset(localBlockOffset); +// logger.info("init offset ok: {}", localBlockOffset); +// } +// } catch (JSONRPC2SessionException e) { +// logger.error("set current header error:", e); +// } +// } +// +// @Override +// protected void executeInternal(JobExecutionContext jobExecutionContext) { +// //read current offset +// if (localBlockOffset == null || currentHandleHeader == null) { +//// logger.warn("local offset error, reset it: {}, {}", localOffset, currentHandleHeader); +// initOffset(); +// } +// BlockOffset remoteBlockOffset = elasticSearchHandler.getRemoteOffset(); +// logger.info("current remote offset: {}", remoteBlockOffset); +// if (remoteBlockOffset == null) { +// logger.warn("offset must not null, please check blocks.mapping!!"); +// return; +// } +// if (remoteBlockOffset.getBlockHeight() > localBlockOffset.getBlockHeight()) { +// logger.info("indexer equalize chain blocks."); +// return; +// } +// //read head +// try { +// BlockHeader chainHeader = blockRPCClient.getChainHeader(); +// //calculate bulk size +// long headHeight = chainHeader.getHeight(); +// long bulkNumber = Math.min(headHeight - localBlockOffset.getBlockHeight(), bulkSize); +// int index = 1; +// List blockList = new ArrayList<>(); +// while (index <= bulkNumber) { +// long readNumber = localBlockOffset.getBlockHeight() + index; +// Block block = blockRPCClient.getBlockByHeight(readNumber); +// if (!block.getHeader().getParentHash().equals(currentHandleHeader.getBlockHash())) { +// //fork handle until reach forked point block +// logger.warn("Fork detected, roll back: {}, {}, {}", readNumber, block.getHeader().getParentHash(), currentHandleHeader.getBlockHash()); +// Block lastForkBlock, lastMasterBlock; +// BlockHeader forkHeader = currentHandleHeader; +// long lastMasterNumber = readNumber - 1; +// String forkHeaderParentHash = null; +// do { +// //获取分叉的block +// if (forkHeaderParentHash == null) { +// //第一次先回滚当前最高的分叉块 +// forkHeaderParentHash = forkHeader.getBlockHash(); +// } else { +// forkHeaderParentHash = forkHeader.getParentHash(); +// } +// lastForkBlock = elasticSearchHandler.getBlockContent(forkHeaderParentHash); +// if (lastForkBlock == null) { +// logger.warn("get fork block null: {}", forkHeaderParentHash); +// //read from node +// lastForkBlock = blockRPCClient.getBlockByHash(forkHeaderParentHash); +// } +// if (lastForkBlock != null) { +// elasticSearchHandler.bulkForkedUpdate(lastForkBlock); +// logger.info("rollback forked block ok: {}, {}", lastForkBlock.getHeader().getHeight(), forkHeaderParentHash); +// } else { +// //如果块取不到，先退出当前任务，下一个轮询周期再执行 +// logger.warn("get forked block is null: {}", forkHeaderParentHash); +// return; +// } +// +// //获取上一个高度主块 +// lastMasterBlock = blockRPCClient.getBlockByHeight(lastMasterNumber); +// if (lastMasterBlock != null) { +// long forkNumber = forkHeader.getHeight(); +// logger.info("fork number: {}", forkNumber); +// forkHeader = lastForkBlock.getHeader(); +// //reset offset to handled fork block +// currentHandleHeader = forkHeader; +// localBlockOffset.setBlockHeight(currentHandleHeader.getHeight()); +// localBlockOffset.setBlockHash(currentHandleHeader.getBlockHash()); +// elasticSearchHandler.setRemoteOffset(localBlockOffset); +// if (lastMasterNumber == forkNumber && lastMasterBlock.getHeader().getBlockHash().equals(forkHeaderParentHash)) { +// //find fork point +// logger.info("find fork height: {}", lastMasterNumber); +// break; +// } +// //继续找下一个分叉 +// lastMasterNumber--; +// logger.info("continue last forked block: {}", lastMasterNumber); +// } else { +// logger.warn("get last master Block null: {}", lastMasterNumber); +// } +// } while (true); +// +// logger.info("rollback handle ok: {}", localBlockOffset); +// return; //退出当前任务，重新添加从分叉点之后的block +// } +// +// //set event +// ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block); +// blockList.add(block); +// +// //update current header +// currentHandleHeader = block.getHeader(); +// index++; +// logger.debug("add block: {}", block.getHeader()); +// } +// //bulk execute +// elasticSearchHandler.bulk(blockList); +// +// //update offset +// localBlockOffset.setBlockHeight(currentHandleHeader.getHeight()); +// localBlockOffset.setBlockHash(currentHandleHeader.getBlockHash()); +// elasticSearchHandler.setRemoteOffset(localBlockOffset); +// logger.info("indexer update success: {}", localBlockOffset); +// } catch (JSONRPC2SessionException e) { +// logger.error("chain header error:", e); +// } +// } + + @PostConstruct + public void initOffset() { + if (legacyIndexHandler == null) { + legacyIndexHandler = new LegacyMainIndexHandler( + elasticSearchHandler, + transactionRPCClient, + blockRPCClient, + bulkSize + ); + legacyIndexHandler.initOffset(); + } + } + + @Override + protected void executeInternal(JobExecutionContext jobExecutionContext) { + legacyIndexHandler.execute(); + } +} \ No newline at end of file diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/LegacyMainIndexHandler.java similarity index 85% rename from starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java rename to starcoin-indexer/src/main/java/org/starcoin/indexer/handler/LegacyMainIndexHandler.java index 780f30b..f70b8e5 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/LegacyMainIndexHandler.java @@ -1,46 +1,54 @@ package org.starcoin.indexer.handler; -import com.fasterxml.jackson.core.JsonProcessingException; -import org.quartz.JobExecutionContext; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.quartz.QuartzJobBean; import org.starcoin.api.BlockRPCClient; import org.starcoin.api.TransactionRPCClient; import org.starcoin.bean.Block; import org.starcoin.bean.BlockHeader; import org.starcoin.bean.BlockOffset; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; -import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.List; - -public class IndexerHandle extends QuartzJobBean { - private static final Logger logger = LoggerFactory.getLogger(IndexerHandle.class); +public class LegacyMainIndexHandler { + private static final Logger logger = LoggerFactory.getLogger(LegacyMainIndexHandler.class); private BlockOffset localBlockOffset; - private BlockHeader currentHandleHeader; - @Value("${starcoin.network}") - private String network; - - @Value("${starcoin.indexer.bulk_size}") - private long bulkSize; + private BlockHeader currentHandleHeader; - @Autowired private ElasticSearchHandler elasticSearchHandler; - @Autowired private TransactionRPCClient transactionRPCClient; - @Autowired private BlockRPCClient blockRPCClient; - @PostConstruct + private Long bulkSize; + + public LegacyMainIndexHandler( + ElasticSearchHandler elasticSearchHandler, + TransactionRPCClient transactionRPCClient, + BlockRPCClient blockRPCClient, + Long bulkSize + ) { + this.elasticSearchHandler = elasticSearchHandler; + this.transactionRPCClient = transactionRPCClient; + this.blockRPCClient = blockRPCClient; + this.bulkSize = bulkSize; + } + + public void initOffsetWith(Long height, String blockHash) throws JSONRPC2SessionException { + localBlockOffset = new BlockOffset(height, blockHash); + Block block = blockRPCClient.getBlockByHeight(height); + if (block != null) { + currentHandleHeader = block.getHeader(); + } else { + logger.error("init offset block not exist on chain: {}", localBlockOffset); + } + } + public void initOffset() { localBlockOffset = elasticSearchHandler.getRemoteOffset(); //update current handle header @@ -65,8 +73,8 @@ public void initOffset() { } } - @Override - protected void executeInternal(JobExecutionContext jobExecutionContext) { + + public void execute() { //read current offset if (localBlockOffset == null || currentHandleHeader == null) { // logger.warn("local offset error, reset it: {}, {}", localOffset, currentHandleHeader); @@ -150,8 +158,11 @@ protected void executeInternal(JobExecutionContext jobExecutionContext) { logger.info("rollback handle ok: {}", localBlockOffset); return; //退出当前任务，重新添加从分叉点之后的block } + //set event - ServiceUtils.addBlockToList(transactionRPCClient, blockList, block); + ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block); + blockList.add(block); + //update current header currentHandleHeader = block.getHeader(); index++; @@ -159,13 +170,14 @@ protected void executeInternal(JobExecutionContext jobExecutionContext) { } //bulk execute elasticSearchHandler.bulk(blockList); + //update offset localBlockOffset.setBlockHeight(currentHandleHeader.getHeight()); localBlockOffset.setBlockHash(currentHandleHeader.getBlockHash()); elasticSearchHandler.setRemoteOffset(localBlockOffset); logger.info("indexer update success: {}", localBlockOffset); - } catch (JSONRPC2SessionException | JsonProcessingException e) { + } catch (JSONRPC2SessionException e) { logger.error("chain header error:", e); } } -} \ No newline at end of file +} diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java index 761bd13..1cfbe6f 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapHandle.java @@ -19,7 +19,7 @@ import org.starcoin.api.TokenContractRPCClient; import org.starcoin.bean.*; import org.starcoin.constant.Constant; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import javax.annotation.PostConstruct; import java.io.IOException; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapIndexer.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapIndexer.java index ab5a4cd..a706927 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapIndexer.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/MarketCapIndexer.java @@ -13,8 +13,10 @@ public class MarketCapIndexer extends QuartzJobBean { private static final Logger logger = LoggerFactory.getLogger(MarketCapIndexer.class); + @Autowired private MarketCapHandle handle; + @Autowired private AddressHolderService addressHolderService; diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java index 75f0d27..17bcdad 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java @@ -10,8 +10,7 @@ import org.starcoin.api.TransactionRPCClient; import org.starcoin.bean.Block; import org.starcoin.bean.BlockHeader; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; - +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import java.util.ArrayList; import java.util.HashMap; @@ -67,7 +66,7 @@ public boolean autoRepair(long startNumber, int count) { logger.info("repair too fast: {}", startNumber); return false; } - } catch (JSONRPC2SessionException | JsonProcessingException e) { + } catch (JSONRPC2SessionException e) { logger.error("get master error: ", e); return false; } @@ -101,7 +100,8 @@ public boolean autoRepair(long startNumber, int count) { if (esBlock == null) { logger.warn("es block not exist: {}", block.getHeader().getHeight()); try { - ServiceUtils.addBlockToList(transactionRPCClient, blockList, block); + ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block); + blockList.add(block); } catch (JSONRPC2SessionException e) { logger.error("add block err:", e); } @@ -110,7 +110,8 @@ public boolean autoRepair(long startNumber, int count) { if (!block.getHeader().getBlockHash().equals(esBlock.getHeader().getBlockHash())) { // fork block try { - ServiceUtils.addBlockToList(transactionRPCClient, blockList, block); + ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block); + blockList.add(block); } catch (JSONRPC2SessionException e) { logger.error("add fix block err:", e); } @@ -142,7 +143,8 @@ public void repair(long blockNumber) { if (!blockOnChain.getHeader().getBlockHash().equals(blockOnEs.getHeader().getBlockHash())) { // update block List blockList = new ArrayList<>(); - ServiceUtils.addBlockToList(transactionRPCClient, blockList, blockOnChain); + ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, blockOnChain); + blockList.add(blockOnChain); elasticSearchHandler.updateBlock(blockList); logger.info("repair ok: {}", blockNumber); } else { diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java index 1695639..2a053ed 100644 --- a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java +++ b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java @@ -7,6 +7,7 @@ import com.novi.bcs.BcsDeserializer; import com.novi.serde.Bytes; import com.novi.serde.DeserializationError; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; @@ -28,7 +29,6 @@ import org.starcoin.types.StructTag; import org.starcoin.types.TransactionPayload; import org.starcoin.utils.*; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import java.io.IOException; import java.math.BigDecimal; @@ -180,7 +180,7 @@ static TokenInfo getTokenInfo(StateRPCClient stateRPCClient, String tokenCode) { if (tokenInfo != null) { tokenCache.put(tokenCode, tokenInfo); } - } catch (JSONRPC2SessionException | JsonProcessingException e) { + } catch (JSONRPC2SessionException e) { logger.error("get token info error:", e); } } @@ -198,7 +198,11 @@ public static BigDecimal divideScalingFactor(StateRPCClient stateRPCClient, Stri return actualValue; } - public static Map getTokenReserveFromState(StateRPCClient stateRPCClient, String contractAddress, String stateRoot) throws JSONRPC2SessionException { + public static Map getTokenReserveFromState( + StateRPCClient stateRPCClient, + String contractAddress, + String stateRoot + ) throws JSONRPC2SessionException { ListResource resource = stateRPCClient.getState(contractAddress, true, stateRoot); // System.out.println(resource); Map poolReserves = new HashMap<>(); @@ -220,11 +224,19 @@ public static Map getTokenReserveFromState(StateRPCClient stat return poolReserves; } - static void addBlockToList(TransactionRPCClient transactionRPCClient, List blockList, Block block) throws JSONRPC2SessionException { - List transactionList = transactionRPCClient.getBlockTransactions(block.getHeader().getBlockHash()); - if (transactionList == null) { + public static void fetchTransactionsForBlock(TransactionRPCClient transactionRPCClient, Block block) throws JSONRPC2SessionException { + List transactionList; + try { + transactionList = transactionRPCClient.getBlockTransactions(block.getHeader().getBlockHash()); + } catch (JSONRPC2SessionException e) { + logger.error("get block txn error:", e); return; } + + if (transactionList == null || transactionList.isEmpty()) { + return; + } + for (Transaction transaction : transactionList) { BlockMetadata metadata; Transaction userTransaction = transactionRPCClient.getTransactionByHash(transaction.getTransactionHash()); @@ -268,6 +280,6 @@ static void addBlockToList(TransactionRPCClient transactionRPCClient, List parents_hash) { + Block block = new Block(); + BlockHeader header = new BlockHeader(); + header.setHeight(height); + header.setBlockHash(block_hash); + header.setParentHash(parent_hash); + header.setParentsHash(new Vector<>(parents_hash)); + block.setHeader(header); + return block; + } + + /** + * Test of testUpsertFromBlocks method, of class DagInspector. + */ + @Test + public void testTestUpsertFromBlocks() throws Exception { + + // Initialize blocks + // Block 1 (Genesis) + // Block 1 <- Block 2 + // Block 1 <- Block 3 + // Block 2 <- Block 4 + // Block 3 <- Block 4 + // Block 4 <- Block 5 + // Block 2 <- Block 6 + // Block 3 <- Block 6 + List blockList = new ArrayList<>(); + blockList.add(new_test_block("block_hash_1", 1L, "", new Vector<>())); + blockList.add(new_test_block("block_hash_2", 2L, "block_hash_1", List.of("block_hash_1"))); + blockList.add(new_test_block("block_hash_3", 2L, "block_hash_1", List.of("block_hash_1"))); + blockList.add(new_test_block("block_hash_4", 3L, "block_hash_3", List.of("block_hash_3"))); + blockList.add(new_test_block("block_hash_5", 4L, "block_hash_4", List.of("block_hash_4"))); + blockList.add(new_test_block("block_hash_6", 5L, "block_hash_2", List.of("block_hash_2", "block_hash_3"))); + dagInspectorHandler.upsertDagInfoFromBlocks(blockList); + } + + +} diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java index b407a56..723bb32 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/SwapIndexerTest.java @@ -14,8 +14,8 @@ import org.starcoin.bean.SwapTransaction; import org.starcoin.bean.SwapType; import org.starcoin.indexer.service.SwapTxnService; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.starcoin.utils.SwapApiClient; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; import java.io.IOException; import java.math.BigDecimal; diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java index 3cfe225..c88d23c 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/handler/TvlTest.java @@ -1,11 +1,10 @@ package org.starcoin.indexer.handler; +import com.thetransactioncompany.jsonrpc2.client.JSONRPC2SessionException; import org.junit.Test; import org.starcoin.api.ContractRPCClient; import org.starcoin.api.StateRPCClient; import org.starcoin.bean.ContractCall; -import org.starcoin.jsonrpc.client.JSONRPC2SessionException; - import java.net.MalformedURLException; import java.net.URL; diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexHandlerJobTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexHandlerJobTest.java new file mode 100644 index 0000000..228d85b --- /dev/null +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexHandlerJobTest.java @@ -0,0 +1,31 @@ +package org.starcoin.indexer.test; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.starcoin.api.BlockRPCClient; +import org.starcoin.api.TransactionRPCClient; +import org.starcoin.indexer.handler.ElasticSearchHandler; +import org.starcoin.indexer.handler.LegacyMainIndexHandler; + +public class IndexHandlerJobTest extends IndexerLogicBaseTest { + + @Value("${starcoin.indexer.bulk_size}") + private long bulkSize; + + @Autowired + private ElasticSearchHandler elasticSearchHandler; + + @Autowired + private TransactionRPCClient transactionRPCClient; + + @Autowired + private BlockRPCClient blockRPCClient; + + @Test + public void testIndexerHandle() throws Exception { + LegacyMainIndexHandler legacyMainIndexHandler = new LegacyMainIndexHandler(elasticSearchHandler, transactionRPCClient, blockRPCClient, bulkSize); + legacyMainIndexHandler.initOffsetWith(2713240L, "0x4d58d276809bd061ba422a4699c90c790efc5dd1b6d40e8c2adb0b1cb98dfafd"); + legacyMainIndexHandler.execute(); + } +} diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexerLogicBaseTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexerLogicBaseTest.java new file mode 100644 index 0000000..f559aac --- /dev/null +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexerLogicBaseTest.java @@ -0,0 +1,10 @@ +package org.starcoin.indexer.test; + +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; +import org.starcoin.indexer.IndexerApplication; + +@SpringBootTest(classes = IndexerApplication.class) +@TestPropertySource(locations = "classpath:application-integrationtest.properties") +public class IndexerLogicBaseTest { +} diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/ServiceUtilsTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/ServiceUtilsTest.java new file mode 100644 index 0000000..7d88754 --- /dev/null +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/ServiceUtilsTest.java @@ -0,0 +1,45 @@ +package org.starcoin.indexer.utils; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.starcoin.api.TransactionRPCClient; +import org.starcoin.bean.Block; +import org.starcoin.bean.BlockHeader; +import org.starcoin.indexer.handler.ServiceUtils; +import org.starcoin.indexer.test.IndexerLogicBaseTest; +import org.starcoin.utils.ExceptionWrap; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class ServiceUtilsTest extends IndexerLogicBaseTest { + + @Autowired + private TransactionRPCClient transactionRPCClient; + + @Test + void testFetchTransactionsForBlock() { + List blockHashes = Arrays.asList( + "0x6590d6769e9837e5fbc116daf53efa809f996e16cf55d4a713695094ac79aada", + "0x6b7ca9e9a1aa3f414caab285f214179108d5cf3fc2864efd9e7b5ed4461e7e80", + "0x042218a4750e696942fab2a875479a77f7c3c48049acb400a80c77b24aa06d36", + "0x721c15b57a22687dd90a439d85918f92f49937eef48c7c7afb72a7da43845589", + "0x9a68fd546aec8fdad955c0626cb9b20be121f932bb69f8a523f029248b125f51", + "0x73c0d5b604e130dedf0ecb77102e91de521bf38097b80d3e64c7ed58bc85c232", + "0x9161c9b51a7817cc2a18c190c99b40e96cb7cf4666ad63a7ffa33b2671c3aa31", + "0x020aadcabfbd57cc4c9059d16a91f8bc944b6c5c110d4fc72e6c346658035ecc" + ); + List blocks = blockHashes.stream().map(ExceptionWrap.wrap(blockHash -> { + Block block = new Block(); + BlockHeader header = new BlockHeader(); + header.setBlockHash(blockHash); + block.setHeader(header); + ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block); + System.out.println("Block: " + blockHash); + return block; + })).collect(Collectors.toList()); + blocks.forEach(System.out::println); + + } +} diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/SwapApiClientTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/SwapApiClientTest.java index 7709bab..e6a4e0c 100644 --- a/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/SwapApiClientTest.java +++ b/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/SwapApiClientTest.java @@ -2,21 +2,15 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.TestPropertySource; -import org.starcoin.indexer.IndexerApplication; import org.starcoin.bean.LiquidityPoolInfo; import org.starcoin.bean.SwapToken; +import org.starcoin.indexer.test.IndexerLogicBaseTest; import org.starcoin.utils.SwapApiClient; import java.io.IOException; import java.util.List; -@SpringBootTest( - classes = IndexerApplication.class) -@TestPropertySource( - locations = "classpath:application-integrationtest.properties") -class SwapApiClientTest { +class SwapApiClientTest extends IndexerLogicBaseTest { @Autowired private SwapApiClient swapApiClient; diff --git a/starcoin-indexer/src/test/resources/application-integrationtest.properties b/starcoin-indexer/src/test/resources/application-integrationtest.properties index 63a2905..cdbe740 100644 --- a/starcoin-indexer/src/test/resources/application-integrationtest.properties +++ b/starcoin-indexer/src/test/resources/application-integrationtest.properties @@ -1,20 +1,27 @@ -starcoin.seeds=localhost -starcoin.network=main +# server.port=8300 +starcoin.seeds=vega.seed.starcoin.org +starcoin.network=vega starcoin.bg_task.jobs= -starcoin.indexer.bulk_size=3 +starcoin.indexer.bulk_size=100 +starcoin.indexer.txn_offset=0 jasypt.encryptor.bean=encryptorBean jasypt.encryptor.password= elasticsearch.host=localhost -elasticsearch.port=9200 elasticsearch.protocol=http -elasticsearch.username=elastic +elasticsearch.port=9200 +elasticsearch.username= elasticsearch.connTimeout=10000 elasticsearch.socketTimeout=10000 elasticsearch.connectionRequestTimeout=2000 swap.contract.address=0x8c109349c6bd91411d6bc962e080c4a3 starcoin.swap.api.url=https://swap-api.starswap.xyz -spring.datasource.url=jdbc:postgresql://localhost/starcoin logging.file.name=logs/indexer.log + +## Unittest config +spring.profiles=unittest +spring.datasource.hikari.minimum-idle=1 +spring.datasource.hikari.maximum-pool-size=2 +spring.datasource.url=jdbc:postgresql://localhost/starcoin spring.datasource.username=starcoin -spring.datasource.password= -spring.jpa.properties.hibernate.default_schema=main \ No newline at end of file +spring.datasource.password=starcoin +spring.jpa.properties.hibernate.default_schema=vega \ No newline at end of file diff --git a/starcoin-scan-api/local.env b/starcoin-scan-api/local.env new file mode 100644 index 0000000..fa26630 --- /dev/null +++ b/starcoin-scan-api/local.env @@ -0,0 +1,13 @@ +STARCOIN_ES_URL=localhost +STARCOIN_ES_PROTOCOL=http +STARCOIN_ES_PORT=9200 +STARCOIN_ES_USER= +STARCOIN_ES_INDEX_VERSION= +STARCOIN_ES_PWD= +MAIN_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=main +BARNARD_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=barnard +HALLEY_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=halley +DS_URL=jdbc:postgresql://localhost/starcoin +STARCOIN_USER_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=starcoin_user +DB_USER_NAME=starcoin +DB_PWD=starcoin \ No newline at end of file diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java index fca6985..8f40a1d 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/BlockV2Controller.java @@ -7,9 +7,14 @@ import org.starcoin.api.Result; import org.starcoin.bean.Block; import org.starcoin.bean.BlockInfoEntity; +import org.starcoin.bean.DagInspectorBlock; import org.starcoin.bean.UncleBlock; import org.starcoin.scan.service.BlockInfoService; import org.starcoin.scan.service.BlockService; +import org.starcoin.scan.service.DagInspectorService; +import org.starcoin.scan.service.vo.BlockVo; + +import java.util.List; @Api(tags = "block") @RestController @@ -21,64 +26,107 @@ public class BlockV2Controller { @Autowired private BlockInfoService blockInfoService; + @Autowired + private DagInspectorService dagInspectorService; + @ApiOperation("get block by ID") @GetMapping("/{network}/") - public Block getBlock(@PathVariable("network") String network, @RequestParam String id) throws Exception { + public Block getBlock( + @PathVariable("network") String network, + @RequestParam String id + ) throws Exception { return blockService.getBlock(network, id); } @ApiOperation("get block by hash") @GetMapping("/{network}/hash/{hash}") - public Block getBlockByHash(@PathVariable("network") String network, @PathVariable("hash") String hash) { + public Block getBlockByHash( + @PathVariable("network") String network, + @PathVariable("hash") String hash + ) { return blockService.getBlockByHash(network, hash); } @ApiOperation("get block_info by hash") @GetMapping("/info/{network}/hash/{hash}") - public BlockInfoEntity getBlockInfoByHash(@PathVariable("network") String network, @PathVariable("hash") String hash) { + public BlockInfoEntity getBlockInfoByHash( + @PathVariable("network") String network, + @PathVariable("hash") String hash + ) { return blockInfoService.getBlockInfoByHash(network, hash); } @ApiOperation("get block by height") @GetMapping("/{network}/height/{height}") - public Block getBlockByHeight(@PathVariable("network") String network, @PathVariable("height") long height) { - return blockService.getBlockByHeight(network, height); + public BlockVo getBlockByHeight( + @PathVariable("network") String network, + @PathVariable("height") long height + ) { + Block block = blockService.getBlockByHeight(network, height); + if (block == null) { + return new BlockVo(); + } + BlockVo blockVo = BlockVo.from(block); + List dagBlocks = dagInspectorService.getBlocksByHeight(network, height); + if (dagBlocks == null || dagBlocks.isEmpty()) { + return blockVo; + } + dagBlocks.stream() + .filter(dagBlock -> dagBlock.getBlockHash().equals(blockVo.getId())) + .findFirst().ifPresent(dagBlock -> { + blockVo.setDaaScore(dagBlock.getDaaScore()); + blockVo.setMergedBlueset(dagBlock.getMergeSetBlueIds()); + blockVo.setHeightgroupIndex(dagBlock.getHeightGroupIndex()); + }); + return blockVo; } @ApiOperation("get block list") @GetMapping("/{network}/page/{page}") - public Result getRangeBlocks(@PathVariable("network") String network, @PathVariable("page") int page, - @RequestParam(value = "count", required = false, defaultValue = "20") int count + public Result getRangeBlocks( + @PathVariable("network") String network, + @PathVariable("page") int page, + @RequestParam(value = "count", required = false, defaultValue = "20") int count ) { return blockService.getRange(network, page, count, 0); } @ApiOperation("get block list by start height") @GetMapping("/{network}/start_height/") - public Result getBlocksStartWith(@PathVariable("network") String network, @RequestParam(value = "start_height", required = false, defaultValue = "0") long start_height, - @RequestParam(value = "page", required = false, defaultValue = "1") int page, - @RequestParam(value = "count", required = false, defaultValue = "20") int count + public Result getBlocksStartWith( + @PathVariable("network") String network, + @RequestParam(value = "start_height", required = false, defaultValue = "0") long start_height, + @RequestParam(value = "page", required = false, defaultValue = "1") int page, + @RequestParam(value = "count", required = false, defaultValue = "20") int count ) { return blockService.getBlocksStartWith(network, start_height, page, count); } @ApiOperation("get uncle block list") @GetMapping("/{network}/uncle/page/{page}") - public Result getRangeUncleBlocks(@PathVariable("network") String network, @PathVariable("page") int page, - @RequestParam(value = "count", required = false, defaultValue = "20") int count, - @RequestParam(value = "total", required = false, defaultValue = "0") int start_height) { + public Result getRangeUncleBlocks( + @PathVariable("network") String network, @PathVariable("page") int page, + @RequestParam(value = "count", required = false, defaultValue = "20") int count, + @RequestParam(value = "total", required = false, defaultValue = "0") int start_height + ) { return blockService.getUnclesRange(network, page, count, start_height); } @ApiOperation("get uncle block by height") @GetMapping("/{network}/uncle/height/{height}") - public UncleBlock getUncleBlockByHeight(@PathVariable("network") String network, @PathVariable("height") long height) { + public UncleBlock getUncleBlockByHeight( + @PathVariable("network") String network, + @PathVariable("height") long height + ) { return blockService.getUncleBlockByHeight(network, height); } @ApiOperation("get uncle block by hash") @GetMapping("/{network}/uncle/hash/{hash}") - public UncleBlock getUncleBlockByHash(@PathVariable("network") String network, @PathVariable("hash") String hash) { + public UncleBlock getUncleBlockByHash( + @PathVariable("network") String network, + @PathVariable("hash") String hash + ) { return blockService.getUncleBlockByHash(network, hash); } } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java new file mode 100644 index 0000000..c18b610 --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/DagInspectorController.java @@ -0,0 +1,59 @@ +package org.starcoin.scan.controller; + +import io.swagger.annotations.Api; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import org.starcoin.scan.service.DagInspectorService; +import org.starcoin.scan.service.vo.DIAppConfigVo; +import org.starcoin.scan.service.vo.DIBlocksAndEdgesAndHeightGroupsVo; + +@Api(tags = "dag-inspector") +@RestController +@RequestMapping("v2/dag-inspector") +public class DagInspectorController { + + @Autowired + DagInspectorService dagInspectorService; + + @GetMapping("/{network}/blocksBetweenHeights") + public DIBlocksAndEdgesAndHeightGroupsVo getBlocksBetweenHeights( + @PathVariable("network") String network, + @RequestParam Long startHeight, + @RequestParam Long endHeight + ) throws Exception { + return dagInspectorService.getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); + } + + @GetMapping("/{network}/head") + public DIBlocksAndEdgesAndHeightGroupsVo head( + @PathVariable("network") String network, + @RequestParam Long heightDifference + ) throws Exception { + return dagInspectorService.getHead(network, heightDifference); + } + + @GetMapping("/{network}/blockHash") + public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash( + @PathVariable("network") String network, + @RequestParam String blockHash, + @RequestParam Long heightDifference + ) throws Exception { + return dagInspectorService.getBlockHash(network, blockHash, heightDifference); + } + + @GetMapping("/{network}/blockDAAScore") + public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore( + @PathVariable("network") String network, + @RequestParam Long blockDAAScore, + @RequestParam Long heightDifference + ) throws Exception { + return dagInspectorService.getBlockDAAScore(network, blockDAAScore, heightDifference); + } + + @GetMapping("/{network}/appConfig") + public DIAppConfigVo getAppConfig( + @PathVariable("network") String network + ) throws Exception { + return dagInspectorService.getAppConfig(network); + } +} diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TokenController.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TokenController.java index 9488d38..8457d6a 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TokenController.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TokenController.java @@ -27,7 +27,7 @@ public class TokenController { @ApiOperation("get token aggregate stat list") @GetMapping("/{network}/stats/{page}") public Result getAggregate(@PathVariable("network") String network, @PathVariable("page") int page, - @RequestParam(value = "count", required = false, defaultValue = "50") int count) { + @RequestParam(value = "count", required = false, defaultValue = "10") int count) { return tokenService.tokenAggregateList(network, page, count); } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionController.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionController.java index 12ae212..97c4216 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionController.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionController.java @@ -6,7 +6,7 @@ import org.starcoin.bean.Event; import org.starcoin.bean.PendingTransaction; import org.starcoin.scan.service.TransactionService; -import org.starcoin.scan.service.TransactionWithEvent; +import org.starcoin.scan.service.vo.TransactionWithEvent; import java.io.IOException; diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionV2Controller.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionV2Controller.java index 85e926b..ccd33bf 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionV2Controller.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/controller/TransactionV2Controller.java @@ -10,7 +10,7 @@ import org.starcoin.bean.TokenTransfer; import org.starcoin.bean.Transfer; import org.starcoin.scan.service.TransactionService; -import org.starcoin.scan.service.TransactionWithEvent; +import org.starcoin.scan.service.vo.TransactionWithEvent; import java.io.IOException; diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java index 39b0b02..2ae0fd4 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/BlockService.java @@ -7,7 +7,9 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; @@ -97,17 +99,9 @@ public Result getRange(String network, int page, int count, int start_hei searchSourceBuilder.query(QueryBuilders.matchAllQuery()); //page size searchSourceBuilder.size(count); - //begin offset - int offset; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS && start_height > 0) { - offset = start_height - (page - 1) * count; - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } + //set offset + TransactionService.setBlockSearchBuildFrom(page, count, start_height, searchSourceBuilder); + searchSourceBuilder.sort("header.number", SortOrder.DESC); searchSourceBuilder.trackTotalHits(true); searchRequest.source(searchSourceBuilder); @@ -207,17 +201,9 @@ public Result getUnclesRange(String network, int page, int count, in searchSourceBuilder.query(QueryBuilders.matchAllQuery()); //page size searchSourceBuilder.size(count); - //begin offset - int offset; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS && start_height > 0) { - offset = start_height - (page - 1) * count; - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } + //set offset + TransactionService.setBlockSearchBuildFrom(page, count, start_height, searchSourceBuilder); + searchSourceBuilder.sort("uncle_block_number", SortOrder.DESC); searchSourceBuilder.trackTotalHits(true); searchRequest.source(searchSourceBuilder); @@ -230,6 +216,4 @@ public Result getUnclesRange(String network, int page, int count, in } return ServiceUtils.getSearchResult(searchResponse, UncleBlock.class); } - - } diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java new file mode 100644 index 0000000..8afe770 --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/DagInspectorService.java @@ -0,0 +1,276 @@ +package org.starcoin.scan.service; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.starcoin.api.Result; +import org.starcoin.bean.DagInspectorBlock; +import org.starcoin.bean.DagInspectorEdge; +import org.starcoin.bean.DagInspectorHeightGroup; +import org.starcoin.constant.Constant; +import org.starcoin.scan.service.vo.*; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@Service +public class DagInspectorService extends BaseService { + + private static final Logger logger = LoggerFactory.getLogger(BlockService.class); + + @Autowired + BlockService blockService; + + @Autowired + private RestHighLevelClient client; + + + public DIBlocksAndEdgesAndHeightGroupsVo getBlocksAndEdgesAndHeightGroups(String network, Long startHeight, Long endHeight) throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo groups = new DIBlocksAndEdgesAndHeightGroupsVo(); + + List blockList = getBlockList(network, startHeight, endHeight); + if (blockList == null || blockList.isEmpty()) { + return groups; + } + + groups.setBlocks(blockList); + + List blockEdges = getEdgeList(network, startHeight, endHeight); + groups.setEdges(blockEdges); + + Set heightList = + blockList.stream() + .map(DagInspectorBlock::getHeight) + .collect(Collectors.toSet()); + + // Add edges FromHeight and ToHeight + heightList.addAll( + blockEdges.stream() + .map(DagInspectorEdge::getFromHeight) + .collect(Collectors.toSet()) + ); + heightList.addAll( + blockEdges.stream() + .map(DagInspectorEdge::getToHeight) + .collect(Collectors.toSet()) + ); + groups.setHeightGroups(getHeightGroup(network, heightList)); + + return groups; + } + + public DIBlocksAndEdgesAndHeightGroupsVo getBlockHash( + String network, + String targetHash, + Long heightDifference + ) throws IOException { + DagInspectorBlock block = getBlockWithHashFromStorage(network, targetHash); + if (block == null) { + throw new RuntimeException("Cannot find block by block hash"); + } + long blockHeight = block.getHeight(); + long startHeight = Math.max(blockHeight - heightDifference, 0L); + Long endHeight = blockHeight + heightDifference; + return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); + } + + public DIBlocksAndEdgesAndHeightGroupsVo getBlockDAAScore( + String network, + Long targetDAAScore, + Long heightDifference + ) throws IOException { + DagInspectorBlock block = getHeightWithDAAScoreFromStorage(network, targetDAAScore); + if (block == null) { + throw new RuntimeException("Cannot find block by block hash"); + } + + long blockHeight = block.getHeight(); + long startHeight = Math.max(blockHeight - heightDifference, 0L); + Long endHeight = blockHeight + heightDifference; + return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); + } + + public DIBlocksAndEdgesAndHeightGroupsVo getHead(String network, Long heightDifference) throws IOException { + long endHeight = getMaxHeightFromStorage(network); + long startHeight = Math.max(endHeight - heightDifference, 0L); + return getBlocksAndEdgesAndHeightGroups(network, startHeight, endHeight); + } + + /** + * Get block height from Elastic search storage + * + * @param network + * @param startHeight + * @param endHeight + * @return + */ + private List getBlockList(String network, Long startHeight, Long endHeight) throws ElasticsearchException, IOException { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK_INDEX)); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.query(QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery("height").gte(startHeight).lte(endHeight))); + sourceBuilder.sort("height", SortOrder.ASC); + //sourceBuilder.size(endHeight.intValue() - startHeight.intValue()); + sourceBuilder.size(1000); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); + return result.getContents().stream().peek(block -> { + if (block.getMergeSetBlueIds() == null) { + block.setMergeSetBlueIds(new ArrayList<>()); + } + + if (block.getMergeSetRedIds() == null) { + block.setMergeSetRedIds(new ArrayList<>()); + } + }).collect(Collectors.toList()); + } + + /** + * Get Edge list from ElasticSearch storage + * + * @param network + * @param startHeight + * @param endHeight + * @return + */ + List getEdgeList(String network, Long startHeight, Long endHeight) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_EDGE_INDEX)); + RangeQueryBuilder fromHeightQuery = QueryBuilders.rangeQuery("from_height").gte(startHeight).lte(endHeight); + RangeQueryBuilder toHeightQuery = QueryBuilders.rangeQuery("to_height").gte(startHeight).lte(endHeight); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query(QueryBuilders.boolQuery().must(fromHeightQuery).must(toHeightQuery)) + .sort("to_height", SortOrder.ASC) + .size(100); + + searchRequest.source(sourceBuilder); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("getEdgeList failed, startHeight: {}, endHeight: {}", startHeight, endHeight, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorEdge.class); + return result.getContents(); + } + + /** + * Get heights + * + * @param network + * @param heights + * @return + */ + protected List getHeightGroup(String network, Set heights) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECT_HEIGHT_GROUP_INDEX)); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .query(QueryBuilders.termsQuery("height", heights)); + //sourceBuilder.size(heights.size()); + sourceBuilder.size(1000); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("getHeightGroup failed, heights: {}", heights, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorHeightGroup.class); + return result.getContents(); + } + + public DIAppConfigVo getAppConfig(String network) { + DIAppConfigVo configVo = new DIAppConfigVo(); + configVo.setApiVersion("2.0.0"); + configVo.setNetwork(network); + configVo.setProcessingVersion("0.0.1"); + configVo.setWebVersion("0.0.1"); + configVo.setStarcoinVersion("2.0.0"); + return configVo; + } + + private Long getMaxHeightFromStorage(String network) throws IOException { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK_INDEX)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchSourceBuilder.sort("height", SortOrder.DESC); + searchSourceBuilder.size(1); // We only need the top 1 result + searchRequest.source(searchSourceBuilder); + + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); + List contents = result.getContents(); + if (!contents.isEmpty()) { + return contents.get(0).getHeight(); + } + return 0L; + } + + private DagInspectorBlock getBlockWithHashFromStorage(String network, String blockHash) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK_INDEX)); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.should(QueryBuilders.termQuery("block_hash", blockHash)); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("getBlockWithHashFromStorage failed, blockHash: {}", blockHash, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); + return result.getContents().get(0); + } + + private DagInspectorBlock getHeightWithDAAScoreFromStorage(String network, Long daaScore) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK_INDEX)); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.should(QueryBuilders.termQuery("daa_score", daaScore)); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("getBlockWithHashFromStorage failed, blockHash: {}", daaScore, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); + return result.getContents().get(0); + } + + public List getBlocksByHeight(String network, Long height) { + SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.DAG_INSPECTOR_BLOCK_INDEX)); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.should(QueryBuilders.termQuery("height", height)); + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException | org.elasticsearch.ElasticsearchStatusException e) { + logger.error("getBlocksByHeight failed, blockHash: {}", height, e); + return null; + } + Result result = ServiceUtils.getSearchResult(searchResponse, DagInspectorBlock.class); + return result.getContents(); + } + +} diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java index e01ce06..1829434 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TokenService.java @@ -75,7 +75,7 @@ public Result tokenAggregateList(String network, int page, i Result volumes = tokenVolumeList(network, page, count); Map volumeMap = getVolumeMap(network, volumes); //get market cap - Result market = tokenMarketCap(network, page, ELASTICSEARCH_MAX_HITS); + Result market = tokenMarketCap(network, 1, ELASTICSEARCH_MAX_HITS); Map marketMap = getMarketMap(market); List viewList = new ArrayList<>(); for (TokenStatistic tokenStatistic : holderContents) { @@ -94,7 +94,7 @@ public Result tokenAggregateList(String network, int page, i viewList.add(view); } result.setContents(viewList); - result.setTotal(holders.getTotal()); + result.setTotal(market.getTotal()); return result; } @@ -270,8 +270,8 @@ public Result tokenHolderList(String network, int page, int coun searchSourceBuilder.query(QueryBuilders.matchAllQuery()); TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("holders") .field("type_tag.keyword") + .size(ELASTICSEARCH_MAX_HITS) .order(BucketOrder.aggregation("address_holders", false)) -// .size(count) .subAggregation(AggregationBuilders.count("address_holders").field("address.keyword")) .subAggregation(new BucketSortPipelineAggregationBuilder("bucket_field", null).from(offset).size(count)); @@ -297,16 +297,8 @@ public Result tokenMarketCap(String network, int page, int count .must(QueryBuilders.matchAllQuery()); searchSourceBuilder.query(queryBuilder); //page size - int offset = 0; searchSourceBuilder.size(count); - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS) { - searchSourceBuilder.searchAfter(new Object[]{offset}); - } - } - //begin offset - searchSourceBuilder.from(offset); + TransactionService.setSearchBuildFrom(page, count, searchSourceBuilder); searchSourceBuilder.trackTotalHits(true); searchRequest.source(searchSourceBuilder); searchSourceBuilder.timeout(new TimeValue(20, TimeUnit.SECONDS)); @@ -368,7 +360,6 @@ public Result tokenVolumeList(String network, int page, int coun logger.error("get token volume error:", e); return null; } - } public void loadTokenInfo(String network) { @@ -446,6 +437,7 @@ private Result searchStatistic(SearchResponse searchResponse, St } } result.setContents(statistics); + result.setTotal(searchResponse.getHits().getTotalHits().value); return result; } @@ -453,18 +445,21 @@ public Result getHoldersByToken(String network, int page, int c SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.ADDRESS_INDEX)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(count); - //begin offset + //set offset + boolean shouldAfter = false; int offset = 0; if (page > 1) { offset = (page - 1) * count; if (offset >= ELASTICSEARCH_MAX_HITS) { - searchSourceBuilder.searchAfter(new Object[]{offset}); - searchSourceBuilder.from(0); - } else { - searchSourceBuilder.from(offset); + shouldAfter = true; } } - searchSourceBuilder.from(offset); + if(shouldAfter) { + searchSourceBuilder.from(0); + searchSourceBuilder.searchAfter(new Object[]{offset}); + }else { + searchSourceBuilder.from(offset); + } TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("type_tag.keyword", tokenType); diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java index 8ed937f..44fad73 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionService.java @@ -3,6 +3,7 @@ import com.alibaba.fastjson.JSON; import com.beust.jcommander.internal.Lists; import com.novi.serde.DeserializationError; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchRequest; @@ -24,6 +25,7 @@ import org.starcoin.api.Result; import org.starcoin.bean.*; import org.starcoin.constant.Constant; +import org.starcoin.scan.service.vo.TransactionWithEvent; import org.starcoin.types.AccountAddress; import org.starcoin.types.event.ProposalCreatedEvent; import org.starcoin.utils.ByteUtils; @@ -96,21 +98,18 @@ public Result getRange(String network, int page, int count //page size searchSourceBuilder.size(count); //begin offset - int offset = 0; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS && start_height > 0) { - offset = start_height - (page - 1) * count; - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } - searchSourceBuilder.from(offset); + setBlockSearchBuildFrom(page, count, start_height, searchSourceBuilder); + searchSourceBuilder.sort("timestamp", SortOrder.DESC); searchRequest.source(searchSourceBuilder); searchSourceBuilder.trackTotalHits(true); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + SearchResponse searchResponse = null; + try { + searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + } catch (ElasticsearchException e) { + logger.error("get txn err:", e); + return Result.EmptyResult; + } return ServiceUtils.getSearchResult(searchResponse, TransactionWithEvent.class); } @@ -128,16 +127,8 @@ public Result getTxnByStartTime(String network, long start //page size searchSourceBuilder.size(count); //begin offset - int offset = 0; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS) { - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } - searchSourceBuilder.from(offset); + setSearchBuildFrom(page, count, searchSourceBuilder); + searchSourceBuilder.sort("timestamp", SortOrder.DESC); searchRequest.source(searchSourceBuilder); searchSourceBuilder.trackTotalHits(true); @@ -160,17 +151,9 @@ public Result getNFTTxns(String network, long start_time, } //page size searchSourceBuilder.size(count); - //begin offset - int offset = 0; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS) { - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } - searchSourceBuilder.from(offset); + //set offset + setSearchBuildFrom(page, count, searchSourceBuilder); + List termHashes = new ArrayList<>(); for (Event event : events.getContents()) { termHashes.add(event.getTransactionHash()); @@ -191,23 +174,47 @@ public Result getRangePendingTransaction(String network, int //page size searchSourceBuilder.size(count); - //begin offset + //set offset + setBlockSearchBuildFrom(page, count, start_height, searchSourceBuilder); + + searchSourceBuilder.sort("timestamp", SortOrder.DESC); + searchRequest.source(searchSourceBuilder); + searchSourceBuilder.trackTotalHits(true); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + return ServiceUtils.getSearchResult(searchResponse, PendingTransaction.class); + } + + static void setBlockSearchBuildFrom(int page, int count, int start_height, SearchSourceBuilder searchSourceBuilder) { int offset = 0; + boolean shouldAfter = false; if (page > 1) { offset = (page - 1) * count; if (offset >= ELASTICSEARCH_MAX_HITS && start_height > 0) { offset = start_height - (page - 1) * count; - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); + shouldAfter = true; } } - searchSourceBuilder.from(offset); - searchSourceBuilder.sort("timestamp", SortOrder.DESC); - searchRequest.source(searchSourceBuilder); - searchSourceBuilder.trackTotalHits(true); - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - return ServiceUtils.getSearchResult(searchResponse, PendingTransaction.class); + if(shouldAfter) { + searchSourceBuilder.searchAfter(new Object[]{offset}); + }else { + searchSourceBuilder.from(offset); + } + } + + static void setSearchBuildFrom(int page, int count, SearchSourceBuilder searchSourceBuilder) { + int offset = 0; + boolean shouldAfter = false; + if (page > 1) { + offset = (page - 1) * count; + if (offset >= ELASTICSEARCH_MAX_HITS) { + shouldAfter = true; + } + } + if(shouldAfter) { + searchSourceBuilder.searchAfter(new Object[]{offset}); + }else { + searchSourceBuilder.from(offset); + } } public PendingTransaction getPending(String network, String id) throws IOException { @@ -227,17 +234,9 @@ public Result getRangeTransfers(String network, String typeTag, String receiver, SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); //page size searchSourceBuilder.size(count); - //begin offset - int offset = 0; - if (page > 1) { - offset = (page - 1) * count; - if (offset >= ELASTICSEARCH_MAX_HITS) { - searchSourceBuilder.searchAfter(new Object[]{offset}); - } else { - searchSourceBuilder.from(offset); - } - } - searchSourceBuilder.from(offset); + //set offset + setSearchBuildFrom(page, count, searchSourceBuilder); + searchSourceBuilder.sort("timestamp", SortOrder.DESC); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); if (typeTag != null && typeTag.length() > 0) { @@ -387,12 +386,8 @@ public Result getEventsByAddress(String network, String address, int page SearchRequest searchRequest = new SearchRequest(getIndex(network, Constant.TRANSACTION_EVENT_INDEX)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(count); - //begin offset - int offset = 0; - if (page > 1) { - offset = (page - 1) * count; - } - searchSourceBuilder.from(offset); + //set offset + setSearchBuildFrom(page, count, searchSourceBuilder); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); boolQuery.should(QueryBuilders.termQuery("tag_name", ServiceUtils.depositEvent)); diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/BlockVo.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/BlockVo.java new file mode 100644 index 0000000..d0a9db0 --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/BlockVo.java @@ -0,0 +1,64 @@ +package org.starcoin.scan.service.vo; + +import com.alibaba.fastjson.annotation.JSONField; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.starcoin.bean.*; + +import java.util.ArrayList; +import java.util.List; + +public class BlockVo extends Block { + + @JSONField(name = "daa_score") + @JsonProperty("daa_score") + private Long daaScore; + + @JSONField(name = "heightgroup_index") + @JsonProperty("heightgroup_index") + private Integer heightgroupIndex; + + @JSONField(name = "merged_blueset") + @JsonProperty("merged_blueset") + List mergedBlueset; + + public Long getDaaScore() { + return daaScore; + } + + public void setDaaScore(Long daaScore) { + this.daaScore = daaScore; + } + + public Integer getHeightgroupIndex() { + return heightgroupIndex; + } + + public void setHeightgroupIndex(Integer heightgroupIndex) { + this.heightgroupIndex = heightgroupIndex; + } + + public List getMergedBlueset() { + return mergedBlueset; + } + + public void setMergedBlueset(List mergedBlueset) { + this.mergedBlueset = mergedBlueset; + } + + public static BlockVo from(Block block) { + BlockVo blockVo = new BlockVo(); + + blockVo.setBlockMetadata(block.getBlockMetadata()); + blockVo.setBody(block.getBody()); + blockVo.setUncles(block.getUncles()); + blockVo.setHeader(block.getHeader()); + blockVo.setRaw(block.getRaw()); + blockVo.setBody(block.getBody()); + + blockVo.setDaaScore(0L); + blockVo.setHeightgroupIndex(0); + blockVo.setMergedBlueset(new ArrayList<>()); + + return blockVo; + } +} diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfigVo.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfigVo.java new file mode 100644 index 0000000..6b155e4 --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIAppConfigVo.java @@ -0,0 +1,49 @@ +package org.starcoin.scan.service.vo; + +public class DIAppConfigVo { + String starcoinVersion; + String processingVersion; + String network; + String apiVersion; + String webVersion; + + public String getStarcoinVersion() { + return starcoinVersion; + } + + public void setStarcoinVersion(String starcoinVersion) { + this.starcoinVersion = starcoinVersion; + } + + public String getProcessingVersion() { + return processingVersion; + } + + public void setProcessingVersion(String processingVersion) { + this.processingVersion = processingVersion; + } + + public String getNetwork() { + return network; + } + + public void setNetwork(String network) { + this.network = network; + } + + public String getApiVersion() { + return apiVersion; + } + + public void setApiVersion(String apiVersion) { + this.apiVersion = apiVersion; + } + + public String getWebVersion() { + return webVersion; + } + + public void setWebVersion(String webVersion) { + this.webVersion = webVersion; + } +} diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java new file mode 100644 index 0000000..1ae3ddd --- /dev/null +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/DIBlocksAndEdgesAndHeightGroupsVo.java @@ -0,0 +1,49 @@ +package org.starcoin.scan.service.vo; + +import org.starcoin.bean.DagInspectorBlock; +import org.starcoin.bean.DagInspectorEdge; +import org.starcoin.bean.DagInspectorHeightGroup; + +import java.util.List; + +/** + * DAG Inspector query object + */ +public class DIBlocksAndEdgesAndHeightGroupsVo { + List blocks; + List edges; + List heightGroups; + + public List getBlocks() { + return blocks; + } + + public void setBlocks(List blocks) { + this.blocks = blocks; + } + + public List getEdges() { + return edges; + } + + public void setEdges(List edges) { + this.edges = edges; + } + + public List getHeightGroups() { + return heightGroups; + } + + public void setHeightGroups(List heightGroups) { + this.heightGroups = heightGroups; + } + + @Override + public String toString() { + return "DIBlocksAndEdgesAndHeightGroupsVo{" + + "blocks=" + blocks + + ", edges=" + edges + + ", heightGroups=" + heightGroups + + '}'; + } +} \ No newline at end of file diff --git a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionWithEvent.java b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/TransactionWithEvent.java similarity index 90% rename from starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionWithEvent.java rename to starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/TransactionWithEvent.java index e5d90f5..4397a35 100644 --- a/starcoin-scan-api/src/main/java/org/starcoin/scan/service/TransactionWithEvent.java +++ b/starcoin-scan-api/src/main/java/org/starcoin/scan/service/vo/TransactionWithEvent.java @@ -1,4 +1,4 @@ -package org.starcoin.scan.service; +package org.starcoin.scan.service.vo; import org.starcoin.bean.Event; import org.starcoin.bean.Transaction; diff --git a/starcoin-scan-api/src/main/resources/application.properties b/starcoin-scan-api/src/main/resources/application.properties index 3b116dc..88ab280 100644 --- a/starcoin-scan-api/src/main/resources/application.properties +++ b/starcoin-scan-api/src/main/resources/application.properties @@ -56,6 +56,8 @@ spring.datasource.halley.jdbc-url=${HALLEY_DS_URL} spring.datasource.halley.username=${DB_USER_NAME} spring.datasource.halley.password=${DB_PWD} #default ds config +spring.datasource.hikari.minimum-idle=1 +spring.datasource.hikari.maximum-pool-size=2 spring.datasource.url=${DS_URL} spring.datasource.username=${DB_USER_NAME} spring.datasource.password=${DB_PWD} diff --git a/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java new file mode 100644 index 0000000..84b4f61 --- /dev/null +++ b/starcoin-scan-api/src/test/java/org/starcoin/scan/service/DagInspectServiceTest.java @@ -0,0 +1,66 @@ +package org.starcoin.scan.service; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; +import org.starcoin.scan.ScanApplication; +import org.starcoin.scan.service.vo.DIBlocksAndEdgesAndHeightGroupsVo; + +import java.io.IOException; + +@SpringBootTest(classes = ScanApplication.class) +@TestPropertySource(locations = "classpath:application-unittest.properties") +public class DagInspectServiceTest { + + final static String TEST_NETWORK = "halley"; + + @Autowired + DagInspectorService dagInspectorService; + + @Test + public void testGetBlocksAndEdgesAndHeightGroups() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = + dagInspectorService.getBlocksAndEdgesAndHeightGroups(TEST_NETWORK, 10000L, 10050L); + Assertions.assertNotEquals(0, vo.getBlocks().size()); + Assertions.assertNotEquals(0, vo.getEdges().size()); + Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + } + + @Test + public void testGetBlockHash() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlockHash( + TEST_NETWORK, + "0x93a4fc71929be2e435efe682d02260f8dd46824fe90e926e3b3ea5839f31e67c", + 10L + ); + Assertions.assertNotEquals(0, vo.getBlocks().size()); + Assertions.assertNotEquals(0, vo.getEdges().size()); + Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + System.out.println(vo); + } + + @Test + public void testGetBlockDAAScore() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getBlockDAAScore( + TEST_NETWORK, + 10409L, + 1L + ); + Assertions.assertNotEquals(0, vo.getBlocks().size()); + Assertions.assertNotEquals(0, vo.getEdges().size()); + Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + System.out.println(vo); + } + + + @Test + public void testGetHead() throws IOException { + DIBlocksAndEdgesAndHeightGroupsVo vo = dagInspectorService.getHead(TEST_NETWORK, 100L); + Assertions.assertNotEquals(0, vo.getBlocks().size()); + Assertions.assertNotEquals(0, vo.getEdges().size()); + Assertions.assertNotEquals(0, vo.getHeightGroups().size()); + System.out.println(vo); + } +} diff --git a/starcoin-scan-api/src/test/resources/application-unittest.properties b/starcoin-scan-api/src/test/resources/application-unittest.properties new file mode 100644 index 0000000..c03bbf4 --- /dev/null +++ b/starcoin-scan-api/src/test/resources/application-unittest.properties @@ -0,0 +1,72 @@ +# server.port=8500 +spring.application.name=stccan +indexer.version=${STARCOIN_ES_INDEX_VERSION} +jasypt.encryptor.bean=encryptorBean +jasypt.encryptor.password=${STARCOIN_ES_PWD} +elasticsearch.host=${STARCOIN_ES_URL} +elasticsearch.protocol=${STARCOIN_ES_PROTOCOL} +elasticsearch.port=${STARCOIN_ES_PORT} +elasticsearch.username=${STARCOIN_ES_USER} +elasticsearch.connTimeout=10000 +elasticsearch.socketTimeout=10000 +elasticsearch.connectionRequestTimeout=2000 +logging.file.name=logs/scan.log +logging.level.org.springframework.web.filter.CommonsRequestLoggingFilter=DEBUG + +#zuul config +zuul.ratelimit.enabled = true +zuul.ratelimit.repository= BUCKET4J_HAZELCAST +zuul.ratelimit.response-headers=standard +zuul.ratelimit.deny-request.response-status-code=429 +zuul.ratelimit.default-policy-list[0].limit=5 +zuul.ratelimit.default-policy-list[0].quota=1000 +zuul.ratelimit.default-policy-list[0].refresh-interval=60 +zuul.ratelimit.default-policy-list[0].type[0]=user +zuul.ratelimit.default-policy-list[0].type[1]=origin +zuul.ratelimit.default-policy-list[0].type[2]=url +zuul.ratelimit.default-policy-list[0].type[3]=http_method +zuul.ratelimit.policy-list.userGrade[0].limit=5 +zuul.ratelimit.policy-list.userGrade[0].quota=1000 +zuul.ratelimit.policy-list.userGrade[0].refresh-interval=60 +zuul.ratelimit.policy-list.userGrade[0].type[0]=user +zuul.ratelimit.policy-list.userGrade[0].type[1]=origin +zuul.ratelimit.policy-list.userGrade[0].type[2]=url +zuul.ratelimit.policy-list.userGrade[1].limit=10 +zuul.ratelimit.policy-list.userGrade[1].quota=1000 +zuul.ratelimit.policy-list.userGrade[1].refresh-interval=60 +zuul.ratelimit.policy-list.userGrade[1].type[0]=user +zuul.ratelimit.policy-list.userGrade[1].type[1]=origin +zuul.ratelimit.policy-list.userGrade[1].type[2]=url +zuul.ratelimit.policy-list.userGrade[2].limit=20 +zuul.ratelimit.policy-list.userGrade[2].quota=1000 +zuul.ratelimit.policy-list.userGrade[2].refresh-interval=60 +zuul.ratelimit.policy-list.userGrade[2].type[0]=user +zuul.ratelimit.policy-list.userGrade[2].type[1]=origin +zuul.ratelimit.policy-list.userGrade[2].type[2]=url +#main ds config +spring.datasource.main.jdbc-url=${MAIN_DS_URL} +spring.datasource.main.username=${DB_USER_NAME} +spring.datasource.main.password=${DB_PWD} +#barnard ds config +spring.datasource.barnard.jdbc-url=${BARNARD_DS_URL} +spring.datasource.barnard.username=${DB_USER_NAME} +spring.datasource.barnard.password=${DB_PWD} +#halley ds config +spring.datasource.halley.jdbc-url=${HALLEY_DS_URL} +spring.datasource.halley.username=${DB_USER_NAME} +spring.datasource.halley.password=${DB_PWD} +# vega ds config +spring.datasource.vega.jdbc-url=${HALLEY_DS_URL} +spring.datasource.vega.username=${DB_USER_NAME} +spring.datasource.vega.password=${DB_PWD} + +#default ds config +spring.datasource.hikari.minimum-idle=1 +spring.datasource.hikari.maximum-pool-size=2 +spring.datasource.url=${DS_URL} +spring.datasource.username=${DB_USER_NAME} +spring.datasource.password=${DB_PWD} +#user_info ds config +spring.datasource.userinfo.jdbc-url=${STARCOIN_USER_DS_URL} +spring.datasource.userinfo.username=${DB_USER_NAME} +spring.datasource.userinfo.password=${DB_PWD} \ No newline at end of file