Dag inspector

.github/workflows/build.yml

@@ -20,11 +20,6 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: 11
-      - name: maven-settings-xml-action
-        uses: whelk-io/maven-settings-xml-action@v18
-        with:
-          repositories: '[{ "id": "github", "url": "https://maven.pkg.github.com/starcoinorg/*" }]'
-          servers: '[{ "id": "github", "username": "${{ github.actor }}", "password": "${{ secrets.GIT_PACKAGE_TOKEN }}" }]'
       - name: Cache Maven packages
         uses: actions/cache@v1
         with:

.github/workflows/docker_build_indexer.yml

@@ -22,7 +22,7 @@ jobs:
         id: docker_meta
         uses: crazy-max/ghaction-docker-meta@v1
         with:
-          images: starcoin/starcoin_indexer,ghcr.io/starcoinorg/starcoin_indexer
+          images: starcoin/starcoin_indexer
           tag-sha: true
       - name: Set up Docker Buildx
         uses: docker/[email protected]
@@ -33,12 +33,6 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GIT_PACKAGE_TOKEN }}
       - name: maven-settings-xml-action
         uses: whelk-io/maven-settings-xml-action@v18
         with:

.github/workflows/docker_build_scanapi.yml

@@ -22,7 +22,7 @@ jobs:
         id: docker_meta
         uses: crazy-max/ghaction-docker-meta@v1
         with:
-          images: starcoin/starcoin_scan,ghcr.io/starcoinorg/starcoin_scan
+          images: starcoin/starcoin_scan
           tag-sha: true
       - name: Set up Docker Buildx
         uses: docker/[email protected]

README.md

@@ -1 +1,3 @@
-# starcoin-search
+# stc-scan
+Established some independent handles for indexing data with scan to customize statistics data for swap, used in conjunction with the [swap-stat-api](https://github.com/Elements-Studio/swap-stat-api) repository.
+

docker-compose/README.md

@@ -0,0 +1,63 @@
+# How to build local environment
+
+## 1. Install "Docker" and "Docker Compose"
+## 2. Run the command to start the database and components
+```bash
+docker-compose up
+```
+## 3. Start starcoin-index project
+
+### Config the startup environment variable
+```dotenv
+HOSTS=localhost  
+NETWORK=halley # select which network to scan
+BG_TASK_JOBS=dag_inspector
+TXN_OFFSET=0
+BULK_SIZE=100
+STARCOIN_ES_PWD=
+STARCOIN_ES_URL=localhost
+STARCOIN_ES_PROTOCOL=http
+STARCOIN_ES_PORT=9200
+STARCOIN_ES_USER=
+SWAP_API_URL=https://swap-api.starswap.xyz
+SWAP_CONTRACT_ADDR=0x8c109349c6bd91411d6bc962e080c4a3
+DS_URL=jdbc:postgresql://localhost/starcoin
+DB_SCHEMA=halley
+DB_USER_NAME=starcoin
+DB_PWD=starcoin
+PROGRAM_ARGS=
+# auto_repair 9411700
+```
+
+### Configuration Elasticsearch template
+[IMPORTANT!!] Make sure your template has added to Elastic search service before add data, including component template and index template to ES.
+Following file: [[es_pipeline.scripts](..%2Fkube%2Fmappings%2Fes_pipeline.scripts)]
+
+1. Open the 'Kibana' site has been started in the docker-compose environment, usually the url is http://localhost:5601 
+2. Navigate to 'Dev Tools'
+3. Follow the instructions in the file of giving above to add the template to ES
+
+### Add SQL tables for network
+[IMPORTANT!!] Add the [tables](../starcoin-indexer/deploy/create_table.sql) for the network you want to scan, including main, barnard, halley, etc.
+
+## 4. Start starcoin-scan-api project
+
+### Config the startup enviroment variable
+```dotenv
+STARCOIN_ES_URL=localhost
+STARCOIN_ES_PROTOCOL=http
+STARCOIN_ES_PORT=9200
+STARCOIN_ES_USER=
+STARCOIN_ES_INDEX_VERSION=
+STARCOIN_ES_PWD=
+MAIN_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=main
+BARNARD_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=barnard
+HALLEY_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=halley
+DS_URL=jdbc:postgresql://localhost/starcoin
+STARCOIN_USER_DS_URL="jdbc:postgresql://localhost/starcoin?currentSchema=starcoin_user"
+DB_USER_NAME=starcoin
+DB_PWD=starcoin
+```
+
+### Add SQL tables for network
+[IMPORTANT!!] Add the [tables](../starcoin-scan-api/deploy/create_table.sql) for the network you want to scan, including main, barnard, halley, etc.

docker-compose/docker-compose.yaml

@@ -0,0 +1,67 @@
+# This composer file is used to configure the local environment for debugging
+version: '3.8'
+
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.2
+    container_name: elasticsearch
+    environment:
+      - node.name=elasticsearch
+      - cluster.name=docker-cluster
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - esdata:/usr/share/elasticsearch/data
+    ports:
+      - "9200:9200"
+
+  hazelcast:
+    image: hazelcast/hazelcast:latest
+    container_name: hazelcast
+    ports:
+      - "5701:5701"
+    environment:
+      - HZ_CLUSTERNAME=stcscan-hazelcast-cluster
+      - HZ_NETWORK_JOIN_MULTICAST_ENABLED=false
+      - HZ_NETWORK_JOIN_TCPIP_ENABLED=true
+      - HZ_NETWORK_JOIN_TCPIP_MEMBERS=hazelcast
+      - HZ_CACHE_CODE_CACHE_STATISTICS_ENABLED=true
+      - HZ_CACHE_CODE_CACHE_EVICTION_SIZE=10000
+      - HZ_CACHE_CODE_CACHE_EVICTION_MAX_SIZE_POLICY=ENTRY_COUNT
+      - HZ_CACHE_CODE_CACHE_EVICTION_EVICTION_POLICY=LFU
+      - HZ_CACHE_SESSION_STATISTICS_ENABLED=true
+      - HZ_CACHE_SESSION_EVICTION_SIZE=50000
+      - HZ_CACHE_SESSION_EVICTION_MAX_SIZE_POLICY=ENTRY_COUNT
+      - HZ_CACHE_SESSION_EVICTION_EVICTION_POLICY=LRU
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:7.17.2
+    container_name: kibana
+    environment:
+      ELASTICSEARCH_URL: http://elasticsearch:9200
+      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
+    ports:
+      - 5601:5601
+    depends_on:
+      - elasticsearch
+
+  postgresql:
+    image: postgres:13.2
+    container_name: postgres_db
+    environment:
+      POSTGRES_USER: starcoin
+      POSTGRES_PASSWORD: starcoin
+      POSTGRES_DB: starcoin
+    volumes:
+      - db_data:/var/lib/postgresql/data
+    ports:
+      - 5432:5432
+
+volumes:
+  esdata:
+    driver: local

kube/base-components/allowaccess-network-policy.yaml

@@ -0,0 +1,92 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-access-pg-from-vega
+  namespace: default
+spec:
+  podSelector:
+    matchLabels:
+      app: postgres-service
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              name: starcoin-vega
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-access-es-from-vega
+  namespace: default
+spec:
+  podSelector:
+    matchLabels:
+      app: elasticsearch
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              name: starcoin-vega
+
+
+---
+# Postgres service for starcoin-api
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-access-pg-from-api
+  namespace: default
+spec:
+  podSelector:
+    matchLabels:
+      app: postgres-service
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              name: starcoin-api
+
+---
+# Elasticsearch service for starcoin-api
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-access-es-from-api
+  namespace: default
+spec:
+  podSelector:
+    matchLabels:
+      app: elasticsearch
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+      - namespaceSelector:
+         matchLabels:
+          name: starcoin-api
+
+---
+# Default namespace access for elasticsearch
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-internal-elasticsearch
+  namespace: default
+spec:
+  podSelector:
+    matchLabels:
+      app: elasticsearch
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - podSelector: {}
+      ports:
+        - protocol: TCP
+          port: 9200

kube/base-components/elasticsearch-deployment.yaml

@@ -0,0 +1,100 @@
+# elasticsearch-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elasticsearch
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elasticsearch
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      initContainers:
+        - name: init-permissions
+          image: busybox
+          command: [ "sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data" ]
+          volumeMounts:
+            - name: es-data
+              mountPath: /usr/share/elasticsearch/data
+      containers:
+        - name: elasticsearch
+          image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
+          resources:
+            requests:
+              cpu: "1"
+              memory: "4Gi"
+            limits:
+              cpu: "2"
+              memory: "6Gi"
+          ports:
+            - containerPort: 9200
+              name: http
+            - containerPort: 9300
+              name: transport
+          volumeMounts:
+            - name: es-data
+              mountPath: /usr/share/elasticsearch/data
+            - name: elasticsearch-config
+              mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
+              subPath: elasticsearch.yml
+          env:
+            - name: discovery.type
+              value: single-node
+            - name: ELASTIC_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: elasticsearch
+                  key: username
+            - name: ELASTIC_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: elasticsearch
+                  key: password
+      volumes:
+        - name: es-data
+          persistentVolumeClaim:
+            claimName: es-pvc
+        - name: elasticsearch-config
+          configMap:
+            name: elasticsearch-config
+---
+# Elasticsearch Configuration
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elasticsearch-config
+data:
+  elasticsearch.yml: |
+    xpack.security.enabled: true
+    xpack.license.self_generated.type: basic
+    network.host: 0.0.0.0
+
+---
+# Elasticsearch Persistent Volume Claim
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: es-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 100Gi
+  storageClassName: do-block-storage-retain
+
+---
+# Elasticsearch Service
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+spec:
+  ports:
+    - port: 9200
+  selector:
+    app: elasticsearch