Merge pull request #6 from ixofoundation/develop

Develop

.releaserc.json

@@ -50,13 +50,6 @@
         "npmPublish": false
       }
     ],
-    [
-      "@semantic-release/git",
-      {
-        "assets": ["package.json"],
-        "message": "chore(release): ${nextRelease.version} [skip ci]\n${nextRelease.notes}"
-      }
-    ],
     "@semantic-release/github"
   ]
-} 
+}

README.md

@@ -51,10 +51,12 @@ cd ixo-blocksync-core/
 ```
 
 Copy `.env.example` to `.env` and configure. If this step is skipped, ixo-blocksync will use `.env.example` as the configuration by default.
-Don't use quotations when asign env vars for docker  
+Don't use quotations when asign env vars for docker
 Create a role(e.g. app_user) in the DB for postgress to work.
 
 ```bash
 docker build -t ixofoundation/ixo-blocksync-core:latest .
 docker compose up -d
 ```
+
+.

package.json

@@ -20,7 +20,7 @@
   "author": "Ixo Foundation",
   "license": "Apache 2",
   "dependencies": {
-    "@ixo/impactxclient-sdk": "1.2.0",
+    "@ixo/impactxclient-sdk": "2.3.1",
     "@sentry/node": "7.36.0",
     "@sentry/tracing": "7.36.0",
     "body-parser": "1.20.1",
@@ -46,4 +46,4 @@
     "semantic-release": "22",
     "typedoc": "0.23.24"
   }
-}
+}

src/app.ts

@@ -5,6 +5,7 @@ import * as Sentry from "@sentry/node";
 import { SENTRYDSN, TRUST_PROXY } from "./util/secrets";
 import helmet from "helmet";
 import rateLimit from "express-rate-limit";
+import { getCoreBlock } from "./postgres/query_block";
 
 const limiter = rateLimit({
   windowMs: 1 * 60 * 1000, // 1 minutes
@@ -44,3 +45,16 @@ app.get("/", (req, res) => {
 //     headers: request.headers["x-forwarded-for"],
 //   })
 // );
+
+// =================================
+// Custom helpers for local development
+// =================================
+
+app.get("/api/development/getCoreBlock/:height", async (req, res, next) => {
+  try {
+    const result = await getCoreBlock(Number(req.params.height || 0));
+    res.json(result);
+  } catch (error) {
+    res.status(500).send(error.message);
+  }
+});

src/postgres/query_block.ts

@@ -0,0 +1,88 @@
+import { dbQuery } from "./client";
+
+export type BlockCore = {
+  height: number;
+  time: Date;
+  transactions: TransactionCore[];
+  events: EventCore[];
+};
+
+export type TransactionCore = {
+  hash: string;
+  code: number;
+  fee: any; // JSON
+  gasUsed: string;
+  gasWanted: string;
+  memo: string;
+  messages: MessageCore[];
+};
+
+export type MessageCore = {
+  typeUrl: string;
+  value: any; // JSON
+};
+
+export type EventCore = {
+  type: string;
+  attributes: any[]; // JSON
+};
+
+const sqlTransactions = `
+SELECT
+  t."hash",
+  t."code",
+  t."fee",
+  t."gasUsed",
+  t."gasWanted",
+  t."memo",
+  json_agg(json_build_object('typeUrl', "m"."typeUrl", 'value', m.value)) AS messages
+FROM "TransactionCore" as t
+LEFT OUTER JOIN "MessageCore" as m ON t.hash = m."transactionHash"
+WHERE t."blockHeight" = $1
+Group By t.hash;
+`;
+const sqlEvents = `
+SELECT
+  b."height",
+  b."time",
+  json_agg(json_build_object('type', e.type, 'attributes', e.attributes)) AS events
+FROM "BlockCore" as b
+LEFT OUTER JOIN (
+  SELECT "type", attributes
+    from "EventCore"
+    where "blockHeight" = $1
+    order by id asc
+  ) as e on TRUE
+WHERE b.height = $1
+GROUP BY b.height, b."time"
+`;
+export const getCoreBlock = async (
+  blockHeight: number
+): Promise<BlockCore | null> => {
+  try {
+    let blockAndEvents: any = await dbQuery(sqlEvents, [blockHeight]);
+    // If no block is found, return null before querying transactions
+    if (blockAndEvents.rows.length === 0) return null;
+    let transactions: any = await dbQuery(sqlTransactions, [blockHeight]);
+
+    blockAndEvents = blockAndEvents.rows[0];
+    transactions = transactions.rows.map((row: any) => ({
+      hash: row.hash,
+      code: row.code,
+      fee: row.fee,
+      gasUsed: row.gasUsed,
+      gasWanted: row.gasWanted,
+      memo: row.memo,
+      messages: row.messages,
+    }));
+
+    return {
+      height: blockAndEvents.height,
+      time: blockAndEvents.time,
+      transactions,
+      events: blockAndEvents.events,
+    };
+  } catch (error) {
+    throw error;
+  }
+};

src/sync/sync_blocks.ts

@@ -1,13 +1,13 @@
 import * as Proto from "../util/proto";
 import * as BlockSyncHandler from "../sync_handlers/block_sync_handler";
-import { Event } from "@cosmjs/tendermint-rpc/build/tendermint34/responses";
 import { currentChain } from "./sync_chain";
 import { utils } from "@ixo/impactxclient-sdk";
 import { sleep } from "../util/helpers";
 import { getChain, updateChain } from "../postgres/chain";
 import { getMemoryUsage } from "../util/memory";
 import { withTransaction } from "../postgres/client";
 import { PoolClient } from "pg";
+import { Event } from "@ixo/impactxclient-sdk/types/codegen/tendermint/abci/types";
 
 let syncing: boolean;
 
@@ -34,7 +34,7 @@ export const startSync = async () => {
   while (syncing) {
     currentPool = undefined;
 
-    // if (currentBlock === 2792945) return; // if need custom end block
+    // if (currentBlock === 2) return; // if need custom end block
     // console.log("wait then get block:", currentBlock, getMemoryUsage().rss);
     // await sleep(4000);
 
@@ -61,6 +61,34 @@ export const startSync = async () => {
 
         const blockHeight = Number(block.block!.header!.height.low);
 
+        // if blockTM has finalizeBlockEvents, we need to split them into beginBlockEvents and endBlockEvents
+        // loop through finalizeBlockEvents and find the mode (BeginBlock or EndBlock) and add it to the corresponding array
+        // pushing original objects for optimization
+        let beginBlockEvents: Event[] = [];
+        let endBlockEvents: Event[] = [];
+        if (blockTM.finalizeBlockEvents.length > 0) {
+          for (let i = 0; i < blockTM.finalizeBlockEvents.length; i++) {
+            const event = blockTM.finalizeBlockEvents[i];
+            const attributes = event.attributes;
+            if (!attributes) continue;
+
+            for (let j = 0; j < attributes.length; j++) {
+              const attr = attributes[j];
+              if (attr.key === "mode") {
+                if (attr.value === "BeginBlock") {
+                  beginBlockEvents.push(event as any);
+                } else if (attr.value === "EndBlock") {
+                  endBlockEvents.push(event as any);
+                }
+                break; // Stop searching attributes once mode is found
+              }
+            }
+          }
+        } else {
+          beginBlockEvents = blockTM.beginBlockEvents as any;
+          endBlockEvents = blockTM.endBlockEvents as any;
+        }
+
         await withTransaction(async (client) => {
           currentPool = client;
           await Promise.all([
@@ -69,8 +97,8 @@ export const startSync = async () => {
               block.blockId!.hash!,
               utils.proto.fromTimestamp(block.block!.header!.time!),
               txsEvent.txResponses,
-              blockTM.beginBlockEvents as Event[],
-              blockTM.endBlockEvents as Event[]
+              beginBlockEvents,
+              endBlockEvents
             ),
             updateChain({
               chainId: currentChain.chainId,

src/sync/sync_chain.ts

@@ -1,4 +1,4 @@
-import { Tendermint34Client } from "@cosmjs/tendermint-rpc";
+// import { connectComet, CometClient } from "@cosmjs/tendermint-rpc";
 import * as Proto from "../util/proto";
 import { createQueryClient, createRegistry } from "@ixo/impactxclient-sdk";
 import { RPC } from "../util/secrets";
@@ -7,15 +7,13 @@ import { Chain, createChain, getChain } from "../postgres/chain";
 export let currentChain: Chain;
 export let queryClient: Awaited<ReturnType<typeof createQueryClient>>;
 export let registry: ReturnType<typeof createRegistry>;
-export let tendermintClient: Awaited<
-  ReturnType<typeof Tendermint34Client.connect>
->;
+// export let cometClient: CometClient;
 
 export const syncChain = async () => {
   try {
     queryClient = await createQueryClient(RPC);
     registry = createRegistry();
-    tendermintClient = await Tendermint34Client.connect(RPC);
+    // cometClient = await connectComet(RPC);
 
     const res = await Proto.getLatestBlock();
     const chainId = res?.block?.header?.chainId || "";

src/sync_handlers/block_sync_handler.ts

@@ -1,4 +1,4 @@
-import { Event } from "@cosmjs/tendermint-rpc/build/tendermint34/responses";
+import { Event } from "@ixo/impactxclient-sdk/types/codegen/tendermint/abci/types";
 import { TxResponse } from "@ixo/impactxclient-sdk/types/codegen/cosmos/base/abci/v1beta1/abci";
 import { upperHexFromUint8Array } from "../util/helpers";
 import { syncEvents } from "./event_sync_handler";

src/sync_handlers/event_sync_handler.ts

@@ -1,7 +1,7 @@
-import { Event } from "@cosmjs/tendermint-rpc/build/tendermint34/responses";
 import { EventTypesSet } from "../types/Event";
 import { decodeEvent } from "../util/proto";
 import { EventCore } from "../postgres/block";
+import { Event } from "@ixo/impactxclient-sdk/types/codegen/tendermint/abci/types";
 
 export const syncEvents = (
   beginBlockEvents: Event[],

src/types/Event.ts

@@ -5,16 +5,26 @@ export enum EventTypes {
   // entity
   createEntity = "ixo.entity.v1beta1.EntityCreatedEvent",
   updateEntity = "ixo.entity.v1beta1.EntityUpdatedEvent",
+  verifiedEntity = "ixo.entity.v1beta1.EntityVerifiedUpdatedEvent",
+  transferEntity = "ixo.entity.v1beta1.EntityTransferredEvent",
+  accountCreatedEntity = "ixo.entity.v1beta1.EntityAccountCreatedEvent",
   // claims
   createCollection = "ixo.claims.v1beta1.CollectionCreatedEvent",
   updateCollection = "ixo.claims.v1beta1.CollectionUpdatedEvent",
   submitClaim = "ixo.claims.v1beta1.ClaimSubmittedEvent",
   updateClaim = "ixo.claims.v1beta1.ClaimUpdatedEvent",
+  evaluateClaim = "ixo.claims.v1beta1.ClaimEvaluatedEvent",
   disputeClaim = "ixo.claims.v1beta1.ClaimDisputedEvent",
+  submitIntent = "ixo.claims.v1beta1.IntentSubmittedEvent",
+  updateIntent = "ixo.claims.v1beta1.IntentUpdatedEvent",
   // token
   createToken = "ixo.token.v1beta1.TokenCreatedEvent",
   updateToken = "ixo.token.v1beta1.TokenUpdatedEvent",
   mintToken = "ixo.token.v1beta1.TokenMintedEvent",
+  transferToken = "ixo.token.v1beta1.TokenTransferredEvent",
+  cancelToken = "ixo.token.v1beta1.TokenCancelledEvent",
+  retireToken = "ixo.token.v1beta1.TokenRetiredEvent",
+  transferCredit = "ixo.token.v1beta1.CreditsTransferredEvent",
   // bonds
   createBond = "ixo.bonds.v1beta1.BondCreatedEvent",
   updateBond = "ixo.bonds.v1beta1.BondUpdatedEvent",
@@ -27,6 +37,22 @@ export enum EventTypes {
   reserveWithdrawalBond = "ixo.bonds.v1beta1.BondWithdrawReserveEvent",
   // Wasm
   wasm = "wasm",
+  // epochs
+  startEpoch = "ixo.epochs.v1beta1.EpochStartEvent",
+  endEpoch = "ixo.epochs.v1beta1.EpochEndEvent",
+  // mint
+  epochProvisionsMinted = "ixo.mint.v1beta1.MintEpochProvisionsMintedEvent",
+  // smartaccount
+  authAdded = "ixo.smartaccount.v1beta1.AuthenticatorAddedEvent",
+  authRemoved = "ixo.smartaccount.v1beta1.AuthenticatorRemovedEvent",
+  authSetActive = "ixo.smartaccount.v1beta1.AuthenticatorSetActiveStateEvent",
+  // liquidstake
+  lsParamsUpdated = "ixo.liquidstake.v1beta1.LiquidStakeParamsUpdatedEvent",
+  lsStake = "ixo.liquidstake.v1beta1.LiquidStakeEvent",
+  lsUpstake = "ixo.liquidstake.v1beta1.LiquidUnstakeEvent",
+  lsAddLSValidator = "ixo.liquidstake.v1beta1.AddLiquidValidatorEvent",
+  lsRebalanced = "ixo.liquidstake.v1beta1.RebalancedLiquidStakeEvent",
+  lsAutoCompound = "ixo.liquidstake.v1beta1.AutocompoundStakingRewardsEvent",
 }
 
 const EventTypesArray = Object.values(EventTypes) as string[];

src/util/proto.ts

@@ -1,8 +1,8 @@
-import { Event } from "@cosmjs/tendermint-rpc/build/tendermint34/responses";
 import { TxResponse } from "@ixo/impactxclient-sdk/types/codegen/cosmos/base/abci/v1beta1/abci";
-import { cosmos, utils } from "@ixo/impactxclient-sdk";
+import { cosmos, customQueries, utils } from "@ixo/impactxclient-sdk";
 import Long from "long";
-import { queryClient, registry, tendermintClient } from "../sync/sync_chain";
+import { queryClient, registry } from "../sync/sync_chain";
+import { RPC } from "./secrets";
 
 export const getLatestBlock = async () => {
   try {
@@ -37,6 +37,9 @@ export const getTxsEvent = async (height: number) => {
     const res = await queryClient.cosmos.tx.v1beta1.getTxsEvent({
       events: [`tx.height=${height}`],
       orderBy: cosmos.tx.v1beta1.OrderBy.ORDER_BY_ASC,
+      limit: Long.fromNumber(10000),
+      page: Long.fromNumber(1),
+      query: `tx.height=${height}`,
     });
     return res;
   } catch (error) {
@@ -47,7 +50,9 @@ export const getTxsEvent = async (height: number) => {
 
 export const getTMBlockbyHeight = async (height: number) => {
   try {
-    const res = await tendermintClient.blockResults(height);
+    const res = await customQueries.comet.blockResults(height, RPC, false);
+    // const res = await cometClient.blockResults(height);
+    // console.dir(res, { depth: null });
     return res;
   } catch (error) {
     if (!error.toString().includes('"code":-32603'))
@@ -85,7 +90,17 @@ export const decodeMessage = (tx: any) => {
   }
 };
 
-export const decodeEvent = (event: Event) => {
+export const decodeEvent = (event: any) => {
+  // If event attributes is stringified, then it is a cosmos-sdk event, no need
+  // to map over attributes, slight optimization
+  if (typeof event.attributes?.[0]?.value === "string") {
+    return {
+      type: event.type,
+      attributes: event.attributes,
+    };
+  }
+
+  // otherwise we can assume all attributes are Uint8Array
   const attributes = event.attributes.map((attr) => ({
     key: utils.conversions.Uint8ArrayToJS(attr.key),
     value: utils.conversions.Uint8ArrayToJS(attr.value),