feat: added suport new datasource subquery

codegenConfig/codegen-subquery-nova-query.ts

@@ -0,0 +1,23 @@
+import type { CodegenConfig } from '@graphql-codegen/cli';
+import * as dotenv from 'dotenv';
+dotenv.config({ path: `${__dirname}/../.env.local` });
+
+const queryUrl = process.env.DATA_SOURCE_SUBQUERY_NOVA_POLKADOT;
+if (!queryUrl) throw new Error('Codegen error: Datahub Mutation URL not set');
+
+const config: CodegenConfig = {
+  overwrite: true,
+  schema: queryUrl,
+  documents: 'src/utils/graphQl/subQueryNova/query.ts',
+  generates: {
+    'src/utils/graphQl/subQueryNova/subquery-nova-query.ts': {
+      plugins: [
+        'typescript',
+        'typescript-operations',
+        'typescript-document-nodes',
+      ],
+    },
+  },
+};
+
+export default config;

deployment/feature-based/all.yaml

@@ -61,11 +61,21 @@ data:
   AGGREGATOR_REDIS_ENABLE_SSL: 'false'
   AGGREGATOR_HISTORY_RENEW_INTERVAL_MS: '60000'
   AGGREGATOR_GS_MAIN_CHUNK_BLOCKS_SIZE: '1000000'
-  DATA_SOURCE_GSQUID_MAIN_POLKADOT: 'https://squid.subsquid.io/gs-main-polkadot/graphql'
-  DATA_SOURCE_GSQUID_MAIN_KUSAMA: 'https://squid.subsquid.io/gs-main-kusama/graphql'
-  DATA_SOURCE_GSQUID_MAIN_MOONBEAM: 'https://squid.subsquid.io/gs-main-moonbeam/graphql'
-  DATA_SOURCE_GSQUID_MAIN_MOONRIVER: 'https://squid.subsquid.io/gs-main-moonriver/graphql'
-  DATA_SOURCE_GSQUID_MAIN_ASTAR: 'https://squid.subsquid.io/gs-main-astar/graphql'
+
+  DATA_SOURCE__SUBSQUID__POLKADOT__TRANSFER: 'https://squid.subsquid.io/gs-main-polkadot/graphql'
+  DATA_SOURCE__SUBSQUID__KUSAMA__TRANSFER: 'https://squid.subsquid.io/gs-main-kusama/graphql'
+  DATA_SOURCE__SUBSQUID__MOONBEAM__TRANSFER: 'https://squid.subsquid.io/gs-main-moonbeam/graphql'
+  DATA_SOURCE__SUBSQUID__MOONRIVER__TRANSFER: 'https://squid.subsquid.io/gs-main-moonriver/graphql'
+  DATA_SOURCE__SUBSQUID__ASTAR__TRANSFER: 'https://squid.subsquid.io/gs-main-astar/graphql'
+
+  DATA_SOURCE__SUBQUERY__POLKADOT__TRANSFER: 'https://api.subquery.network/sq/nova-wallet/nova-wallet-polkadot'
+  DATA_SOURCE__SUBQUERY__KUSAMA__TRANSFER: 'https://api.subquery.network/sq/nova-wallet/nova-wallet-kusama'
+  DATA_SOURCE__SUBQUERY__MOONBEAM__TRANSFER: 'https://api.subquery.network/sq/nova-wallet/nova-wallet-moonbeam'
+  DATA_SOURCE__SUBQUERY__MOONRIVER__TRANSFER: 'https://api.subquery.network/sq/nova-wallet/nova-wallet-moonriver'
+  DATA_SOURCE__SUBQUERY__ASTAR__TRANSFER: 'https://api.subquery.network/sq/nova-wallet/nova-wallet-astar'
+
+  DATA_SOURCE_PROVIDER_TRANSFER: 'SUBQUERY'
+
   NODE_ENV: 'staging'
   APP_PORT: '3000'
 

deployment/prod/all.yaml

@@ -61,10 +61,20 @@ data:
   AGGREGATOR_REDIS_ENABLE_SSL: 'false'
   AGGREGATOR_HISTORY_RENEW_INTERVAL_MS: '60000'
   AGGREGATOR_GS_MAIN_CHUNK_BLOCKS_SIZE: '1000000'
-  DATA_SOURCE_GSQUID_MAIN_POLKADOT: 'https://squid.subsquid.io/gs-main-polkadot/graphql'
-  DATA_SOURCE_GSQUID_MAIN_KUSAMA: 'https://squid.subsquid.io/gs-main-kusama/graphql'
-  DATA_SOURCE_GSQUID_MAIN_MOONBEAM: 'https://squid.subsquid.io/gs-main-moonbeam/graphql'
-  DATA_SOURCE_GSQUID_MAIN_MOONRIVER: 'https://squid.subsquid.io/gs-main-moonriver/graphql'
-  DATA_SOURCE_GSQUID_MAIN_ASTAR: 'https://squid.subsquid.io/gs-main-astar/graphql'
+
+  DATA_SOURCE__SUBSQUID__POLKADOT__TRANSFER: 'https://squid.subsquid.io/gs-main-polkadot/graphql'
+  DATA_SOURCE__SUBSQUID__KUSAMA__TRANSFER: 'https://squid.subsquid.io/gs-main-kusama/graphql'
+  DATA_SOURCE__SUBSQUID__MOONBEAM__TRANSFER: 'https://squid.subsquid.io/gs-main-moonbeam/graphql'
+  DATA_SOURCE__SUBSQUID__MOONRIVER__TRANSFER: 'https://squid.subsquid.io/gs-main-moonriver/graphql'
+  DATA_SOURCE__SUBSQUID__ASTAR__TRANSFER: 'https://squid.subsquid.io/gs-main-astar/graphql'
+
+  DATA_SOURCE__SUBQUERY__POLKADOT__TRANSFER: 'https://api.subquery.network/sq/nova-wallet/nova-wallet-polkadot'
+  DATA_SOURCE__SUBQUERY__KUSAMA__TRANSFER: 'https://api.subquery.network/sq/nova-wallet/nova-wallet-kusama'
+  DATA_SOURCE__SUBQUERY__MOONBEAM__TRANSFER: 'https://api.subquery.network/sq/nova-wallet/nova-wallet-moonbeam'
+  DATA_SOURCE__SUBQUERY__MOONRIVER__TRANSFER: 'https://api.subquery.network/sq/nova-wallet/nova-wallet-moonriver'
+  DATA_SOURCE__SUBQUERY__ASTAR__TRANSFER: 'https://api.subquery.network/sq/nova-wallet/nova-wallet-astar'
+
+  DATA_SOURCE_PROVIDER_TRANSFER: 'SUBQUERY'
+
   NODE_ENV: 'production'
   APP_PORT: '3000'

package.json

@@ -20,7 +20,8 @@
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
     "test:e2e": "jest --config ./test/jest-e2e.json",
-    "codegen:gsquid-main-query": "graphql-codegen --require dotenv/config --config codegenConfig/codegen-gsquid-main-query.ts"
+    "codegen:gsquid-main-query": "graphql-codegen --require dotenv/config --config codegenConfig/codegen-gsquid-main-query.ts",
+    "codegen:subquery-nova-query": "graphql-codegen --require dotenv/config --config codegenConfig/codegen-subquery-nova-query.ts"
   },
   "dependencies": {
     "@apollo/server": "^4.7.2",

schema.gql

@@ -6,6 +6,8 @@ type Blockchain {
   id: String!
   text: String!
   info: String!
+  symbols: [String!]
+  prefix: Int
   tag: BlockchainTag!
   decimal: Int!
   logo: String!
@@ -22,10 +24,13 @@ enum BlockchainTag {
 
 type TransferNative {
   id: String!
+  blockchain: Blockchain!
   blockNumber: Int!
   extrinsicHash: String
+  eventIndex: Int
   timestamp: DateTime!
   amount: BigInt!
+  fee: BigInt
   success: Boolean!
   from: Account!
   to: Account!

src/common/entities/enums.ts

@@ -3,7 +3,7 @@ import {
   FindAccountTxHistoryOrderBy,
   NativeTransactionKind,
   QueryOrder,
-  TransactionKind,
+  TransactionKind, TransferDirection,
   VoteResult,
 } from '../../constants/common';
 import { BlockchainTag } from '../../constants/blockchain';
@@ -31,3 +31,7 @@ registerEnumType(NativeTransactionKind, {
 registerEnumType(BlockchainTag, {
   name: 'BlockchainTag',
 });
+
+registerEnumType(TransferDirection, {
+  name: 'TransferDirection',
+});

src/config.module.ts

@@ -1,9 +1,10 @@
 import { Global, Module, Provider } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { transformAndValidateSync } from 'class-transformer-validator';
-import { IsNotEmpty } from 'class-validator';
+import { IsEnum, IsNotEmpty } from 'class-validator';
 import * as dotenv from 'dotenv';
 import { Transform } from 'class-transformer';
+import { DataSourceProviders } from './constants/common';
 
 dotenv.config({ path: `${__dirname}/../.env.local` });
 
@@ -39,31 +40,56 @@ export class AppConfig {
   @IsNotEmpty()
   readonly AGGREGATOR_REDIS_ENABLE_SSL: boolean;
 
+  @Transform(({ value }: { value: string }) => +value)
   @IsNotEmpty()
-  readonly DATA_SOURCE_GSQUID_MAIN_POLKADOT: string;
+  readonly AGGREGATOR_HISTORY_RENEW_INTERVAL_MS: number;
 
+  @Transform(({ value }: { value: string }) => +value)
   @IsNotEmpty()
-  readonly DATA_SOURCE_GSQUID_MAIN_KUSAMA: string;
+  readonly AGGREGATOR_GS_MAIN_CHUNK_BLOCKS_SIZE: number;
 
   @IsNotEmpty()
-  readonly DATA_SOURCE_GSQUID_MAIN_MOONBEAM: string;
+  readonly NODE_ENV: string;
+
+  /**
+   * === DATA SOURCES ===
+   */
 
   @IsNotEmpty()
-  readonly DATA_SOURCE_GSQUID_MAIN_MOONRIVER: string;
+  @Transform(({ value }) => ('' + value).toUpperCase())
+  @IsEnum(DataSourceProviders)
+  readonly DATA_SOURCE_PROVIDER_TRANSFER: DataSourceProviders;
+
 
   @IsNotEmpty()
-  readonly DATA_SOURCE_GSQUID_MAIN_ASTAR: string;
+  readonly DATA_SOURCE__SUBSQUID__POLKADOT__TRANSFER: string;
 
-  @Transform(({ value }: { value: string }) => +value)
   @IsNotEmpty()
-  readonly AGGREGATOR_HISTORY_RENEW_INTERVAL_MS: number;
+  readonly DATA_SOURCE__SUBSQUID__KUSAMA__TRANSFER: string;
 
-  @Transform(({ value }: { value: string }) => +value)
   @IsNotEmpty()
-  readonly AGGREGATOR_GS_MAIN_CHUNK_BLOCKS_SIZE: number;
+  readonly DATA_SOURCE__SUBSQUID__MOONBEAM__TRANSFER: string;
 
   @IsNotEmpty()
-  readonly NODE_ENV: string;
+  readonly DATA_SOURCE__SUBSQUID__MOONRIVER__TRANSFER: string;
+
+  @IsNotEmpty()
+  readonly DATA_SOURCE__SUBSQUID__ASTAR__TRANSFER: string;
+
+  @IsNotEmpty()
+  readonly DATA_SOURCE__SUBQUERY__POLKADOT__TRANSFER: string;
+
+  @IsNotEmpty()
+  readonly DATA_SOURCE__SUBQUERY__KUSAMA__TRANSFER: string;
+
+  @IsNotEmpty()
+  readonly DATA_SOURCE__SUBQUERY__MOONBEAM__TRANSFER: string;
+
+  @IsNotEmpty()
+  readonly DATA_SOURCE__SUBQUERY__MOONRIVER__TRANSFER: string;
+
+  @IsNotEmpty()
+  readonly DATA_SOURCE__SUBQUERY__ASTAR__TRANSFER: string;
 }
 
 @Global()

src/constants/blockchain.ts

@@ -11,44 +11,56 @@ export enum BlockchainTag {
 
 // https://github.com/polkadot-js/apps/blob/7c12692ee34aadd815b282ff4dca3d0a4763a9ca/packages/apps-config/src/endpoints/productionRelayPolkadot.ts#L645
 
+// https://github.com/paritytech/ss58-registry/blob/main/ss58-registry.json
+
 export const supportedBlockchainDetails: Omit<Blockchain, 'id'>[] = [
   {
+    prefix: 5,
     text: 'Astar',
     info: 'astar',
     tag: BlockchainTag.ASTAR,
-    decimal: 10,
+    decimal: 18,
+    symbols: ['ASTR'],
     logo: '',
     color: '#1b6dc1d9',
   },
   {
+    prefix: 0,
     text: 'Polkadot',
     info: 'polkadot',
     tag: BlockchainTag.POLKADOT,
     decimal: 10,
+    symbols: ['DOT'],
     logo: '',
     color: '#1b6dc1d9',
   },
   {
+    prefix: 2,
     text: 'Kusama',
     info: 'kusama',
     tag: BlockchainTag.KUSAMA,
-    decimal: 10,
+    decimal: 12,
+    symbols: ['KSM'],
     logo: '',
     color: '#1b6dc1d9',
   },
   {
+    prefix: 1284,
     text: 'Moonbeam',
     info: 'moonbeam',
     tag: BlockchainTag.MOONBEAM,
     decimal: 10,
+    symbols: ['GLMR'],
     logo: '',
     color: '#1b6dc1d9',
   },
   {
+    prefix: 1285,
     text: 'Moonriver',
     info: 'moonriver',
     tag: BlockchainTag.MOONRIVER,
-    decimal: 10,
+    decimal: 18,
+    symbols: ['MOVR'],
     logo: '',
     color: '#1b6dc1d9',
   },
@@ -58,46 +70,36 @@ export const blockchainDataSourceConfigs = [
   {
     tag: BlockchainTag.POLKADOT,
     events: {
-      [NativeTransactionKind.TRANSFER]:
-        'https://squid.subsquid.io/gs-main-polkadot/graphql',
-      // [NativeTransactionKind.REWARD]:
-      //   'https://squid.subsquid.io/gs-main-polkadot/graphql',
+      [NativeTransactionKind.TRANSFER]: null,
+      // [NativeTransactionKind.REWARD]: null,
     },
   },
   {
     tag: BlockchainTag.KUSAMA,
     events: {
-      [NativeTransactionKind.TRANSFER]:
-        'https://squid.subsquid.io/gs-main-kusama/graphql',
-      // [NativeTransactionKind.REWARD]:
-      //   'https://squid.subsquid.io/gs-main-kusama/graphql',
+      [NativeTransactionKind.TRANSFER]: null,
+      // [NativeTransactionKind.REWARD]: null,
     },
   },
   {
     tag: BlockchainTag.MOONBEAM,
     events: {
-      [NativeTransactionKind.TRANSFER]:
-        'https://squid.subsquid.io/gs-main-moonbeam/graphql',
-      // [NativeTransactionKind.REWARD]:
-      //   'https://squid.subsquid.io/gs-main-moonbeam/graphql',
+      [NativeTransactionKind.TRANSFER]: null,
+      // [NativeTransactionKind.REWARD]: null,
     },
   },
   {
     tag: BlockchainTag.MOONRIVER,
     events: {
-      [NativeTransactionKind.TRANSFER]:
-        'https://squid.subsquid.io/gs-main-moonriver/graphql',
-      // [NativeTransactionKind.REWARD]:
-      //   'https://squid.subsquid.io/gs-main-moonriver/graphql',
+      [NativeTransactionKind.TRANSFER]: null,
+      // [NativeTransactionKind.REWARD]: null,
     },
   },
   {
     tag: BlockchainTag.ASTAR,
     events: {
-      [NativeTransactionKind.TRANSFER]:
-        'https://squid.subsquid.io/gs-main-astar/graphql',
-      // [NativeTransactionKind.REWARD]:
-      //   'https://squid.subsquid.io/gs-main-astar/graphql',
+      [NativeTransactionKind.TRANSFER]: null,
+      // [NativeTransactionKind.REWARD]: null,
     },
   },
 ] as const;

src/constants/common.ts

@@ -19,10 +19,18 @@ export enum TransactionKind {
   REWARD = 'REWARD',
 }
 
+export enum TransferDirection {
+  FROM = 'FROM',
+  TO = 'TO',
+}
+
 export enum NativeTransactionKind {
   TRANSFER = 'TRANSFER',
   VOTE = 'VOTE',
   REWARD = 'REWARD',
 }
 
-
+export enum DataSourceProviders {
+  SUBSQUID = 'SUBSQUID',
+  SUBQUERY = 'SUBQUERY',
+}

src/dependencyServiceModule.module.ts

@@ -1,7 +1,7 @@
 import { Module } from '@nestjs/common';
 import { CryptoUtils } from './utils/cryptoUtils';
 import { CommonUtils } from './utils/commonUtils';
-import { DataSourceUtils } from './utils/dataSourceUtils';
+import { DataSourceUtils } from './utils/dataSources/dataSourceUtils';
 
 @Module({
   providers: [CryptoUtils, CommonUtils, DataSourceUtils],

src/modules/dataAggregator/dataAggregator.module.ts

@@ -8,7 +8,6 @@ import { AccountService } from '../entities/account/account.service';
 import { Account } from '../entities/account/entities/account.entity';
 import { Blockchain } from '../entities/blockchain/entities/blockchain.entity';
 import { AggregationHelper } from './services/aggregation.helper';
-import { DataSourceUtils } from '../../utils/dataSourceUtils';
 import { TransferNativeService } from '../entities/transferNative/transferNative.service';
 import { TransactionService } from '../entities/transaction/transaction.service';
 import { AccountTransactionService } from '../entities/accountTransaction/accountTransaction.service';
@@ -18,6 +17,7 @@ import { BullModule } from '@nestjs/bull';
 import { SubIdAggregatorQueueName } from '../../constants/queues';
 import { Transaction } from '../entities/transaction/entities/transaction.entity';
 import { DatasourceChunksParallelHandlingProducer } from '../queueProcessor/services/producers/datasourceChunksParallelHandling.producer';
+import { DataSourceUtils } from '../../utils/dataSources/dataSourceUtils';
 
 @Module({
   imports: [

src/modules/dataAggregator/dto/collectTransfersChunkHandlerResponse.response.ts

@@ -1,5 +1,5 @@
-import { GetTransfersByAccountQuery } from '../../../utils/graphQl/gsquidMain/gsquid-main-query';
+import { TransferDecoratedDto } from './transfersByAccountDecorated.dto';
 
 export class CollectTransfersChunkHandlerResponseResponse {
-  fetchedChunkData: GetTransfersByAccountQuery['transfers'];
+  fetchedChunkData: TransferDecoratedDto[];
 }

src/modules/dataAggregator/dto/getIndexerLastProcessedHeight.args.dto.ts

@@ -0,0 +1,3 @@
+export class GetIndexerLastProcessedHeightArgs {
+  queryUrl: string;
+}

src/modules/dataAggregator/dto/getTransfersByAccount.args.dto.ts

@@ -1,8 +1,12 @@
+import { BlockchainTag } from '../../../constants/blockchain';
+
 export class GetTransfersByAccountArgs {
+  blockchainTag: BlockchainTag;
   limit: number;
   offset: number;
   blockNumber_gt: number;
   blockNumber_lt: number | null;
-  publicKey: string;
+  publicKey?: string;
+  address?: string;
   queryUrl: string;
 }