Feat/back 574

package.json

@@ -55,7 +55,7 @@
     "@godaddy/terminus": "4.9.0",
     "@snapshot-labs/snapshot.js": "^0.3.56",
     "@types/pg": "^8.6.1",
-    "axios": "0.24.0",
+    "axios": "^0.27.2",
     "axios-cache-adapter": "^2.7.3",
     "axios-rate-limit": "^1.3.0",
     "axios-retry": "^3.2.4",
@@ -67,6 +67,7 @@
     "dotenv": "10.0.0",
     "ethers": "^5.6.2",
     "express": "4.17.1",
+    "http-status-codes": "^2.2.0",
     "jsonwebtoken": "^8.5.1",
     "lodash": "4.17.21",
     "log4js": "6.3.0",

scripts/gas-refund-program/staking/spsp-stakes-tracker.ts

@@ -3,7 +3,6 @@ import { BigNumber as EthersBN, Contract, Event, EventFilter } from 'ethers';
 import {
   CHAIN_ID_MAINNET,
   NULL_ADDRESS,
-  PSP_ADDRESS,
 } from '../../../src/lib/constants';
 import { Provider } from '../../../src/lib/provider';
 import * as ERC20ABI from '../../../src/lib/abi/erc20.abi.json';
@@ -21,9 +20,12 @@ import {
   SPSPAddresses,
   SPSPHelper,
 } from '../../../src/lib/staking/spsp-helper';
+import { configLoader } from '../../../src/config';
 
 const logger = global.LOGGER('SPSPStakesTracker');
 
+const config = configLoader.getConfig(CHAIN_ID_MAINNET);
+
 const SPSPAddressesSet = new Set(SPSPAddresses);
 
 const SPSPPrototypeContract = new Contract(
@@ -33,7 +35,7 @@ const SPSPPrototypeContract = new Contract(
 );
 
 const PSPContract = new Contract(
-  PSP_ADDRESS[CHAIN_ID_MAINNET],
+  config.pspAddress,
   ERC20ABI,
   Provider.getJsonRpcProvider(CHAIN_ID_MAINNET),
 );

scripts/gas-refund-program/transactions-indexing/txs-covalent.ts

@@ -1,12 +1,15 @@
+import { configLoader } from '../../../src/config';
 import { covalentClient } from '../../../src/lib/utils/data-providers-clients';
 import {
   CovalentAPI,
   CovalentTransaction,
   GasRefundTransaction,
 } from '../types';
 
+const globalConfig = configLoader.getGlobalConfig();
+
 interface GetContractTXsByNetworkInput {
-  chainId: number;
+  chainId: number
   contract: string;
   startTimestamp: number;
   endTimestamp: number;
@@ -31,7 +34,6 @@ export const covalentGetTXsForContract = async ({
     contract,
   });
 
-  const { COVALENT_API_KEY } = process.env;
   const path = (page: number) => {
     /* Covalent API only has time relative pagination for tx scanning (give me tx within last X seconds).
      * We take a safety margin to counter possible edge case of relative - not absolute - range bounds
@@ -58,7 +60,7 @@ export const covalentGetTXsForContract = async ({
       throw new Error('only query historic data');
     }
 
-    return `/${chainId}/address/${contract}/transactions_v2/?key=${COVALENT_API_KEY}&no-logs=true&page-number=${page}&page-size=1000&block-signed-at-limit=${startSecondsAgo}&block-signed-at-span=${duration}&match={"to_address": "${contract}"}`;
+    return `/${chainId}/address/${contract}/transactions_v2/?key=${globalConfig.covalentV1ApiKey}&no-logs=true&page-number=${page}&page-size=1000&block-signed-at-limit=${startSecondsAgo}&block-signed-at-span=${duration}&match={"to_address": "${contract}"}`;
   };
 
   // todo: better would be to first call the end point with page-size=0 just to get the total number of items, and then construct many request promises and run concurrently - currently this isn't possible (as `total_count` is null) in the covalent api but scheduled

src/app.ts

@@ -1,12 +1,12 @@
 import {
-  VOLUME_TRACKER_INIT_TIME,
   VOLUME_TRACKER_SUPPORTED_NETWORKS,
 } from './lib/constants';
 import { shutdown as shutdownLog4js } from './lib/log4js';
 import Database from './database';
 import VolumeTracker from './lib/volume-tracker';
 import WebServer from './web-server';
 import { PoolInfo } from './lib/pool-info';
+import { configLoader } from './config';
 
 const logger = global.LOGGER();
 
@@ -23,13 +23,14 @@ export async function startApp() {
 
     await Promise.all(
       VOLUME_TRACKER_SUPPORTED_NETWORKS.map(network => {
-        if (!VOLUME_TRACKER_INIT_TIME[network]) {
+        const config = configLoader.getConfig(network);
+        if (!config.volumeTrackerInitTime) {
           throw new Error(
             'VolumeTracker INIT_TIME env var is missing for network ' + network,
           );
         }
         return VolumeTracker.createInstance(
-          VOLUME_TRACKER_INIT_TIME[network],
+          config.volumeTrackerInitTime,
           network,
         ).startIndexing();
       }),

src/config/index.ts

@@ -0,0 +1,113 @@
+/* eslint-disable */
+import axios from 'axios';
+import { SERVICE_CONFIGURATION_SERVICE_HTTP } from '../env';
+import { sleep } from './utils';
+import { ApplicationError } from '../errors';
+import { CHAIN_ID_MAINNET } from '../lib/constants';
+
+const serviceName = 'volumetracker';
+
+export const CORS_ALLOWED_HEADERS = 'Accept, Content-Type, Origin';
+export const REQUEST_BODY_SIZE_LIMIT_BYTES = 256 * 1024; // 256KB
+export const CACHE_CONTROL_PREFLIGHT_REQUESTS_MAX_AGE_SECS = 24 * 60 * 60; // 24 hours
+
+const CONFIG_SERVICE_TIMEOUT = 5000;
+const CONFIG_SERVICE_RETRY_INTERVAL = 3000;
+
+export type NetworkMap<T> = { [network: number]: T };
+
+export type Config = {
+  network: number;
+  augustusAddress: string;
+  augustusV4Address: string;
+  pspAddress: string;
+  isStaking: boolean;
+  rewardDistributionAddress: string;
+  safetyModuleAddress: string;
+  privateHttpArchiveProvider: string;
+  coinGeckoPlatform: string;
+  multicallV2Address: string;
+  volumeTrackerInitTime: number;
+};
+
+type GlobalConfig = {
+  apiKeyCoingecko: string;
+  covalentV1ApiKey: string;
+  covalentV1HttpUrl: string;
+  apiPrefineryHttp: string;
+  apiKeyPrefinery: string;
+  apiAplcapiHttp: string;
+  apiKeyAplcapi: string;
+  databaseUrl: string;
+  apiKeyCaptcha: string;
+  apiKeySubmitAccount: string;
+};
+
+type ConfigResponse = {
+  networks: NetworkMap<Config>;
+  global: GlobalConfig;
+};
+
+class ConfigLoader {
+  public byNetwork: NetworkMap<Config> = {};
+
+  public enabledNetworks: number[] = [];
+
+  public global?: GlobalConfig;
+
+  public isLoaded: Promise<void>;
+
+  public hasStartedNotifier?: (value: void | PromiseLike<void>) => void;
+
+  constructor() {
+    this.isLoaded = new Promise(resolve => {
+      this.hasStartedNotifier = resolve;
+    });
+  }
+
+  async load() {
+    console.log(`Try to get config from ${SERVICE_CONFIGURATION_SERVICE_HTTP}`);
+    while (true) {
+      try {
+        const configs = (
+          await axios.get<ConfigResponse>(
+            `${SERVICE_CONFIGURATION_SERVICE_HTTP}/configuration?service=${serviceName}`,
+            { timeout: CONFIG_SERVICE_TIMEOUT },
+          )
+        ).data;
+        this.global = configs.global;
+        for (const network in configs.networks) {
+          const config = configs.networks[network];
+          this.byNetwork[network] = config;
+        }
+        this.enabledNetworks.push(CHAIN_ID_MAINNET);
+        break;
+      } catch (e) {
+        console.error('Error downloading configuration:', e);
+      }
+      await sleep(CONFIG_SERVICE_RETRY_INTERVAL);
+    }
+    console.log(`received config`);
+  }
+
+  getConfig(network: number): Config {
+    const config = this.byNetwork[network];
+    if (!config) {
+      throw new ApplicationError(`Missing config for network ${network}`);
+    }
+    return config;
+  }
+
+  getGlobalConfig(): GlobalConfig {
+    if (!this.global) {
+      throw new ApplicationError(`Missing global config`);
+    }
+    return this.global;
+  }
+}
+export const configLoader = new ConfigLoader();
+
+export const init = async () => {
+  await configLoader.load();
+  configLoader.hasStartedNotifier!();
+};

src/config/utils.ts

@@ -0,0 +1,3 @@
+/* eslint-disable */
+export const sleep = (ms: number) =>
+  new Promise(resolve => setTimeout(resolve, ms));

src/database.ts

@@ -1,16 +1,14 @@
 import { Client } from 'pg';
 import { Sequelize } from 'sequelize-typescript';
 import * as cls from 'cls-hooked';
+import { IS_DEV } from './env';
+import { configLoader } from './config';
 
-const logger = global.LOGGER();
-
-const IS_DEV = process.env.NODE_ENV === 'development';
+const globalConfig = configLoader.getGlobalConfig();
 
-const DATABASE_URL =
-  process.env.DATABASE_URL ||
-  'postgres://paraswap:[email protected]:32780/volume_tracker';
+const logger = global.LOGGER();
 
-const DATABASE_NAME = process.env.DATABASE_NAME || 'volume_tracker';
+const DATABASE_NAME = 'volume_tracker';
 
 export class Database {
   sequelize: Sequelize;
@@ -22,14 +20,15 @@ export class Database {
     }
 
     // create a volume-tracker DB if it doesn't exist already
-    const connectionStringParts = DATABASE_URL.split('/');
+    const connectionStringParts = globalConfig.databaseUrl.split('/');
     const connectionStringDBName =
       connectionStringParts[connectionStringParts.length - 1];
     if (connectionStringDBName !== DATABASE_NAME) {
       logger.info(
         'Database name in connection string is different than expected',
       );
-      const client = new Client({ connectionString: DATABASE_URL });
+
+      const client = new Client({ connectionString: globalConfig.databaseUrl });
       await client.connect();
       try {
         await client.query(`CREATE DATABASE ${DATABASE_NAME};`);
@@ -62,7 +61,7 @@ export class Database {
     });
 
     try {
-      logger.info('Connecting to database...');
+      logger.info('Connecting to database...', connectionStringDBName);
       await this.sequelize.authenticate();
       logger.info('Connected to database');
     } catch (e) {

src/env.ts

@@ -0,0 +1,18 @@
+import { EnvironmentError } from './errors/environment-error';
+
+function getEnvOrThrow(
+  name: string,
+  defaultValue: string | undefined = undefined,
+) {
+  const value = process.env[name] || defaultValue;
+  if (value === undefined) {
+    throw new EnvironmentError(name);
+  }
+  return value;
+}
+export const SERVICE_CONFIGURATION_SERVICE_HTTP = getEnvOrThrow(
+  'SERVICE_CONFIGURATION_SERVICE_HTTP',
+);
+export const NODE_ENV = getEnvOrThrow('NODE_ENV', 'development');
+export const IS_DEV = NODE_ENV === 'development';
+export const PORT = parseInt(getEnvOrThrow('PORT', '3236'), 10);

src/errors/application-error.ts

@@ -0,0 +1,6 @@
+export class ApplicationError extends Error {
+  // eslint-disable-next-line no-useless-constructor
+  constructor(message: string, public isLogged: boolean = false) {
+    super(message);
+  }
+}

src/errors/config-error.ts

@@ -0,0 +1,8 @@
+import { ApplicationError } from './application-error';
+
+export class ConfigError extends ApplicationError {
+  // eslint-disable-next-line no-useless-constructor
+  constructor(message: string) {
+    super(message);
+  }
+}

src/errors/database-error.ts

@@ -0,0 +1,8 @@
+import { ApplicationError } from './application-error';
+
+export class DatabaseError extends ApplicationError {
+  // eslint-disable-next-line no-useless-constructor
+  constructor(message: string) {
+    super(message);
+  }
+}

src/errors/database-rollback-error.ts

@@ -0,0 +1,10 @@
+import { DatabaseError } from './database-error';
+
+// This error indicates only that we want to rollback, but not send the user InternalServerError
+// So this error should be caught after transaction and later returned the value you want
+export class DatabaseRollbackError extends DatabaseError {
+  // eslint-disable-next-line no-useless-constructor
+  constructor(message?: string) {
+    super(message || 'Database Rollback');
+  }
+}

src/errors/environment-error.ts

@@ -0,0 +1,7 @@
+import { ApplicationError } from './application-error';
+
+export class EnvironmentError extends ApplicationError {
+  constructor(key: string) {
+    super(`Required environment variable '${key}' expected!`);
+  }
+}

src/errors/index.ts

@@ -0,0 +1,13 @@
+export { ApplicationError } from './application-error';
+
+export { EnvironmentError } from './environment-error';
+export { DatabaseError } from './database-error';
+export { ConfigError } from './config-error';
+export { LimitOrderError } from './limit-order-error';
+
+export { RestError } from './rest-errors/rest-error';
+export { BadRequestError } from './rest-errors/bad-request-error';
+export { InternalServerError } from './rest-errors/internal-server-error';
+export { NotFoundError } from './rest-errors/not-found-error';
+
+export { ValidationError } from './rest-errors/validation-error';

src/errors/limit-order-error.ts

@@ -0,0 +1,8 @@
+import { ApplicationError } from './application-error';
+
+export class LimitOrderError extends ApplicationError {
+  // eslint-disable-next-line no-useless-constructor
+  constructor(message: string) {
+    super(message);
+  }
+}

src/errors/rest-errors/bad-request-error.ts

@@ -0,0 +1,8 @@
+import { StatusCodes } from 'http-status-codes';
+import { ClientSideError } from './client-side-error';
+
+export class BadRequestError extends ClientSideError {
+  constructor(message: string) {
+    super(message, StatusCodes.BAD_REQUEST);
+  }
+}

src/errors/rest-errors/black-list-error.ts

@@ -0,0 +1,7 @@
+import { ApplicationError } from '../application-error';
+
+export class BlackListError extends ApplicationError {
+  constructor(message: string, readonly statusCode: number) {
+    super(message);
+  }
+}

src/errors/rest-errors/client-side-error.ts

@@ -0,0 +1,8 @@
+import { RestError } from './rest-error';
+
+export class ClientSideError extends RestError {
+  // eslint-disable-next-line no-useless-constructor
+  constructor(message: string, statusCode: number) {
+    super(message, statusCode);
+  }
+}

src/errors/rest-errors/internal-server-error.ts

@@ -0,0 +1,8 @@
+import { StatusCodes } from 'http-status-codes';
+import { RestError } from './rest-error';
+
+export class InternalServerError extends RestError {
+  constructor(message: string) {
+    super(message, StatusCodes.INTERNAL_SERVER_ERROR);
+  }
+}