Indexer Integration (#5)

* feat: Implement a configuration parser * test: Add Tests for Uniqueness * Create pull_request_template.md * chore: Add Sample config.yaml * feat: init bin and api crate * feat: LinearRegressionEstimator * feat: storage and api updates * chore: Refactor mongodb and user apis * feat: RouteSource Implementation for BungeeClient * test: BungeeClient * feat: Token Price Provider and Utils * fix: Replace BigInt by Ruint * test: Fix Bungee Tests * feat: Build Estimators in Routes Indexer * fix: Add BUNGEE_API_KEY in github action * feat: refactor engine * fix: add display in all structs * feat: added chainning instead of options * Update test.yml * Update test.yml * feat: Estimator Flow * feat: Redis Model Store * chore: refactor of storage and AAS * fix: conflict resolve * temp: fix build * chore: remove code * edit readme * feat: Redis Publish Subscribe * fix: Migrate to thiserror * feat: Token Price Provider Integration * fix: Indexer * fix: Tests * temp: broken indexer * fix: abolish scheduler * fix: tests * chore: Enhance Logging * feat: Command Line Config Parser * feat: Dockerfile * fix: Dockerfile * fix: Github Tests * fix: Github Tests * fix: Github Tests * fix: Github Tests * fix: Make Config::test private --------- Co-authored-by: amanraj1608 <[email protected]> Co-authored-by: Aman Raj <[email protected]>
bcnmy · Jun 25, 2024 · d6c93a6 · d6c93a6
1 parent bd0f44f
commit d6c93a6
Show file tree

Hide file tree

Showing 24 changed files with 890 additions and 537 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -23,3 +23,6 @@
 
 # include example files
 !/examples
+
+*.yaml
+*.env
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,28 +1,30 @@
 name: Test Suite
-on: [pull_request, push]
+on: [ pull_request ]
 
 jobs:
   test:
     name: cargo test
     runs-on: ubuntu-latest
-
     services:
       mongodb:
-        image: mongo:6
+        image: mongo:latest
         ports:
           - 27017:27017
         options: >-
           --health-cmd="mongosh --eval 'db.adminCommand({ ping: 1 })'"
           --health-interval=30s
           --health-timeout=10s
           --health-retries=10
-
+      redis:
+        image: redis:latest
+        ports:
+          - 6379:6379
     steps:
       - uses: actions/checkout@v4
-
-      - name: Set up Rust
-        uses: dtolnay/rust-toolchain@stable
-
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo test --all-features
     env:
       BUNGEE_API_KEY: ${{ secrets.BUNGEE_API_KEY }}
+      COINGECKO_API_KEY: ${{ secrets.COINGECKO_API_KEY }}
     environment: Testing
diff --git a/.gitignore b/.gitignore
@@ -22,4 +22,4 @@ Cargo.lock
 
 *.env
 *.swp
-config.yaml
+*.yaml
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,11 @@
+FROM rust:latest as builder
+WORKDIR /reflux
+COPY . .
+RUN cargo install --path bin/reflux --profile release
+
+FROM debian:latest
+RUN apt-get update
+RUN apt-get upgrade -y
+RUN apt-get install -y libssl-dev ca-certificates
+COPY --from=builder /usr/local/cargo/bin/reflux /app/reflux
+
diff --git a/bin/reflux/Cargo.toml b/bin/reflux/Cargo.toml
@@ -8,6 +8,8 @@ tokio = { version = "1.38.0", features = ["full"] }
 tower-http = { version = "0.5.2", features = ["cors"] }
 axum = "0.7.5"
 log = "0.4.21"
+simple_logger = "5.0.0"
+clap = { version = "4.5.7", features = ["derive"] }
 
 # workspace dependencies
 account-aggregation = { workspace = true }

diff --git a/bin/reflux/src/main.rs b/bin/reflux/src/main.rs
@@ -1,36 +1,87 @@
+use std::time::Duration;
+
+use axum::http::Method;
+use clap::Parser;
+use log::{debug, error, info};
+use tokio;
+use tokio::signal;
+use tower_http::cors::{Any, CorsLayer};
+
 use account_aggregation::service::AccountAggregationService;
 use api::service_controller::ServiceController;
-use axum::http::Method;
 use config::Config;
-use log::info;
+use routing_engine::{BungeeClient, CoingeckoClient, Indexer};
 use routing_engine::engine::RoutingEngine;
+use routing_engine::estimator::LinearRegressionEstimator;
 use storage::mongodb_provider::MongoDBProvider;
-use tokio;
-use tokio::signal;
-use tower_http::cors::{Any, CorsLayer};
+use storage::RedisClient;
+
+#[derive(Parser, Debug)]
+struct Args {
+    /// Run the Solver (default)
+    #[arg(short, long)]
+    solver: bool,
+
+    /// Run the Indexer
+    #[arg(short, long)]
+    indexer: bool,
+
+    /// Config file path
+    #[arg(short, long, default_value = "config.yaml")]
+    config: String,
+}
 
 #[tokio::main]
 async fn main() {
+    simple_logger::SimpleLogger::new().env().init().unwrap();
+
+    let mut args = Args::parse();
+    debug!("Args: {:?}", args);
+
+    if args.indexer && args.solver {
+        panic!("Cannot run both indexer and solver at the same time");
+    }
+
+    if !args.indexer && !args.solver {
+        args.solver = true;
+        debug!("Running Solver by default");
+    }
+
     // Load configuration from yaml
-    let config = Config::from_file("config.yaml").expect("Failed to load config file");
-    let mongodb_uri = config.infra.mongo_url;
-    let (app_host, app_port) = (config.server.host, config.server.port);
+    let config = Config::from_file(&args.config).expect("Failed to load config file");
+
+    if args.indexer {
+        run_indexer(config).await;
+    } else if args.solver {
+        run_solver(config).await;
+    }
+}
+
+async fn run_solver(config: Config) {
+    info!("Starting Reflux Server");
+
+    let (app_host, app_port) = (config.server.host.clone(), config.server.port.clone());
 
     // Instance of MongoDBProvider for users and account mappings
-    let user_db_provider =
-        MongoDBProvider::new(&mongodb_uri, "reflux".to_string(), "users".to_string(), true)
-            .await
-            .expect("Failed to create MongoDB provider for users");
+    let user_db_provider = MongoDBProvider::new(
+        &config.infra.mongo_url,
+        "reflux".to_string(),
+        "users".to_string(),
+        true,
+    )
+    .await
+    .expect("Failed to create MongoDB provider for users");
     let account_mapping_db_provider = MongoDBProvider::new(
-        &mongodb_uri,
+        &config.infra.mongo_url,
         "reflux".to_string(),
         "account_mappings".to_string(),
         false,
     )
     .await
     .expect("Failed to create MongoDB provider for account mappings");
 
-    let (covalent_base_url, covalent_api_key) = (config.covalent.base_url, config.covalent.api_key);
+    let (covalent_base_url, covalent_api_key) =
+        (config.covalent.base_url.clone(), config.covalent.api_key.clone());
 
     // Initialize account aggregation service for api
     let account_service = AccountAggregationService::new(
@@ -65,6 +116,40 @@ async fn main() {
     info!("Server stopped.");
 }
 
+async fn run_indexer(config: Config) {
+    info!("Configuring Indexer");
+
+    let config = config;
+
+    let redis_provider = RedisClient::build(&config.infra.redis_url)
+        .await
+        .expect("Failed to instantiate redis client");
+
+    let bungee_client = BungeeClient::new(&config.bungee.base_url, &config.bungee.api_key)
+        .expect("Failed to Instantiate Bungee Client");
+
+    let token_price_provider = CoingeckoClient::new(
+        &config.coingecko.base_url,
+        &config.coingecko.api_key,
+        &redis_provider,
+        Duration::from_secs(config.coingecko.expiry_sec),
+    );
+
+    let indexer: Indexer<BungeeClient, RedisClient, RedisClient, CoingeckoClient<RedisClient>> =
+        Indexer::new(
+            &config,
+            &bungee_client,
+            &redis_provider,
+            &redis_provider,
+            &token_price_provider,
+        );
+
+    match indexer.run::<LinearRegressionEstimator>().await {
+        Ok(_) => info!("Indexer Job Completed"),
+        Err(e) => error!("Indexer Job Failed: {}", e),
+    };
+}
+
 async fn shutdown_signal() {
     let ctrl_c = async {
         signal::ctrl_c().await.expect("Unable to handle ctrl+c");

diff --git a/config.yaml.example b/config.yaml.example
@@ -2,57 +2,76 @@ chains:
   - id: 1
     name: Ethereum
     is_enabled: true
-  - id: 56
-    name: Binance Smart Chain
+  - id: 42161
+    name: Arbitrum
     is_enabled: true
 tokens:
-  - symbol: ETH
+  - symbol: USDC
     is_enabled: true
+    coingecko_symbol: usd-coin
     by_chain:
       1:
         is_enabled: true
-        decimals: 18
-        address: '0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE'
-      56:
+        decimals: 6
+        address: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48'
+      42161:
         is_enabled: true
-        decimals: 18
-        address: '0x2170Ed0880ac9A755fd29B2688956BD959F933F8'
-  - symbol: BNB
-    is_enabled: true
-    by_chain:
-      1:
-        is_enabled: false
-        decimals: 18
-        address: '0xB8c77482e45F1F44dE1745F52C74426C631bDD52'
-      56:
-        is_enabled: true
-        decimals: 18
-        address: '0xEEeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
+        decimals: 6
+        address: '0xaf88d065e77c8cC2239327C5EDb3A432268e5831'
 buckets:
   - from_chain_id: 1
-    to_chain_id: 56
-    from_token: ETH
-    to_token: BNB
-    is_smart_contract_deposit_supported: true
-    token_amount_from_usd: 0.1
+    to_chain_id: 42161
+    from_token: USDC
+    to_token: USDC
+    is_smart_contract_deposit_supported: false
+    token_amount_from_usd: 1
+    token_amount_to_usd: 10
+  - from_chain_id: 1
+    to_chain_id: 42161
+    from_token: USDC
+    to_token: USDC
+    is_smart_contract_deposit_supported: false
+    token_amount_from_usd: 10
     token_amount_to_usd: 100
+  - from_chain_id: 1
+    to_chain_id: 42161
+    from_token: USDC
+    to_token: USDC
+    is_smart_contract_deposit_supported: false
+    token_amount_from_usd: 100
+    token_amount_to_usd: 1000
+  - from_chain_id: 1
+    to_chain_id: 42161
+    from_token: USDC
+    to_token: USDC
+    is_smart_contract_deposit_supported: false
+    token_amount_from_usd: 1000
+    token_amount_to_usd: 10000
+  - from_chain_id: 1
+    to_chain_id: 42161
+    from_token: USDC
+    to_token: USDC
+    is_smart_contract_deposit_supported: false
+    token_amount_from_usd: 10000
+    token_amount_to_usd: 100000
 bungee:
-    base_url: 'https://api.bungee.exchange'
-    api_key: 'my-api'
+  base_url: https://api.socket.tech/v2
+  api_key:
 covalent:
-    base_url: 'https://api.bungee.exchange'
-    api_key: 'my-api'
+  base_url: ''
+  api_key: 'my-api'
 coingecko:
-    base_url: 'https://api.coingecko.com'
-    api_key: 'my-api'
+  base_url: https://api.coingecko.com/api/v3
+  api_key:
+  expiry_sec: 300
 infra:
-    redis_url: 'redis://localhost:6379'
-    rabbitmq_url: 'amqp://localhost:5672'
-    mongo_url: 'mongodb://localhost:27017'
+  redis_url: redis://localhost:6379
+  mongo_url: mongodb://127.0.0.1:27017
 server:
-    port: 8080
-    host: 'localhost'
+  port: 8080
+  host: localhost
 indexer_config:
-    is_indexer: true
-    indexer_update_topic: indexer_update
-    indexer_update_message: message
+  indexer_update_topic: indexer_update
+  indexer_update_message: message
+  points_per_bucket: 3
+
-Original file line number
+Diff line change
@@ Expand Up / @@ -23,3 +23,6 @@ @@
     # include example files
     !/examples
+    *.yaml
+    *.env