Feature/clap (#179)

* update: basic cli setup

* update: run_cmd for settlement client

* update: working instrumentation run_cmd

* update: run_cmd for storage

* update: run_cmd for server, aws config

* update: run_cmd for da and alert

* update: run_cmd for sharp and snos

* update: run_cmd for madara rpc url

* update: run_cmd adaptation for: testcases

* update: rework e2e

* update: lint cleanu

* update: working mutual exclusion

* update: fixed telemetry tests

* update: tests cleanup

* update: coverage build fix

* update: lint fix

* update: lint e2e cargo fix

* lint fix

* update: reworked cli params for tests

* update: aws args renamed

* update: moved cli from utils to orchestrator

* update: removed envsettingsprovider

* update: removed serde added strum for queuetype

* update: added orchestrator config

* update: env vars renamed

* update: env fixes for tests

* update: fixed aws related issue

* update: fixing env

* update: coverage fixes

* update: coverage fixes

* update: fixes for test_Settle and e2e

* fixed: coverage tests

* update: test_settle fixed

* update: allow e2e and coverage to work together based off env

* update: fixed db issue

* update: lint fixes

* update: lint fix

* update: clean up

* update: unwraps to expect and regrouping

* update: env correctiong

* update: removed config

* update: removed config.rs

* update: renamed XYZParams to XYZValidatedArgs

* update: lint fixes

* update: lint fixes

* update: fixed e2e testcase

* update: added tests for validate fns

* update: testcase fix

* update: added command run / setup and e2e with setup_config

* update: setup and run fns

* update: systematic rework

* update: ProviderValidatedArgs

* update: reworked commands run and setup

* update: optimised fixes

* update: added e2e corrections

* update: e2e fixed

* update: fixed coverage

* update: MADARA_ORCHESTRATOR_EVENT_BRIDGE_CRON_TIME to 60 seconds

* update: testcase fixes

* resolved: comments

* resolve: removed bucket name from require_all

* chore: comments added for the decisions taken

* chore: setup cloud todo removed, plan dropped to move it to main.rs

---------

Co-authored-by: mohiiit <[email protected]>

.env.example

@@ -1,8 +1,8 @@
 ##### ORCHESTRATOR #####
 
-HOST=
-PORT=
-MAX_BLOCK_TO_PROCESS=
+MADARA_ORCHESTRATOR_HOST=
+MADARA_ORCHESTRATOR_PORT=
+MADARA_ORCHESTRATOR_MAX_BLOCK_NO_TO_PROCESS=
 
 ##### AWS CONFIG #####
 
@@ -17,7 +17,7 @@ AWS_DEFAULT_REGION="localhost"
 ##### STORAGE #####
 
 DATA_STORAGE=
-AWS_S3_BUCKET_NAME=
+MADARA_ORCHESTRATOR_AWS_S3_BUCKET_NAME=
 
 ##### QUEUE #####
 
@@ -30,39 +30,39 @@ SQS_WORKER_TRIGGER_QUEUE_URL=
 ##### SNS #####
 
 ALERTS="sns"
-AWS_SNS_ARN="arn:aws:sns:us-east-1:000000000000:madara-orchestrator-arn"
+MADARA_ORCHESTRATOR_AWS_SNS_ARN="arn:aws:sns:us-east-1:000000000000:madara-orchestrator-arn"
 
 ##### DATABASE #####
 
 DATABASE=
-MONGODB_CONNECTION_STRING=
-DATABASE_NAME=
+MADARA_ORCHESTRATOR_MONGODB_CONNECTION_URL=
+MADARA_ORCHESTRATOR_DATABASE_NAME=
 
 ##### PROVER #####
 
 PROVER_SERVICE=
-SHARP_CUSTOMER_ID=
-SHARP_URL=
-SHARP_USER_CRT=
-SHARP_USER_KEY=
-SHARP_SERVER_CRT=
-SHARP_PROOF_LAYOUT=
+MADARA_ORCHESTRATOR_SHARP_CUSTOMER_ID=
+MADARA_ORCHESTRATOR_SHARP_URL=
+MADARA_ORCHESTRATOR_SHARP_USER_CRT=
+MADARA_ORCHESTRATOR_SHARP_USER_KEY=
+MADARA_ORCHESTRATOR_SHARP_SERVER_CRT=
+MADARA_ORCHESTRATOR_SHARP_PROOF_LAYOUT=
 
 ##### ON CHAIN CONFIG #####
 
 DA_LAYER=
 SETTLEMENT_LAYER=
-SETTLEMENT_RPC_URL=
-MADARA_RPC_URL=
-GPS_VERIFIER_CONTRACT_ADDRESS=
+MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL=
+MADARA_ORCHESTRATOR_MADARA_RPC_URL=
+MADARA_ORCHESTRATOR_GPS_VERIFIER_CONTRACT_ADDRESS=
 PRIVATE_KEY=
-ETHEREUM_PRIVATE_KEY=
-L1_CORE_CONTRACT_ADDRESS=
+MADARA_ORCHESTRATOR_ETHEREUM_PRIVATE_KEY=
+MADARA_ORCHESTRATOR_L1_CORE_CONTRACT_ADDRESS=
 
 ##### STARKNET SETTLEMENT (L3s) #####
-STARKNET_PRIVATE_KEY=
-STARKNET_ACCOUNT_ADDRESS=
+MADARA_ORCHESTRATOR_STARKNET_PRIVATE_KEY=
+MADARA_ORCHESTRATOR_STARKNET_ACCOUNT_ADDRESS=
 
 ##### Instrumentation #####
-OTEL_SERVICE_NAME=
-OTEL_COLLECTOR_ENDPOINT=
+MADARA_ORCHESTRATOR_OTEL_SERVICE_NAME=
+MADARA_ORCHESTRATOR_OTEL_COLLECTOR_ENDPOINT=

.github/workflows/coverage.yml

@@ -110,7 +110,7 @@ jobs:
           mv target/debug/madara ../madara-binary
           cd ..
           echo -e "
-          MADARA_BINARY_PATH=\"$(pwd)/madara-binary\"" >> .env.test
+          MADARA_ORCHESTRATOR_MADARA_BINARY_PATH=\"$(pwd)/madara-binary\"" >> .env.test
           cat .env.test
 
       - name: Getting neccesary files for testing
@@ -128,8 +128,8 @@ jobs:
 
       - name: Run llvm-cov tests
         env:
-          SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
-          RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
+          MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
+          MADARA_ORCHESTRATOR_RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
           # the self hosted runner has a different region so we override it here
           AWS_REGION: us-east-1
         run: RUST_LOG=debug RUST_BACKTRACE=1 cargo llvm-cov nextest --release --features testing --lcov --output-path lcov.info --test-threads=1 --workspace --exclude=e2e-tests --no-fail-fast

.github/workflows/e2e-test.yml

@@ -83,8 +83,8 @@ jobs:
 
       - name: Run e2e test
         env:
-          SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
-          RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
+          MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
+          MADARA_ORCHESTRATOR_RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
           # the self hosted runner has a different region so we override it here
           AWS_REGION: us-east-1
         run: |

CHANGELOG.md

@@ -6,7 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Added
 
-- setup functions added for cloud and db
+- Added cli args support for all the services
+- Setup functions added for cloud and db
 - panic handling in process job
 - upgrade ETH L1 bridge for withdrawals to work
 - added makefile and submodules

Cargo.toml

@@ -33,6 +33,7 @@ alloy = { version = "0.2.1", features = [
   "json-rpc",
   "rpc-client",
 ] }
+alloy-primitives = { version = "0.7.7", default-features = false }
 aws-config = { version = "1.1.7", features = ["behavior-version-latest"] }
 aws-sdk-s3 = { version = "1.38.0", features = ["behavior-version-latest"] }
 aws-sdk-eventbridge = { version = "1.41.0", features = [
@@ -60,6 +61,8 @@ rstest = "0.22.0"
 serde = { version = "1.0.197", features = ["derive"] }
 serde_json = "1.0.114"
 starknet = "0.11.0"
+strum = "0.26.0"
+strum_macros = "0.26.0"
 tempfile = "3.12.0"
 thiserror = "1.0.57"
 tokio = { version = "1.37.0" }
@@ -74,6 +77,7 @@ lazy_static = "1.4.0"
 stark_evm_adapter = "0.1.1"
 hex = "0.4"
 itertools = "0.13.0"
+clap = { version = "4.4", features = ["derive", "env"] }
 mockall = "0.13.0"
 testcontainers = "0.18.0"
 once_cell = "1.8"

crates/da-clients/ethereum/src/lib.rs

@@ -13,19 +13,30 @@ use da_client_interface::{DaClient, DaVerificationStatus};
 use mockall::automock;
 use mockall::predicate::*;
 use reqwest::Client;
+use serde::{Deserialize, Serialize};
 use url::Url;
-use utils::settings::Settings;
 
-use crate::config::EthereumDaConfig;
-
-pub const DA_SETTINGS_NAME: &str = "ethereum";
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct EthereumDaValidatedArgs {
+    pub ethereum_da_rpc_url: Url,
+}
 
-pub mod config;
 pub struct EthereumDaClient {
     #[allow(dead_code)]
     provider: RootProvider<Ethereum, Http<Client>>,
 }
 
+impl EthereumDaClient {
+    pub async fn new_with_args(ethereum_da_params: &EthereumDaValidatedArgs) -> Self {
+        let client = RpcClient::new_http(
+            Url::from_str(ethereum_da_params.ethereum_da_rpc_url.as_str())
+                .expect("Failed to parse ethereum_da_rpc_url"),
+        );
+        let provider = ProviderBuilder::<_, Ethereum>::new().on_client(client);
+        Self { provider }
+    }
+}
+
 #[automock]
 #[async_trait]
 impl DaClient for EthereumDaClient {
@@ -47,14 +58,3 @@ impl DaClient for EthereumDaClient {
         131072
     }
 }
-
-impl EthereumDaClient {
-    pub fn new_with_settings(settings: &impl Settings) -> Self {
-        let config = EthereumDaConfig::new_with_settings(settings)
-            .expect("Not able to create EthereumDaClient from given settings.");
-        let client =
-            RpcClient::new_http(Url::from_str(config.rpc_url.as_str()).expect("Failed to parse SETTLEMENT_RPC_URL"));
-        let provider = ProviderBuilder::<_, Ethereum>::new().on_client(client);
-        EthereumDaClient { provider }
-    }
-}

crates/orchestrator/Cargo.toml

@@ -31,6 +31,7 @@ bytes.workspace = true
 c-kzg = { workspace = true }
 cairo-vm = { workspace = true }
 chrono = { workspace = true }
+clap.workspace = true
 color-eyre = { workspace = true }
 da-client-interface = { workspace = true }
 dotenvy = { workspace = true }
@@ -62,7 +63,8 @@ starknet = { workspace = true }
 starknet-core = "0.9.0"
 starknet-os = { workspace = true }
 starknet-settlement-client = { workspace = true }
-strum_macros = "0.26.4"
+strum = { workspace = true }
+strum_macros = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] }

crates/orchestrator/src/alerts/aws_sns/mod.rs

@@ -1,28 +1,24 @@
-mod config;
-
-use std::sync::Arc;
-
 use async_trait::async_trait;
+use aws_config::SdkConfig;
 use aws_sdk_sns::Client;
-use utils::settings::Settings;
 
-use crate::alerts::aws_sns::config::AWSSNSConfig;
 use crate::alerts::Alerts;
-use crate::config::ProviderConfig;
 
-pub const AWS_SNS_SETTINGS_NAME: &str = "sns";
+#[derive(Debug, Clone)]
+pub struct AWSSNSValidatedArgs {
+    // TODO: convert to ARN type, and validate it
+    // NOTE: aws is using str to represent ARN : https://docs.aws.amazon.com/sdk-for-rust/latest/dg/rust_sns_code_examples.html
+    pub topic_arn: String,
+}
 
 pub struct AWSSNS {
     client: Client,
     topic_arn: String,
 }
 
 impl AWSSNS {
-    pub async fn new_with_settings(settings: &impl Settings, provider_config: Arc<ProviderConfig>) -> Self {
-        let sns_config =
-            AWSSNSConfig::new_with_settings(settings).expect("Not able to get Aws sns config from provided settings");
-        let config = provider_config.get_aws_client_or_panic();
-        Self { client: Client::new(config), topic_arn: sns_config.sns_arn }
+    pub async fn new_with_args(aws_sns_params: &AWSSNSValidatedArgs, aws_config: &SdkConfig) -> Self {
+        Self { client: Client::new(aws_config), topic_arn: aws_sns_params.topic_arn.clone() }
     }
 }
 
@@ -39,4 +35,8 @@ impl Alerts for AWSSNS {
         log::info!("SNS topic created. Topic ARN: {}", topic_arn);
         Ok(())
     }
+
+    async fn get_topic_name(&self) -> String {
+        self.topic_arn.split(":").last().expect("Failed to get last part of topic ARN").to_string()
+    }
 }

crates/orchestrator/src/alerts/mod.rs

@@ -1,6 +1,5 @@
 use async_trait::async_trait;
 use mockall::automock;
-use utils::settings::Settings;
 
 pub mod aws_sns;
 
@@ -9,10 +8,11 @@ pub mod aws_sns;
 pub trait Alerts: Send + Sync {
     /// To send an alert message to our alert service
     async fn send_alert_message(&self, message_body: String) -> color_eyre::Result<()>;
+    async fn get_topic_name(&self) -> String;
     async fn create_alert(&self, topic_name: &str) -> color_eyre::Result<()>;
-    async fn setup(&self, settings_provider: Box<dyn Settings>) -> color_eyre::Result<()> {
-        let sns_topic_name = settings_provider.get_settings_or_panic("ALERT_TOPIC_NAME");
-        self.create_alert(&sns_topic_name).await?;
+    async fn setup(&self) -> color_eyre::Result<()> {
+        let topic_name = self.get_topic_name().await;
+        self.create_alert(&topic_name).await?;
         Ok(())
     }
 }