refactor aws and clean .env files (#98)

* refactor aws * env cleanup and build_database_client * remove get_credentials_and_region_from_config * region for test case * fix localstack not working * add port and host to .env.test * fix eth pk * fix CI --------- Co-authored-by: Arun Jangra <[email protected]>
madara-alliance · Aug 26, 2024 · 77bf9c6 · 77bf9c6
1 parent 0ad9bfb
commit 77bf9c6
Show file tree

Hide file tree

Showing 10 changed files with 120 additions and 182 deletions.
diff --git a/.env.example b/.env.example
@@ -1,50 +1,51 @@
+##### ORCHESTRATOR #####
+
 HOST=
 PORT=
-DATABASE_URL=
-MADARA_RPC_URL=
-DA_LAYER=
-SETTLEMENT_LAYER=
 
-# Ethereum
-ETHEREUM_PRIVATE_KEY=
-ETHEREUM_RPC_URL=
-MEMORY_PAGES_CONTRACT_ADDRESS=
-STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS=
+##### AWS CONFIG #####
 
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_REGION=
+AWS_ENDPOINT_URL=
 
-# Starknet
-STARKNET_PUBLIC_KEY=
-STARNET_PRIVATE_KEY=
-STARKNET_RPC_URL=
-STARKNET_CAIRO_CORE_CONTRACT_ADDRESS=
+##### STORAGE #####
 
-# MongoDB connection string
-MONGODB_CONNECTION_STRING=
+DATA_STORAGE=
+AWS_S3_BUCKET_NAME=
 
-# AWS
-AWS_ACCESS_KEY_ID=
-AWS_SECRET_ACCESS_KEY=
-AWS_DEFAULT_REGION=
+##### QUEUE #####
 
-# SQS
+QUEUE_PROVIDER=
 SQS_JOB_PROCESSING_QUEUE_URL=
 SQS_JOB_VERIFICATION_QUEUE_URL=
 SQS_JOB_HANDLE_FAILURE_QUEUE_URL=
 SQS_WORKER_TRIGGER_QUEUE_URL=
 
-# S3
-AWS_S3_BUCKET_NAME=
-AWS_S3_BUCKET_REGION=
+##### DATABASE #####
 
+DATABASE=
+MONGODB_CONNECTION_STRING=
 
-# Ethereum Settlement
-DEFAULT_SETTLEMENT_CLIENT_RPC=
-DEFAULT_L1_CORE_CONTRACT_ADDRESS=
+##### PROVER #####
 
-# Sharp Services
+PROVER_SERVICE=
 SHARP_CUSTOMER_ID=
+SHARP_URL=
+# [IMP!!!] These are test certificates (they don't work)
 SHARP_USER_CRT=
 SHARP_USER_KEY=
 SHARP_SERVER_CRT=
 SHARP_PROOF_LAYOUT=
 
+##### ON CHAIN CONFIG #####
+
+DA_LAYER=
+SETTLEMENT_LAYER=
+ETHEREUM_RPC_URL=
+MADARA_RPC_URL=
+MEMORY_PAGES_CONTRACT_ADDRESS=
+PRIVATE_KEY=
+ETHEREUM_PRIVATE_KEY=
+STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS=
diff --git a/.env.test b/.env.test
@@ -1,45 +1,52 @@
-##### AWS config #####
+##### ORCHESTRATOR #####
+
+HOST=127.0.0.1
+PORT=3000
+
+##### AWS CONFIG #####
 
 AWS_ACCESS_KEY_ID="AWS_ACCESS_KEY_ID"
 AWS_SECRET_ACCESS_KEY="AWS_SECRET_ACCESS_KEY"
-AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket"
-AWS_S3_BUCKET_REGION="us-east-1"
+AWS_REGION="us-east-1"
 AWS_ENDPOINT_URL="http://localhost.localstack.cloud:4566"
+
+##### STORAGE #####
+
+DATA_STORAGE="s3"
+AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket"
+
+##### QUEUE #####
+
+QUEUE_PROVIDER="sqs"
 SQS_JOB_PROCESSING_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_processing_queue"
 SQS_JOB_VERIFICATION_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_verification_queue"
 SQS_JOB_HANDLE_FAILURE_QUEUE_URL=
 SQS_WORKER_TRIGGER_QUEUE_URL=
-AWS_DEFAULT_REGION="localhost"
-
-##### On chain config #####
-
-MADARA_RPC_URL="http://localhost:3000"
-ETHEREUM_RPC_URL="http://localhost:3001"
-MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead"
-PRIVATE_KEY="0xdead"
-# Private key of Test wallet provided by Anvil
-ETHEREUM_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
-STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead"
 
-##### Config URLs #####
+##### DATABASE #####
 
-DA_LAYER="ethereum"
-PROVER_SERVICE="sharp"
-SETTLEMENT_LAYER="ethereum"
-DATA_STORAGE="s3"
+DATABASE="mongodb"
 MONGODB_CONNECTION_STRING="mongodb://localhost:27017"
-DEFAULT_SETTLEMENT_CLIENT_RPC="http://localhost:3000"
 
-# Ethereum Settlement
-DEFAULT_L1_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4"
-SHOULD_IMPERSONATE_ACCOUNT="true"
-TEST_DUMMY_CONTRACT_ADDRESS="0xE5b6F5e695BA6E4aeD92B68c4CC8Df1160D69A81"
+##### PROVER #####
 
-# Sharp Services
+PROVER_SERVICE="sharp"
 SHARP_CUSTOMER_ID="sharp_consumer_id"
 SHARP_URL="http://127.0.0.1:5000"
 # [IMP!!!] These are test certificates (they don't work)
 SHARP_USER_CRT="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR4ekNDQXErZ0F3SUJBZ0lVTjBSK0xpb1MzL2ZadUZsK291RjZNNFk2RnRZd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2N6RUxNQWtHQTFVRUJoTUNTVTR4RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeElUQWZCZ05WQkFvTQpHRWx1ZEdWeWJtVjBJRmRwWkdkcGRITWdVSFI1SUV4MFpERU5NQXNHQTFVRUF3d0VVMVJTU3pFZE1Cc0dDU3FHClNJYjNEUUVKQVJZT1lXSmpRR3RoY201dmRDNTRlWG93SGhjTk1qUXdPREV6TVRNd05UTTBXaGNOTWpVd09ERXoKTVRNd05UTTBXakJ6TVFzd0NRWURWUVFHRXdKSlRqRVRNQkVHQTFVRUNBd0tVMjl0WlMxVGRHRjBaVEVoTUI4RwpBMVVFQ2d3WVNXNTBaWEp1WlhRZ1YybGtaMmwwY3lCUWRIa2dUSFJrTVEwd0N3WURWUVFEREFSVFZGSkxNUjB3Ckd3WUpLb1pJaHZjTkFRa0JGZzVoWW1OQWEyRnlibTkwTG5oNWVqQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQUQKZ2dFUEFEQ0NBUW9DZ2dFQkFOSEtaUGRqWSs4QWo4ZFV2V0xReEl5NTNrK1BHY001T2FlYnpTV3FER0xGSlBOdgpkVzJvWjFCSnNEb2hobWZFSCt5ZEFoQXEvbzc4NDljblg2VDJTOVhta25wdnNud2dRckU5Z3lqSmV3MUxBRzNHCm10U0lOMWJJSm9peWJ3QUR5NGxPd0xrVzUzdFdueHBSazVVVmZUU1hLYVRRTnlHd2o3Q2xMSGthcnlZYVk3OVkKOXlHMFJ2RkFkb1IzczBveWthNkFLV0d1WjhOdWd4NTY2bysyWllRenJteWVNU1NGYkhNdW1aUkxYb0hpazhBSgpLZXJ0bnNBRC9LMVJRYm80Y21ubHFoTVRhQktiTEFVVjVteFVvMlpveFBJVU9tREE5N3IyMmRTYkRkRlVjeC9kCjhQcDB6VXNycXdQckJlcW5SMXdLOE80MUlHajUzRnUzVmxDeS94MENBd0VBQWFOVE1GRXdIUVlEVlIwT0JCWUUKRkc0T0lvKzcvckJyZlR4S2FFMGx2L1dwRDJ3UE1COEdBMVVkSXdRWU1CYUFGRzRPSW8rNy9yQnJmVHhLYUUwbAp2L1dwRDJ3UE1BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFEMURDZkR3CnpoSXRGMWd5YVdhWURZRHErZjJSUHBFRWVaWk1BSDdJV0ZTajRrTzhmVHN1RnN6bFoyNXNlR3ZHYW4xQ3F4alQKYnJ3MXliVlJQeGZMUWgxRlZMMGhFeDZWYXhGditxMmtqUmlCQmZURFBxWGxYcmpaaUYrZTNPS3lKSVhnNkpIUAppbVpBV0dyRFBHNkorQi90bHRaQ3VLZVhLK1FUcnRSOVVCL29hOWVaQWc5RXNkOVJsZDRNeVo5b0NtdUNPU1hmCnk1THFkVlgrNENpTnJXQ3BwM1B2M2MyL28rZ0RMQjUzZ252R056RjR6Q1FIZ0RtN0RNZnpmZlY1TUMwV1MvWXkKVnpyUG11Sys0Y0tSK3dMOFZITVNEeC9ybTFhYnh0dEN2VW92MUw5dVZ1QUNGc29yNmdsR0N1RDNNQ0dIa0pNNgpxaS8rM1haeHhxeGw1Rzg9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"
 SHARP_USER_KEY="LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRRFJ5bVQzWTJQdkFJL0gKVkwxaTBNU011ZDVQanhuRE9UbW5tODBscWd4aXhTVHpiM1Z0cUdkUVNiQTZJWVpueEIvc25RSVFLdjZPL09QWApKMStrOWt2VjVwSjZiN0o4SUVLeFBZTW95WHNOU3dCdHhwclVpRGRXeUNhSXNtOEFBOHVKVHNDNUZ1ZDdWcDhhClVaT1ZGWDAwbHltazBEY2hzSSt3cFN4NUdxOG1HbU8vV1BjaHRFYnhRSGFFZDdOS01wR3VnQ2xocm1mRGJvTWUKZXVxUHRtV0VNNjVzbmpFa2hXeHpMcG1VUzE2QjRwUEFDU25xN1o3QUEveXRVVUc2T0hKcDVhb1RFMmdTbXl3RgpGZVpzVktObWFNVHlGRHBnd1BlNjl0blVtdzNSVkhNZjNmRDZkTTFMSzZzRDZ3WHFwMGRjQ3ZEdU5TQm8rZHhiCnQxWlFzdjhkQWdNQkFBRUNnZ0VBQU9mcDFiT2xLOVFKeXVlUHhjeDIvTkNVcUMxTEJDL01FdkEyUzVKWGFWbkcKbGhLR0pFb1U0Q0RoVk83dUlLYVZLTFZvMjk4RHFHUnBLM1d0RVE1TE40bytXYTcveTA5c1drMlVzbWxrVWFOZwpSaGtVZEJSK2dLNXVsQ3FKRml2dUJoTEQvRWlnQ1VWUGZKS2JtNG96TnpYcjVSMU5ENlV1aWFtODdtenlFcTBLCmZsVXlhR0RZNGdIdFNBOVBENVBFYlUveFpKeitKaHk5T2l3aVRXV0MrSHoyb2c3UWRDRDE2RlhGcit2VHpQN0MKb2tFb0VDZFNPRWlMalVENjBhS2ZxRmFCVm5MTkVudC9QSytmY1RBM05mNGtSMnFDNk9ZWjVFb09zYm1ka29ZTgpyU3NJZW9XblMxOEhvekZud2w3Z05wTUtjNmRzQzRBTldOVDFsTkhCb1FLQmdRRHlaUDFJSlppZUh6NlExaUVTCm5zd2tnblZCQUQ0SlVLR1ZDMHA3dk4yclNDZXh4c05ZZXFPTEEyZGZCUGpOVjd3blFKcUgxT05XellOMUJVSUUKeThLTCtFZVl6Q3RZa21LL21wSGJIMzNjd2tJODBuMHJROU1BalZMTlJ2YVVEOWp1NFBsRzFqaEFZUVVyTkViZQpKRlVpSk83aDVQa1llZG50SitqSHFpQnRoUUtCZ1FEZGtPbndmL0szYk4xenR0bXZQd0VicjhkVWJjRVh5NDFOCkl5VWwrZW1WSlgzYktKM0duNDZnQ2RsTTdkYmpwS3JVZ3oxL2JsZTgvMkVFckJvSEFRNkMrU2pEaGhvL01CbnIKekZheTBoK3YxbjBnZnNNVzRoOEF4cEFwc25OYnh6K2g1Wm5uSnRTd0srUjB3U0VJVVEzRjAxL2hMWWhLQ2l5OApwbW5HQi9hU3VRS0JnRzdxd1cvVExGd214ZlYyMXBsenFzeUdHZXVObGRXalhOMGIxcEI2b3lDdW11TmhwYUFHCk5uSDFNOGNxT2tPVWd4ZWZHMWRPbGx6eEc5ZGZlWTlDUWhyVW1NYVZucndmK0NuZkxDRU43d1VtcXpLenl1MFMKVXlwc2dOaElRYXNNK1dLTjllTnhRVHBNYXhZVERONjMxM0VSWDNKazJZdFdydDh6cFBSQXFDZ1ZBb0dCQU54egpUa0NMbmJ6aFphbTNlZm9DenlCMEVma3dSdHBkSGxkc3E0NlFqTmRuK1VSd3NpTXBLR2lWeEE3bDZsU1B4NlV3CmU2VHA3Z1JQZUlHRWwxVDJ1VENacGZSODNtcVdlb1FCeVJXZE9nZmplcFkxYWZpL3ZhY3c2Y21ERTRKeXloNVUKYTMveFE5ZVJwSHFDbWxKREMxZ1V5eVlwL3B2a2FjUytNeW5sVEhHSkFvR0FQekdTSzdXOHBUYldSVEFoaTVrSQpwZk5kWk1tcnRodUxNT3F6TGhyRjZublpldk9OdTBoYXVhZktlVElFd2w0clhYZHFKQlJBaWZKMFFsLzZKWFFkCmd1VzFrZWk1Ui8rUFZ5eUhab042c3NXSTNWYklwUUloUmt6UENnTDZhbHEwSzFpT1dlV1lIOHdORGRRdlB1T2UKRkZPOEovSzNxV0NtWjU0ODBBbTNhT0U9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K"
 SHARP_SERVER_CRT="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURhekNDQWxPZ0F3SUJBZ0lVRUR0Rjd0YVNTUnVQQTJ6Uk1aNWNzY2JCRm5jd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1JURUxNQWtHQTFVRUJoTUNTVTR4RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeElUQWZCZ05WQkFvTQpHRWx1ZEdWeWJtVjBJRmRwWkdkcGRITWdVSFI1SUV4MFpEQWVGdzB5TkRBNE1UTXhNekEzTVROYUZ3MHlOVEE0Ck1UTXhNekEzTVROYU1FVXhDekFKQmdOVkJBWVRBa2xPTVJNd0VRWURWUVFJREFwVGIyMWxMVk4wWVhSbE1TRXcKSHdZRFZRUUtEQmhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXdnZ0VpTUEwR0NTcUdTSWIzRFFFQgpBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRFRHcEEwNEZ1QlNFaE5PNVYvMGxTaDkvSEgxeVRZT2dRVFdoOG43eDlRCnZGMHpvZFZueVFIdjE5elU5eVdia2xvOEkvOXFBVm9lRzdXTnpUVFg2Q295ZlNjb1YvazN0Q2UwVnVWMlFJTVQKdW82SzJSU05CVHB1TlNqNTlzUiszVTQ2OFRBQnY0YVpsYjU4TU5CRXM3MVRieVpLRHBGRVRkMkg3T0ZKajg4QQpNRi9MaXJkeDZPOFdZL0tDeisxd1ZXL1JRdytYYjRJSWx4bXJFOC9UZ3FNSEo4dFUxYkZiOWJNcTEvOTN5YWtJClU1V2J2NVhXKzFwZFVyTUFNcTFFaC9vZThMN2pFaFdvZXZrNzgyU0kwUk0xeG5MaEtrUUVBYXd6Zkg2ODZiR2YKUHQ3RkFIQ1pGaWJ4KzZzSkg0R1M3S25iK0x5bk9ud3phMWZPUXZEZmcvRm5BZ01CQUFHalV6QlJNQjBHQTFVZApEZ1FXQkJUYlFUdmlUTW1xNXlNK2ZJRVI4VjdTZk1pK3B6QWZCZ05WSFNNRUdEQVdnQlRiUVR2aVRNbXE1eU0rCmZJRVI4VjdTZk1pK3B6QVBCZ05WSFJNQkFmOEVCVEFEQVFIL01BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRREYKTllyRnpBa2RIVkhjUkd5SUNsTi9IVGswaldOcTVSdTB1RUpDQ21Dbm9ZY1pRSTlDQlcwTkl3dGpZUkpTKzR1UwordWh4VWpSYTA5YXdOWDhvYmU0dDZjK25HRnhZMGZqamk0cGZnbU1kMWNJeGdsM3E3Nlp0ZkllRGR6alRLRXN1CjRFUTVadnEwMnJvTEZ0ZjEvL3dRVG0xNkNKdFpGWnhNZ1phYnNxc2JRc3M2dWdMUGtTTmdBWjI1L2VhcWhnQ20KTjFUV2FxL0xJMVBLSkxPK085NFlMa2FsNVpyOTJCOXk4Q0VKVUVuSTA1R1N1MmJUOFM2a0ZBMEpadEszTW9SbwpqRWZWV1lQVHR5TFR4amNvRndCcDlHaXZYSDdSdHBxMDlmSmFhU1pNekxmNGlyNHpBdXprbExBNWZvampPNXlKCllnYlVaQUU2aS81N1NFWjR3VmxTCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"
 SHARP_PROOF_LAYOUT="small"
+
+##### ON CHAIN CONFIG #####
+
+DA_LAYER="ethereum"
+SETTLEMENT_LAYER="ethereum"
+SETTLEMENT_RPC_URL="http://localhost:3001"
+MADARA_RPC_URL="http://localhost:3000"
+L1_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4"
+MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead"
+PRIVATE_KEY="0xdead"
+ETHEREUM_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
+STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Changed
 
+- refactor AWS config usage and clean .env files
 - GitHub's coverage CI yml file for localstack and db testing.
 - Orchestrator :Moved TestConfigBuilder to `config.rs` in tests folder.
 - `.env` file requires two more variables which are queue urls for processing

diff --git a/crates/da-clients/ethereum/src/config.rs b/crates/da-clients/ethereum/src/config.rs
@@ -19,14 +19,14 @@ pub struct EthereumDaConfig {
 impl DaConfig<EthereumDaClient> for EthereumDaConfig {
     fn new_from_env() -> Self {
         Self {
-            rpc_url: get_env_var_or_panic("ETHEREUM_RPC_URL"),
+            rpc_url: get_env_var_or_panic("SETTLEMENT_RPC_URL"),
             memory_pages_contract: get_env_var_or_panic("MEMORY_PAGES_CONTRACT_ADDRESS"),
             private_key: get_env_var_or_panic("PRIVATE_KEY"),
         }
     }
     async fn build_client(&self) -> EthereumDaClient {
         let client =
-            RpcClient::new_http(Url::from_str(self.rpc_url.as_str()).expect("Failed to parse ETHEREUM_RPC_URL"));
+            RpcClient::new_http(Url::from_str(self.rpc_url.as_str()).expect("Failed to parse SETTLEMENT_RPC_URL"));
         let provider = ProviderBuilder::<_, Ethereum>::new().on_client(client);
 
         EthereumDaClient { provider }

diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs
@@ -1,9 +1,10 @@
 use std::sync::Arc;
 
-use crate::data_storage::aws_s3::config::{AWSS3Config, AWSS3ConfigType};
+use crate::data_storage::aws_s3::config::AWSS3Config;
 use crate::data_storage::aws_s3::AWSS3;
 use crate::data_storage::{DataStorage, DataStorageConfig};
 use arc_swap::{ArcSwap, Guard};
+use aws_config::SdkConfig;
 use da_client_interface::{DaClient, DaConfig};
 use dotenvy::dotenv;
 use ethereum_da_client::config::EthereumDaConfig;
@@ -54,18 +55,24 @@ pub async fn init_config() -> Config {
     ));
 
     // init database
-    let database = Box::new(MongoDb::new(MongoDbConfig::new_from_env()).await);
+    let database = build_database_client().await;
+
+    // init AWS
+    let aws_config = aws_config::load_from_env().await;
 
     // init the queue
-    let queue = Box::new(SqsQueue {});
+    // TODO: we use omniqueue for now which doesn't support loading AWS config
+    // from `SdkConfig`. We can later move to using `aws_sdk_sqs`. This would require
+    // us stop using the generic omniqueue abstractions for message ack/nack
+    let queue = build_queue_client(&aws_config);
 
     let da_client = build_da_client().await;
 
     let settings_provider = DefaultSettingsProvider {};
     let settlement_client = build_settlement_client(&settings_provider).await;
     let prover_client = build_prover_service(&settings_provider);
 
-    let storage_client = build_storage_client().await;
+    let storage_client = build_storage_client(&aws_config).await;
 
     Config::new(Arc::new(provider), da_client, prover_client, settlement_client, database, queue, storage_client)
 }
@@ -177,9 +184,23 @@ pub async fn build_settlement_client(
     }
 }
 
-pub async fn build_storage_client() -> Box<dyn DataStorage + Send + Sync> {
+pub async fn build_storage_client(aws_config: &SdkConfig) -> Box<dyn DataStorage + Send + Sync> {
     match get_env_var_or_panic("DATA_STORAGE").as_str() {
-        "s3" => Box::new(AWSS3::new(AWSS3ConfigType::WithoutEndpoint(AWSS3Config::new_from_env())).await),
+        "s3" => Box::new(AWSS3::new(AWSS3Config::new_from_env(), aws_config)),
         _ => panic!("Unsupported Storage Client"),
     }
 }
+
+pub fn build_queue_client(_aws_config: &SdkConfig) -> Box<dyn QueueProvider + Send + Sync> {
+    match get_env_var_or_panic("QUEUE_PROVIDER").as_str() {
+        "sqs" => Box::new(SqsQueue {}),
+        _ => panic!("Unsupported Queue Client"),
+    }
+}
+
+pub async fn build_database_client() -> Box<dyn Database + Send + Sync> {
+    match get_env_var_or_panic("DATABASE").as_str() {
+        "mongodb" => Box::new(MongoDb::new(MongoDbConfig::new_from_env()).await),
+        _ => panic!("Unsupported Database Client"),
+    }
+}
diff --git a/crates/orchestrator/src/data_storage/aws_s3/config.rs b/crates/orchestrator/src/data_storage/aws_s3/config.rs
@@ -2,64 +2,17 @@ use utils::env_utils::get_env_var_or_panic;
 
 use crate::data_storage::DataStorageConfig;
 
-/// Represents the type of the config which one wants to pass to create the client
-#[derive(Clone)]
-pub enum AWSS3ConfigType {
-    WithEndpoint(S3LocalStackConfig),
-    WithoutEndpoint(AWSS3Config),
-}
-
 /// Represents AWS S3 config struct with all the necessary variables.
 #[derive(Clone)]
 pub struct AWSS3Config {
-    /// AWS ACCESS KEY ID
-    pub s3_key_id: String,
-    /// AWS ACCESS KEY SECRET
-    pub s3_key_secret: String,
     /// S3 Bucket Name
-    pub s3_bucket_name: String,
-    /// S3 Bucket region
-    pub s3_bucket_region: String,
-}
-
-/// Represents AWS S3 config struct with all the necessary variables.
-#[derive(Clone)]
-pub struct S3LocalStackConfig {
-    /// AWS ACCESS KEY ID
-    pub s3_key_id: String,
-    /// AWS ACCESS KEY SECRET
-    pub s3_key_secret: String,
-    /// S3 Bucket Name
-    pub s3_bucket_name: String,
-    /// S3 Bucket region
-    pub s3_bucket_region: String,
-    /// Endpoint url
-    pub endpoint_url: String,
+    pub bucket_name: String,
 }
 
 /// Implementation of `DataStorageConfig` for `AWSS3Config`
 impl DataStorageConfig for AWSS3Config {
     /// To return the config struct by creating it from the environment variables.
     fn new_from_env() -> Self {
-        Self {
-            s3_key_id: get_env_var_or_panic("AWS_ACCESS_KEY_ID"),
-            s3_key_secret: get_env_var_or_panic("AWS_SECRET_ACCESS_KEY"),
-            s3_bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME"),
-            s3_bucket_region: get_env_var_or_panic("AWS_S3_BUCKET_REGION"),
-        }
-    }
-}
-
-/// Implementation of `DataStorageConfig` for `S3LocalStackConfig`
-impl DataStorageConfig for S3LocalStackConfig {
-    /// To return the config struct by creating it from the environment variables.
-    fn new_from_env() -> Self {
-        Self {
-            s3_key_id: get_env_var_or_panic("AWS_ACCESS_KEY_ID"),
-            s3_key_secret: get_env_var_or_panic("AWS_SECRET_ACCESS_KEY"),
-            s3_bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME"),
-            s3_bucket_region: get_env_var_or_panic("AWS_S3_BUCKET_REGION"),
-            endpoint_url: get_env_var_or_panic("AWS_ENDPOINT_URL"),
-        }
+        Self { bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME") }
     }
 }