From 57c18847d9936c6854804776527a4957db132afd Mon Sep 17 00:00:00 2001 From: Leonid Tyurin Date: Wed, 18 Dec 2024 11:23:56 +0100 Subject: [PATCH] feat(multichain): enhance search api (#1151) * feat: add paginated address prefix search * fix: address prefix search * feat: add quick search for dapps * fix: filter out chains with invalid chain_id * fix: adjust order_by * fix: proto fields * feat: add logs for failed search * feat: add setting for default page size --- multichain-aggregator/Cargo.lock | 15 ++- multichain-aggregator/Cargo.toml | 3 +- multichain-aggregator/README.md | 13 ++- multichain-aggregator/justfile | 2 +- .../multichain-aggregator-logic/Cargo.toml | 4 + .../src/dapp_client.rs | 46 ++++++++ .../multichain-aggregator-logic/src/lib.rs | 5 +- .../src/repository/addresses.rs | 71 ++++++++---- .../multichain-aggregator-logic/src/search.rs | 57 ++++++++-- .../src/types/dapp.rs | 36 +++++++ .../src/types/mod.rs | 1 + .../src/types/search_results.rs | 4 +- .../proto/v1/api_config_http.yaml | 3 + .../proto/v1/multichain-aggregator.proto | 25 +++++ .../v1/multichain-aggregator.swagger.yaml | 62 +++++++++++ .../multichain-aggregator-server/Cargo.toml | 2 + .../config/example.toml | 3 + .../src/server.rs | 16 ++- .../src/services/multichain_aggregator.rs | 101 +++++++++++++++--- .../src/settings.rs | 52 +++++++++ 20 files changed, 460 insertions(+), 61 deletions(-) create mode 100644 multichain-aggregator/multichain-aggregator-logic/src/dapp_client.rs create mode 100644 multichain-aggregator/multichain-aggregator-logic/src/types/dapp.rs diff --git a/multichain-aggregator/Cargo.lock b/multichain-aggregator/Cargo.lock index 48bcad87f..f5229c923 100644 --- a/multichain-aggregator/Cargo.lock +++ b/multichain-aggregator/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "actix-codec" @@ -933,8 +933,8 @@ dependencies = [ [[package]] name = "blockscout-chains" -version = "0.1.0" -source = "git+https://github.com/blockscout/blockscout-rs#3892914d5d334699a92cd90f571988e86c7c9b2b" +version = "0.2.0" +source = "git+https://github.com/blockscout/blockscout-rs#76a11fcda774625f3f00948e431b18bfeea41fff" dependencies = [ "reqwest 0.12.9", "reqwest-middleware", @@ -1613,7 +1613,7 @@ dependencies = [ [[package]] name = "env-collector" version = "0.1.1" -source = "git+https://github.com/blockscout/blockscout-rs#3892914d5d334699a92cd90f571988e86c7c9b2b" +source = "git+https://github.com/blockscout/blockscout-rs#76a11fcda774625f3f00948e431b18bfeea41fff" dependencies = [ "anyhow", "config 0.14.1", @@ -2810,11 +2810,15 @@ dependencies = [ "multichain-aggregator-proto", "pretty_assertions", "regex", + "reqwest 0.12.9", "sea-orm", + "serde", + "serde_json", "thiserror", "tokio", "tonic", "tracing", + "url", ] [[package]] @@ -2846,6 +2850,7 @@ name = "multichain-aggregator-server" version = "0.1.0" dependencies = [ "actix-web", + "alloy-primitives", "anyhow", "async-trait", "blockscout-chains", @@ -2863,6 +2868,7 @@ dependencies = [ "tokio", "tonic", "tracing", + "url", ] [[package]] @@ -5775,6 +5781,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/multichain-aggregator/Cargo.toml b/multichain-aggregator/Cargo.toml index 51c459de1..ea4e858cf 100644 --- a/multichain-aggregator/Cargo.toml +++ b/multichain-aggregator/Cargo.toml @@ -55,10 +55,11 @@ alloy-primitives = "0.8" anyhow = "1.0" async-std = { version = "1", features = ["attributes", "tokio1"] } async-trait = "0.1" -blockscout-chains = { git = "https://github.com/blockscout/blockscout-rs", version = "0.1.0" } +blockscout-chains = { git = "https://github.com/blockscout/blockscout-rs", version = "0.2.0" } config = "0.13" env-collector = { git = "https://github.com/blockscout/blockscout-rs", version = "0.1.1" } pretty_assertions = "1.3" regex = "1.10" reqwest = "0.12" thiserror = "1.0" +url = { version = "2.4" } diff --git a/multichain-aggregator/README.md b/multichain-aggregator/README.md index e90dc4e85..559b327ad 100644 --- a/multichain-aggregator/README.md +++ b/multichain-aggregator/README.md @@ -29,10 +29,13 @@ Service-specific environment variables. Common environment variables are listed [anchor]: <> (anchors.envs.start) -| Variable | Req​uir​ed | Description | Default value | -| -------------------------------------------------- | ------------------------ | ---------------------------------- | -------------- | -| `MULTICHAIN_AGGREGATOR__DATABASE__CONNECT__URL` | true | Postgres connect URL to service DB | | -| `MULTICHAIN_AGGREGATOR__DATABASE__CREATE_DATABASE` | | Create database if doesn't exist | `false` | -| `MULTICHAIN_AGGREGATOR__DATABASE__RUN_MIGRATIONS` | | Run database migrations | `false` | +| Variable | Req​uir​ed | Description | Default value | +| -------------------------------------------------------- | ------------------------ | ----------------------------------- | ------------- | +| `MULTICHAIN_AGGREGATOR__DATABASE__CONNECT__URL` | true | Postgres connect URL to service DB | | +| `MULTICHAIN_AGGREGATOR__DATABASE__CREATE_DATABASE` | | Create database if doesn't exist | `false` | +| `MULTICHAIN_AGGREGATOR__DATABASE__RUN_MIGRATIONS` | | Run database migrations | `false` | +| `MULTICHAIN_AGGREGATOR__SERVICE__DAPP_CLIENT__URL` | true | e.g. `http://localhost:8080/api/v1` | | +| `MULTICHAIN_AGGREGATOR__SERVICE__API__DEFAULT_PAGE_SIZE` | | | `50` | +| `MULTICHAIN_AGGREGATOR__SERVICE__API__MAX_PAGE_SIZE` | | | `100` | [anchor]: <> (anchors.envs.end) diff --git a/multichain-aggregator/justfile b/multichain-aggregator/justfile index 85a2eb1d0..213d97bd9 100644 --- a/multichain-aggregator/justfile +++ b/multichain-aggregator/justfile @@ -27,7 +27,7 @@ stop-test-postgres: run: MULTICHAIN_AGGREGATOR__DATABASE__CONNECT__URL={{DATABASE_URL}} \ - cargo run --bin multichain-aggregator-server + dotenvy -f multichain-aggregator-server/config/base.env cargo run --bin multichain-aggregator-server generate-entities: sea-orm-cli generate entity --lib -o multichain-aggregator-entity/src diff --git a/multichain-aggregator/multichain-aggregator-logic/Cargo.toml b/multichain-aggregator/multichain-aggregator-logic/Cargo.toml index c6eeb7c2e..e13a09755 100644 --- a/multichain-aggregator/multichain-aggregator-logic/Cargo.toml +++ b/multichain-aggregator/multichain-aggregator-logic/Cargo.toml @@ -12,9 +12,13 @@ tracing = { workspace = true } sea-orm = { workspace = true } alloy-primitives = { workspace = true } regex = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } tonic = { workspace = true } tokio = { workspace = true } +reqwest = { workspace = true } +url = { workspace = true } [dev-dependencies] blockscout-service-launcher = { workspace = true } diff --git a/multichain-aggregator/multichain-aggregator-logic/src/dapp_client.rs b/multichain-aggregator/multichain-aggregator-logic/src/dapp_client.rs new file mode 100644 index 000000000..204948b40 --- /dev/null +++ b/multichain-aggregator/multichain-aggregator-logic/src/dapp_client.rs @@ -0,0 +1,46 @@ +use crate::error::ServiceError; +use serde::Deserialize; +use url::Url; + +pub struct DappClient { + http: reqwest::Client, + url: Url, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Dapp { + pub id: String, + pub title: String, + pub logo: String, + pub short_description: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DappWithChainId { + pub dapp: Dapp, + pub chain_id: String, +} + +impl DappClient { + pub fn new(url: Url) -> Self { + let http = reqwest::Client::new(); + Self { http, url } + } + + pub async fn search_dapps(&self, query: &str) -> Result, ServiceError> { + let mut url = self.url.clone(); + url.set_path("/api/v1/marketplace/dapps:search"); + url.query_pairs_mut().append_pair("query", query); + + self.http + .get(url) + .send() + .await + .map_err(|e| ServiceError::Internal(e.into()))? + .json::>() + .await + .map_err(|e| ServiceError::Internal(e.into())) + } +} diff --git a/multichain-aggregator/multichain-aggregator-logic/src/lib.rs b/multichain-aggregator/multichain-aggregator-logic/src/lib.rs index 5be7f1a6f..77e545e4d 100644 --- a/multichain-aggregator/multichain-aggregator-logic/src/lib.rs +++ b/multichain-aggregator/multichain-aggregator-logic/src/lib.rs @@ -1,4 +1,5 @@ pub mod api_key_manager; +pub mod dapp_client; pub mod error; mod import; mod proto; @@ -7,4 +8,6 @@ pub mod search; mod types; pub use import::batch_import; -pub use types::{api_keys::ApiKey, batch_import_request::BatchImportRequest, chains::Chain}; +pub use types::{ + api_keys::ApiKey, batch_import_request::BatchImportRequest, chains::Chain, ChainId, +}; diff --git a/multichain-aggregator/multichain-aggregator-logic/src/repository/addresses.rs b/multichain-aggregator/multichain-aggregator-logic/src/repository/addresses.rs index c25cc7206..431f28f25 100644 --- a/multichain-aggregator/multichain-aggregator-logic/src/repository/addresses.rs +++ b/multichain-aggregator/multichain-aggregator-logic/src/repository/addresses.rs @@ -1,12 +1,13 @@ use crate::{ error::{ParseError, ServiceError}, - types::addresses::Address, + types::{addresses::Address, ChainId}, }; +use alloy_primitives::Address as AddressAlloy; use entity::addresses::{ActiveModel, Column, Entity, Model}; use regex::Regex; use sea_orm::{ - prelude::Expr, sea_query::OnConflict, ActiveValue::NotSet, ColumnTrait, ConnectionTrait, DbErr, - EntityTrait, Iterable, QueryFilter, QuerySelect, + prelude::Expr, sea_query::OnConflict, ActiveValue::NotSet, ConnectionTrait, DbErr, EntityTrait, + IntoSimpleExpr, Iterable, QueryFilter, QueryOrder, QuerySelect, }; use std::sync::OnceLock; @@ -15,6 +16,11 @@ fn words_regex() -> &'static Regex { RE.get_or_init(|| Regex::new(r"[a-zA-Z0-9]+").unwrap()) } +fn hex_regex() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"^(0x)?[0-9a-fA-F]{3,40}$").unwrap()) +} + pub async fn upsert_many(db: &C, addresses: Vec
) -> Result<(), DbErr> where C: ConnectionTrait, @@ -44,32 +50,45 @@ where Ok(()) } -pub async fn find_by_address( - db: &C, - address: alloy_primitives::Address, -) -> Result, ServiceError> +pub async fn search_by_query(db: &C, q: &str) -> Result, ServiceError> where C: ConnectionTrait, { - let res = Entity::find() - .filter(Column::Hash.eq(address.as_slice())) - .all(db) - .await? - .into_iter() - .map(Address::try_from) - .collect::, _>>()?; - - Ok(res) + search_by_query_paginated(db, q, None, 100) + .await + .map(|(addresses, _)| addresses) } -pub async fn search_by_query(db: &C, q: &str) -> Result, ServiceError> +pub async fn search_by_query_paginated( + db: &C, + q: &str, + page_token: Option<(AddressAlloy, ChainId)>, + limit: u64, +) -> Result<(Vec
, Option<(AddressAlloy, ChainId)>), ServiceError> where C: ConnectionTrait, { - let mut query = Entity::find(); + let page_token = page_token.unwrap_or((AddressAlloy::ZERO, ChainId::MIN)); + let mut query = Entity::find() + .filter( + Expr::tuple([ + Column::Hash.into_simple_expr(), + Column::ChainId.into_simple_expr(), + ]) + .gte(Expr::tuple([ + page_token.0.as_slice().into(), + page_token.1.into(), + ])), + ) + .order_by_asc(Column::Hash) + .order_by_asc(Column::ChainId) + .limit(limit + 1); - if let Ok(address) = try_parse_address(q) { - query = query.filter(Column::Hash.eq(address.as_slice())); + if hex_regex().is_match(q) { + query = query.filter(Expr::cust_with_expr( + "encode(hash, 'hex') LIKE $1", + format!("{}%", q.to_lowercase().strip_prefix("0x").unwrap_or(q)), + )); } else { let ts_query = prepare_ts_query(q); query = query.filter(Expr::cust_with_expr( @@ -80,14 +99,20 @@ where )); } - let res = query - .limit(50) + let addresses = query .all(db) .await? .into_iter() .map(Address::try_from) .collect::, _>>()?; - Ok(res) + + match addresses.get(limit as usize) { + Some(a) => Ok(( + addresses[0..limit as usize].to_vec(), + Some((a.hash, a.chain_id)), + )), + None => Ok((addresses, None)), + } } fn non_primary_columns() -> impl Iterator { diff --git a/multichain-aggregator/multichain-aggregator-logic/src/search.rs b/multichain-aggregator/multichain-aggregator-logic/src/search.rs index 2d07f6d9b..8753102c6 100644 --- a/multichain-aggregator/multichain-aggregator-logic/src/search.rs +++ b/multichain-aggregator/multichain-aggregator-logic/src/search.rs @@ -1,15 +1,17 @@ use crate::{ + dapp_client::DappClient, error::ServiceError, repository::{addresses, block_ranges, hashes}, types::{ chains::Chain, + dapp::MarketplaceDapp, search_results::{ChainSearchResult, SearchResults}, ChainId, }, }; use sea_orm::DatabaseConnection; use std::collections::BTreeMap; -use tokio::try_join; +use tokio::join; macro_rules! populate_search_results { ($target:expr, $explorers:expr, $from:expr, $field:ident) => { @@ -28,16 +30,18 @@ macro_rules! populate_search_results { pub async fn quick_search( db: &DatabaseConnection, + dapp_client: &DappClient, query: String, chains: &[Chain], ) -> Result { let raw_query = query.trim(); - let ((blocks, transactions), block_numbers, addresses) = try_join!( + let (hashes, block_numbers, addresses, dapps) = join!( hashes::search_by_query(db, raw_query), block_ranges::search_by_query(db, raw_query), - addresses::search_by_query(db, raw_query) - )?; + addresses::search_by_query(db, raw_query), + dapp_client.search_dapps(raw_query), + ); let explorers: BTreeMap = chains .iter() @@ -45,10 +49,47 @@ pub async fn quick_search( .collect(); let mut results = SearchResults::default(); - populate_search_results!(results, explorers, addresses, addresses); - populate_search_results!(results, explorers, blocks, blocks); - populate_search_results!(results, explorers, transactions, transactions); - populate_search_results!(results, explorers, block_numbers, block_numbers); + + match hashes { + Ok((blocks, transactions)) => { + populate_search_results!(results, explorers, blocks, blocks); + populate_search_results!(results, explorers, transactions, transactions); + } + Err(err) => { + tracing::error!(error = ?err, "failed to search hashes"); + } + } + + match block_numbers { + Ok(block_numbers) => { + populate_search_results!(results, explorers, block_numbers, block_numbers); + } + Err(err) => { + tracing::error!(error = ?err, "failed to search block numbers"); + } + } + + match addresses { + Ok(addresses) => { + populate_search_results!(results, explorers, addresses, addresses); + } + Err(err) => { + tracing::error!(error = ?err, "failed to search addresses"); + } + } + + match dapps { + Ok(dapps) => { + let dapps: Vec = dapps + .into_iter() + .filter_map(|d| d.try_into().ok()) + .collect(); + populate_search_results!(results, explorers, dapps, dapps); + } + Err(err) => { + tracing::error!(error = ?err, "failed to search dapps"); + } + } Ok(results) } diff --git a/multichain-aggregator/multichain-aggregator-logic/src/types/dapp.rs b/multichain-aggregator/multichain-aggregator-logic/src/types/dapp.rs new file mode 100644 index 000000000..402168809 --- /dev/null +++ b/multichain-aggregator/multichain-aggregator-logic/src/types/dapp.rs @@ -0,0 +1,36 @@ +use super::ChainId; +use crate::{dapp_client::DappWithChainId, error::ParseError, proto}; + +#[derive(Debug)] +pub struct MarketplaceDapp { + pub id: String, + pub title: String, + pub logo: String, + pub short_description: String, + pub chain_id: ChainId, +} + +impl TryFrom for MarketplaceDapp { + type Error = ParseError; + + fn try_from(v: DappWithChainId) -> Result { + Ok(Self { + id: v.dapp.id, + title: v.dapp.title, + logo: v.dapp.logo, + short_description: v.dapp.short_description, + chain_id: v.chain_id.parse()?, + }) + } +} + +impl From for proto::MarketplaceDapp { + fn from(v: MarketplaceDapp) -> Self { + Self { + id: v.id, + title: v.title, + logo: v.logo, + short_description: v.short_description, + } + } +} diff --git a/multichain-aggregator/multichain-aggregator-logic/src/types/mod.rs b/multichain-aggregator/multichain-aggregator-logic/src/types/mod.rs index a1a3fcf72..af1989286 100644 --- a/multichain-aggregator/multichain-aggregator-logic/src/types/mod.rs +++ b/multichain-aggregator/multichain-aggregator-logic/src/types/mod.rs @@ -3,6 +3,7 @@ pub mod api_keys; pub mod batch_import_request; pub mod block_ranges; pub mod chains; +pub mod dapp; pub mod hashes; pub mod search_results; pub type ChainId = i64; diff --git a/multichain-aggregator/multichain-aggregator-logic/src/types/search_results.rs b/multichain-aggregator/multichain-aggregator-logic/src/types/search_results.rs index 57752bdd2..28e603002 100644 --- a/multichain-aggregator/multichain-aggregator-logic/src/types/search_results.rs +++ b/multichain-aggregator/multichain-aggregator-logic/src/types/search_results.rs @@ -1,7 +1,7 @@ use super::{block_ranges::ChainBlockNumber, ChainId}; use crate::{ proto, - types::{addresses::Address, hashes::Hash}, + types::{addresses::Address, dapp::MarketplaceDapp, hashes::Hash}, }; use std::collections::BTreeMap; @@ -12,6 +12,7 @@ pub struct ChainSearchResult { pub blocks: Vec, pub transactions: Vec, pub block_numbers: Vec, + pub dapps: Vec, } impl From for proto::quick_search_response::ChainSearchResult { @@ -22,6 +23,7 @@ impl From for proto::quick_search_response::ChainSearchResult blocks: v.blocks.into_iter().map(|b| b.into()).collect(), transactions: v.transactions.into_iter().map(|t| t.into()).collect(), block_numbers: v.block_numbers.into_iter().map(|b| b.into()).collect(), + dapps: v.dapps.into_iter().map(|d| d.into()).collect(), } } } diff --git a/multichain-aggregator/multichain-aggregator-proto/proto/v1/api_config_http.yaml b/multichain-aggregator/multichain-aggregator-proto/proto/v1/api_config_http.yaml index 5c0e97610..6a8eafd19 100644 --- a/multichain-aggregator/multichain-aggregator-proto/proto/v1/api_config_http.yaml +++ b/multichain-aggregator/multichain-aggregator-proto/proto/v1/api_config_http.yaml @@ -11,6 +11,9 @@ http: - selector: blockscout.multichainAggregator.v1.MultichainAggregatorService.QuickSearch get: /api/v1/search:quick + - selector: blockscout.multichainAggregator.v1.MultichainAggregatorService.ListAddresses + get: /api/v1/addresses + #################### Health #################### - selector: blockscout.multichainAggregator.v1.Health.Check diff --git a/multichain-aggregator/multichain-aggregator-proto/proto/v1/multichain-aggregator.proto b/multichain-aggregator/multichain-aggregator-proto/proto/v1/multichain-aggregator.proto index 3398325bb..210d32156 100644 --- a/multichain-aggregator/multichain-aggregator-proto/proto/v1/multichain-aggregator.proto +++ b/multichain-aggregator/multichain-aggregator-proto/proto/v1/multichain-aggregator.proto @@ -7,6 +7,12 @@ option go_package = "github.com/blockscout/blockscout-rs/multichain-aggregator"; service MultichainAggregatorService { rpc BatchImport(BatchImportRequest) returns (BatchImportResponse) {} rpc QuickSearch(QuickSearchRequest) returns (QuickSearchResponse) {} + rpc ListAddresses(ListAddressesRequest) returns (ListAddressesResponse) {} +} + +message Pagination { + string page_token = 1; + uint32 page_size = 2; } message Address { @@ -45,6 +51,13 @@ message Hash { HashType hash_type = 2; } +message MarketplaceDapp { + string id = 1; + string title = 2; + string logo = 3; + string short_description = 4; +} + message BatchImportRequest { string chain_id = 1; repeated Address addresses = 2; @@ -69,7 +82,19 @@ message QuickSearchResponse { repeated Hash blocks = 3; repeated Hash transactions = 4; repeated ChainBlockNumber block_numbers = 5; + repeated MarketplaceDapp dapps = 6; } map items = 1; } + +message ListAddressesRequest { + string q = 1; + optional uint32 page_size = 2; + optional string page_token = 3; +} + +message ListAddressesResponse { + repeated Address addresses = 1; + Pagination pagination = 2; +} diff --git a/multichain-aggregator/multichain-aggregator-proto/swagger/v1/multichain-aggregator.swagger.yaml b/multichain-aggregator/multichain-aggregator-proto/swagger/v1/multichain-aggregator.swagger.yaml index eeb2a5a14..5eb1eb3d7 100644 --- a/multichain-aggregator/multichain-aggregator-proto/swagger/v1/multichain-aggregator.swagger.yaml +++ b/multichain-aggregator/multichain-aggregator-proto/swagger/v1/multichain-aggregator.swagger.yaml @@ -10,6 +10,34 @@ consumes: produces: - application/json paths: + /api/v1/addresses: + get: + operationId: MultichainAggregatorService_ListAddresses + responses: + "200": + description: A successful response. + schema: + $ref: '#/definitions/v1ListAddressesResponse' + default: + description: An unexpected error response. + schema: + $ref: '#/definitions/rpcStatus' + parameters: + - name: q + in: query + required: false + type: string + - name: page_size + in: query + required: false + type: integer + format: int64 + - name: page_token + in: query + required: false + type: string + tags: + - MultichainAggregatorService /api/v1/import:batch: post: operationId: MultichainAggregatorService_BatchImport @@ -130,6 +158,11 @@ definitions: items: type: object $ref: '#/definitions/QuickSearchResponseChainBlockNumber' + dapps: + type: array + items: + type: object + $ref: '#/definitions/v1MarketplaceDapp' protobufAny: type: object properties: @@ -216,6 +249,35 @@ definitions: properties: status: $ref: '#/definitions/HealthCheckResponseServingStatus' + v1ListAddressesResponse: + type: object + properties: + addresses: + type: array + items: + type: object + $ref: '#/definitions/v1Address' + pagination: + $ref: '#/definitions/v1Pagination' + v1MarketplaceDapp: + type: object + properties: + id: + type: string + title: + type: string + logo: + type: string + short_description: + type: string + v1Pagination: + type: object + properties: + page_token: + type: string + page_size: + type: integer + format: int64 v1QuickSearchResponse: type: object properties: diff --git a/multichain-aggregator/multichain-aggregator-server/Cargo.toml b/multichain-aggregator/multichain-aggregator-server/Cargo.toml index 9d94ea662..b1c2d8300 100644 --- a/multichain-aggregator/multichain-aggregator-server/Cargo.toml +++ b/multichain-aggregator/multichain-aggregator-server/Cargo.toml @@ -9,6 +9,7 @@ multichain-aggregator-proto = { workspace = true } multichain-aggregator-logic = { workspace = true } multichain-aggregator-migration = { workspace = true } actix-web = { workspace = true } +alloy-primitives = { workspace = true } anyhow = { workspace = true } async-trait = { workspace = true } blockscout-service-launcher = { workspace = true } @@ -21,6 +22,7 @@ tokio = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } env-collector = { workspace = true } +url = { workspace = true, features = ["serde"] } [dev-dependencies] blockscout-service-launcher = { workspace = true, features = [ "test-server","test-database"] } diff --git a/multichain-aggregator/multichain-aggregator-server/config/example.toml b/multichain-aggregator/multichain-aggregator-server/config/example.toml index f2c397190..5d64bb897 100644 --- a/multichain-aggregator/multichain-aggregator-server/config/example.toml +++ b/multichain-aggregator/multichain-aggregator-server/config/example.toml @@ -1,2 +1,5 @@ [database.connect] url = "postgres://postgres:postgres@localhost:5432/postgres" + +[service.dapp_client] +url = "http://localhost:8080/api/v1" diff --git a/multichain-aggregator/multichain-aggregator-server/src/server.rs b/multichain-aggregator/multichain-aggregator-server/src/server.rs index c253b314f..cdff29d98 100644 --- a/multichain-aggregator/multichain-aggregator-server/src/server.rs +++ b/multichain-aggregator/multichain-aggregator-server/src/server.rs @@ -10,7 +10,7 @@ use crate::{ use blockscout_chains::BlockscoutChainsClient; use blockscout_service_launcher::{database, launcher, launcher::LaunchSettings}; use migration::Migrator; -use multichain_aggregator_logic::repository; +use multichain_aggregator_logic::{dapp_client::DappClient, repository}; use std::sync::Arc; const SERVICE_NAME: &str = "multichain_aggregator"; @@ -65,11 +65,21 @@ pub async fn run(settings: Settings) -> Result<(), anyhow::Error> { .fetch_all() .await? .into_iter() - .map(|(id, chain)| (id as i64, chain).into()) + .filter_map(|(id, chain)| { + let id = id.parse::().ok()?; + Some((id, chain).into()) + }) .collect::>(); repository::chains::upsert_many(&db, blockscout_chains.clone()).await?; - let multichain_aggregator = Arc::new(MultichainAggregator::new(db, blockscout_chains)); + let dapp_client = DappClient::new(settings.service.dapp_client.url); + + let multichain_aggregator = Arc::new(MultichainAggregator::new( + db, + blockscout_chains, + dapp_client, + settings.service.api, + )); let router = Router { health, diff --git a/multichain-aggregator/multichain-aggregator-server/src/services/multichain_aggregator.rs b/multichain-aggregator/multichain-aggregator-server/src/services/multichain_aggregator.rs index 2d219c9f3..a6bfe5c61 100644 --- a/multichain-aggregator/multichain-aggregator-server/src/services/multichain_aggregator.rs +++ b/multichain-aggregator/multichain-aggregator-server/src/services/multichain_aggregator.rs @@ -1,14 +1,17 @@ -use crate::proto::{ - multichain_aggregator_service_server::MultichainAggregatorService, BatchImportRequest, - BatchImportResponse, +use crate::{ + proto::{ + multichain_aggregator_service_server::MultichainAggregatorService, BatchImportRequest, + BatchImportResponse, ListAddressesRequest, ListAddressesResponse, Pagination, + QuickSearchRequest, QuickSearchResponse, + }, + settings::ApiSettings, }; use multichain_aggregator_logic::{ - self as logic, api_key_manager::ApiKeyManager, error::ServiceError, Chain, -}; -use multichain_aggregator_proto::blockscout::multichain_aggregator::v1::{ - QuickSearchRequest, QuickSearchResponse, + self as logic, api_key_manager::ApiKeyManager, dapp_client::DappClient, error::ServiceError, + Chain, }; use sea_orm::DatabaseConnection; +use std::str::FromStr; use tonic::{Request, Response, Status}; pub struct MultichainAggregator { @@ -16,16 +19,30 @@ pub struct MultichainAggregator { api_key_manager: ApiKeyManager, // Cached chains chains: Vec, + dapp_client: DappClient, + api_settings: ApiSettings, } impl MultichainAggregator { - pub fn new(db: DatabaseConnection, chains: Vec) -> Self { + pub fn new( + db: DatabaseConnection, + chains: Vec, + dapp_client: DappClient, + api_settings: ApiSettings, + ) -> Self { Self { db: db.clone(), api_key_manager: ApiKeyManager::new(db), chains, + dapp_client, + api_settings, } } + + fn normalize_page_size(&self, size: Option) -> u32 { + size.unwrap_or(self.api_settings.default_page_size) + .clamp(1, self.api_settings.max_page_size) + } } #[async_trait::async_trait] @@ -58,19 +75,75 @@ impl MultichainAggregatorService for MultichainAggregator { })) } + async fn list_addresses( + &self, + request: Request, + ) -> Result, Status> { + let inner = request.into_inner(); + + let page_token: Option<(alloy_primitives::Address, logic::ChainId)> = + inner.page_token.map(parse_query_2).transpose()?; + let page_size = self.normalize_page_size(inner.page_size); + + let (addresses, next_page_token) = logic::repository::addresses::search_by_query_paginated( + &self.db, + &inner.q, + page_token, + page_size as u64, + ) + .await + .map_err(|err| { + tracing::error!(error = ?err, "failed to list addresses"); + Status::internal("failed to list addresses") + })?; + + Ok(Response::new(ListAddressesResponse { + addresses: addresses.into_iter().map(|a| a.into()).collect(), + pagination: next_page_token.map(|(a, c)| Pagination { + page_token: format!("{},{}", a.to_checksum(None), c), + page_size, + }), + })) + } + async fn quick_search( &self, request: Request, ) -> Result, Status> { let inner = request.into_inner(); - let results = logic::search::quick_search(&self.db, inner.q, &self.chains) - .await - .map_err(|err| { - tracing::error!(error = ?err, "failed to quick search"); - Status::internal("failed to quick search") - })?; + let results = + logic::search::quick_search(&self.db, &self.dapp_client, inner.q, &self.chains) + .await + .map_err(|err| { + tracing::error!(error = ?err, "failed to quick search"); + Status::internal("failed to quick search") + })?; Ok(Response::new(results.into())) } } + +#[inline] +fn parse_query(input: String) -> Result +where + ::Err: std::fmt::Display, +{ + T::from_str(&input) + .map_err(|e| Status::invalid_argument(format!("invalid value {}: {e}", input))) +} + +#[inline] +fn parse_query_2(input: String) -> Result<(T1, T2), Status> +where + ::Err: std::fmt::Display, + ::Err: std::fmt::Display, +{ + match input.split(',').collect::>().as_slice() { + [v1, v2] => Ok(( + parse_query::(v1.to_string())?, + parse_query::(v2.to_string())?, + )), + _ => Err(Status::invalid_argument("invalid page_token format")), + } +} diff --git a/multichain-aggregator/multichain-aggregator-server/src/settings.rs b/multichain-aggregator/multichain-aggregator-server/src/settings.rs index 780fafb73..ab7ee8cce 100644 --- a/multichain-aggregator/multichain-aggregator-server/src/settings.rs +++ b/multichain-aggregator/multichain-aggregator-server/src/settings.rs @@ -4,6 +4,7 @@ use blockscout_service_launcher::{ tracing::{JaegerSettings, TracingSettings}, }; use serde::{Deserialize, Serialize}; +use url::Url; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -17,6 +18,40 @@ pub struct Settings { #[serde(default)] pub jaeger: JaegerSettings, pub database: DatabaseSettings, + + pub service: ServiceSettings, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ServiceSettings { + pub dapp_client: DappClientSettings, + #[serde(default)] + pub api: ApiSettings, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ApiSettings { + #[serde(default = "default_default_page_size")] + pub default_page_size: u32, + #[serde(default = "default_max_page_size")] + pub max_page_size: u32, +} + +impl Default for ApiSettings { + fn default() -> Self { + Self { + default_page_size: default_default_page_size(), + max_page_size: default_max_page_size(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct DappClientSettings { + pub url: Url, } impl ConfigSettings for Settings { @@ -35,6 +70,23 @@ impl Settings { create_database: Default::default(), run_migrations: Default::default(), }, + service: ServiceSettings { + dapp_client: DappClientSettings { + url: Url::parse("http://localhost:8050").unwrap(), + }, + api: ApiSettings { + default_page_size: default_default_page_size(), + max_page_size: default_max_page_size(), + }, + }, } } } + +fn default_max_page_size() -> u32 { + 100 +} + +fn default_default_page_size() -> u32 { + 50 +}