From 4503d2399cd48cb8b9e89f27eb6c0640deed92f6 Mon Sep 17 00:00:00 2001 From: Rim Rakhimov Date: Wed, 1 Nov 2023 13:20:06 +0300 Subject: [PATCH 1/4] Add contracts-lookup extractor --- eth-bytecode-db-extractors/Cargo.lock | 102 ++++++++++++------ eth-bytecode-db-extractors/Cargo.toml | 3 + .../contracts-lookup-entity/Cargo.toml | 12 +++ .../src/contract_addresses.rs | 39 +++++++ .../contracts-lookup-entity/src/lib.rs | 7 ++ .../contracts-lookup-entity/src/prelude.rs | 3 + .../contracts-lookup-migration/Cargo.toml | 23 ++++ .../contracts-lookup-migration/README.md | 41 +++++++ .../contracts-lookup-migration/src/lib.rs | 32 ++++++ ..._000001_create_contract_addresses_table.rs | 51 +++++++++ .../contracts-lookup-migration/src/main.rs | 6 ++ .../contracts-lookup/Cargo.toml | 28 +++++ .../contracts-lookup/src/blockscout.rs | 85 +++++++++++++++ .../contracts-lookup/src/client.rs | 94 ++++++++++++++++ .../contracts-lookup/src/lib.rs | 6 ++ .../contracts-lookup/src/main.rs | 42 ++++++++ .../contracts-lookup/src/settings.rs | 41 +++++++ 17 files changed, 585 insertions(+), 30 deletions(-) create mode 100644 eth-bytecode-db-extractors/contracts-lookup-entity/Cargo.toml create mode 100644 eth-bytecode-db-extractors/contracts-lookup-entity/src/contract_addresses.rs create mode 100644 eth-bytecode-db-extractors/contracts-lookup-entity/src/lib.rs create mode 100644 eth-bytecode-db-extractors/contracts-lookup-entity/src/prelude.rs create mode 100644 eth-bytecode-db-extractors/contracts-lookup-migration/Cargo.toml create mode 100644 eth-bytecode-db-extractors/contracts-lookup-migration/README.md create mode 100644 eth-bytecode-db-extractors/contracts-lookup-migration/src/lib.rs create mode 100644 eth-bytecode-db-extractors/contracts-lookup-migration/src/m20220101_000001_create_contract_addresses_table.rs create mode 100644 eth-bytecode-db-extractors/contracts-lookup-migration/src/main.rs create mode 100644 eth-bytecode-db-extractors/contracts-lookup/Cargo.toml create mode 100644 eth-bytecode-db-extractors/contracts-lookup/src/blockscout.rs create mode 100644 eth-bytecode-db-extractors/contracts-lookup/src/client.rs create mode 100644 eth-bytecode-db-extractors/contracts-lookup/src/lib.rs create mode 100644 eth-bytecode-db-extractors/contracts-lookup/src/main.rs create mode 100644 eth-bytecode-db-extractors/contracts-lookup/src/settings.rs diff --git a/eth-bytecode-db-extractors/Cargo.lock b/eth-bytecode-db-extractors/Cargo.lock index 83c43a8ac..4243e6988 100644 --- a/eth-bytecode-db-extractors/Cargo.lock +++ b/eth-bytecode-db-extractors/Cargo.lock @@ -1004,6 +1004,48 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +[[package]] +name = "contracts-lookup" +version = "0.1.0" +dependencies = [ + "anyhow", + "blockscout-display-bytes", + "blockscout-service-launcher", + "contracts-lookup-entity", + "contracts-lookup-migration", + "futures", + "governor", + "job-queue", + "reqwest", + "reqwest-middleware", + "reqwest-rate-limiter", + "reqwest-retry", + "sea-orm", + "serde", + "serde_json", + "serde_path_to_error", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "contracts-lookup-entity" +version = "0.1.0" +dependencies = [ + "job-queue", + "sea-orm", +] + +[[package]] +name = "contracts-lookup-migration" +version = "0.1.0" +dependencies = [ + "async-std", + "job-queue", + "sea-orm-migration", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -1468,9 +1510,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" dependencies = [ "futures-channel", "futures-core", @@ -1483,9 +1525,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", "futures-sink", @@ -1493,15 +1535,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" dependencies = [ "futures-core", "futures-task", @@ -1521,9 +1563,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-lite" @@ -1542,9 +1584,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", @@ -1553,15 +1595,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-timer" @@ -1571,9 +1613,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -3675,9 +3717,9 @@ dependencies = [ [[package]] name = "sea-orm" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5b2d70c255bc5cbe1d49f69c3c8eadae0fbbaeb18ee978edbf2f75775cb94d" +checksum = "14d17105eb8049488d2528580ecc3f0912ab177d600f10e8e292d6994870ba6a" dependencies = [ "async-stream", "async-trait", @@ -3720,9 +3762,9 @@ dependencies = [ [[package]] name = "sea-orm-macros" -version = "0.12.3" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c8d455fad40194fb9774fdc4810c0f2700ff0dc0e93bd5ce9d641cc3f5dd75" +checksum = "a836864040c92d0615497eeccf97e1aee312857bf2ab36d74a74ce1c5c2cefc3" dependencies = [ "heck", "proc-macro2", @@ -3893,18 +3935,18 @@ checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" dependencies = [ "proc-macro2", "quote", @@ -3913,9 +3955,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -4839,9 +4881,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.39" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "log", "pin-project-lite", diff --git a/eth-bytecode-db-extractors/Cargo.toml b/eth-bytecode-db-extractors/Cargo.toml index f9ae4b5ca..4f9bc4352 100644 --- a/eth-bytecode-db-extractors/Cargo.toml +++ b/eth-bytecode-db-extractors/Cargo.toml @@ -4,6 +4,9 @@ members = [ "blockscout", "blockscout-entity", "blockscout-migration", + "contracts-lookup", + "contracts-lookup-entity", + "contracts-lookup-migration", "smart-contract-fiesta", "smart-contract-fiesta-entity", "smart-contract-fiesta-migration", diff --git a/eth-bytecode-db-extractors/contracts-lookup-entity/Cargo.toml b/eth-bytecode-db-extractors/contracts-lookup-entity/Cargo.toml new file mode 100644 index 000000000..e85731bae --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup-entity/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "contracts-lookup-entity" +version = "0.1.0" +edition = "2021" + +[lib] +name = "entity" +path = "src/lib.rs" + +[dependencies] +sea-orm = { version = "^0" } +job-queue = { path = "../job-queue", features = ["entity"] } \ No newline at end of file diff --git a/eth-bytecode-db-extractors/contracts-lookup-entity/src/contract_addresses.rs b/eth-bytecode-db-extractors/contracts-lookup-entity/src/contract_addresses.rs new file mode 100644 index 000000000..1fb01fdd2 --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup-entity/src/contract_addresses.rs @@ -0,0 +1,39 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "contract_addresses")] +pub struct Model { + #[sea_orm( + primary_key, + auto_increment = false, + column_type = "Binary(BlobSize::Blob(None))" + )] + pub contract_address: Vec, + pub fetched_coin_balance_block_number: i64, + pub inserted_at: DateTime, + pub updated_at: DateTime, + #[sea_orm(column_name = "_job_id")] + pub job_id: Uuid, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::job_queue::Entity", + from = "Column::JobId", + to = "super::job_queue::Column::Id", + on_update = "NoAction", + on_delete = "NoAction" + )] + JobQueue, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::JobQueue.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/eth-bytecode-db-extractors/contracts-lookup-entity/src/lib.rs b/eth-bytecode-db-extractors/contracts-lookup-entity/src/lib.rs new file mode 100644 index 000000000..5a9f03355 --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup-entity/src/lib.rs @@ -0,0 +1,7 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +pub mod prelude; + +pub mod contract_addresses; + +pub use job_queue::entity as job_queue; diff --git a/eth-bytecode-db-extractors/contracts-lookup-entity/src/prelude.rs b/eth-bytecode-db-extractors/contracts-lookup-entity/src/prelude.rs new file mode 100644 index 000000000..f9942172a --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup-entity/src/prelude.rs @@ -0,0 +1,3 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +pub use super::{contract_addresses::Entity as ContractAddresses, job_queue::Entity as JobQueue}; diff --git a/eth-bytecode-db-extractors/contracts-lookup-migration/Cargo.toml b/eth-bytecode-db-extractors/contracts-lookup-migration/Cargo.toml new file mode 100644 index 000000000..d7b975305 --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup-migration/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "contracts-lookup-migration" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +name = "migration" +path = "src/lib.rs" + +[dependencies] +async-std = { version = "1", features = ["attributes", "tokio1"] } +job-queue = { path = "../job-queue", features = ["migration"] } + +[dependencies.sea-orm-migration] +version = "^0" +features = [ + # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. + # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. + # e.g. + "runtime-tokio-rustls", # `ASYNC_RUNTIME` feature + "sqlx-postgres", # `DATABASE_DRIVER` feature +] diff --git a/eth-bytecode-db-extractors/contracts-lookup-migration/README.md b/eth-bytecode-db-extractors/contracts-lookup-migration/README.md new file mode 100644 index 000000000..3b438d89e --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup-migration/README.md @@ -0,0 +1,41 @@ +# Running Migrator CLI + +- Generate a new migration file + ```sh + cargo run -- generate MIGRATION_NAME + ``` +- Apply all pending migrations + ```sh + cargo run + ``` + ```sh + cargo run -- up + ``` +- Apply first 10 pending migrations + ```sh + cargo run -- up -n 10 + ``` +- Rollback last applied migrations + ```sh + cargo run -- down + ``` +- Rollback last 10 applied migrations + ```sh + cargo run -- down -n 10 + ``` +- Drop all tables from the database, then reapply all migrations + ```sh + cargo run -- fresh + ``` +- Rollback all applied migrations, then reapply all migrations + ```sh + cargo run -- refresh + ``` +- Rollback all applied migrations + ```sh + cargo run -- reset + ``` +- Check the status of all migrations + ```sh + cargo run -- status + ``` diff --git a/eth-bytecode-db-extractors/contracts-lookup-migration/src/lib.rs b/eth-bytecode-db-extractors/contracts-lookup-migration/src/lib.rs new file mode 100644 index 000000000..7f8d9f90b --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup-migration/src/lib.rs @@ -0,0 +1,32 @@ +pub use sea_orm_migration::prelude::*; +use sea_orm_migration::sea_orm::{Statement, TransactionTrait}; + +mod m20220101_000001_create_contract_addresses_table; + +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![ + Box::new(job_queue::migration::Migration), + Box::new(m20220101_000001_create_contract_addresses_table::Migration), + ] + } +} + +pub async fn from_statements( + manager: &SchemaManager<'_>, + statements: &[&str], +) -> Result<(), DbErr> { + let txn = manager.get_connection().begin().await?; + for statement in statements { + txn.execute(Statement::from_string( + manager.get_database_backend(), + statement.to_string(), + )) + .await + .map_err(|err| DbErr::Migration(format!("{err}\nQuery: {statement}")))?; + } + txn.commit().await +} diff --git a/eth-bytecode-db-extractors/contracts-lookup-migration/src/m20220101_000001_create_contract_addresses_table.rs b/eth-bytecode-db-extractors/contracts-lookup-migration/src/m20220101_000001_create_contract_addresses_table.rs new file mode 100644 index 000000000..81b289868 --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup-migration/src/m20220101_000001_create_contract_addresses_table.rs @@ -0,0 +1,51 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let create_contract_addresses_table = r#" + CREATE TABLE "contract_addresses" ( + "contract_address" bytea NOT NULL, + + "fetched_coin_balance_block_number" bigint NOT NULL, + "inserted_at" timestamp NOT NULL, + "updated_at" timestamp NOT NULL, + + "_job_id" uuid NOT NULL REFERENCES "_job_queue" ("id"), + + PRIMARY KEY ("contract_address") + ); + "#; + + let create_job_queue_connection_statements = + job_queue::migration::create_job_queue_connection_statements("contract_addresses"); + + let mut statements = vec![ + create_contract_addresses_table, + ]; + statements.extend( + create_job_queue_connection_statements + .iter() + .map(|v| v.as_str()), + ); + + crate::from_statements(manager, &statements).await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let drop_job_queue_connection_statements = + job_queue::migration::drop_job_queue_connection_statements("contract_addresses"); + let drop_table_contract_addresses = "DROP TABLE contract_addresses;"; + + let mut statements = drop_job_queue_connection_statements + .iter() + .map(|v| v.as_str()) + .collect::>(); + statements.extend([drop_table_contract_addresses]); + + crate::from_statements(manager, &statements).await + } +} diff --git a/eth-bytecode-db-extractors/contracts-lookup-migration/src/main.rs b/eth-bytecode-db-extractors/contracts-lookup-migration/src/main.rs new file mode 100644 index 000000000..c6b6e48db --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup-migration/src/main.rs @@ -0,0 +1,6 @@ +use sea_orm_migration::prelude::*; + +#[async_std::main] +async fn main() { + cli::run_cli(migration::Migrator).await; +} diff --git a/eth-bytecode-db-extractors/contracts-lookup/Cargo.toml b/eth-bytecode-db-extractors/contracts-lookup/Cargo.toml new file mode 100644 index 000000000..8845935e8 --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "contracts-lookup" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +contracts-lookup-entity = { path = "../contracts-lookup-entity" } +contracts-lookup-migration = { path = "../contracts-lookup-migration" } +job-queue = { path = "../job-queue", features = ["logic"] } + +anyhow = "1.0.75" +blockscout-display-bytes = "1.0.0" +blockscout-service-launcher = { version = "0.9.0", features = ["database-0_12"] } +futures = "0.3.29" +governor = "0.5.1" +reqwest-rate-limiter = { git = "https://github.com/blockscout/blockscout-rs", rev = "edb610b" } +reqwest = "0.11" +reqwest-middleware = "0.2.3" +reqwest-retry = "0.2.3" +sea-orm = "0.12.4" +serde = "1.0.190" +serde_json = "1.0.108" +serde_path_to_error = "0.1.14" +tokio = { version = "1.33.0", features = ["rt-multi-thread", "macros"] } +tracing = "0.1.40" +url = "2.3.1" \ No newline at end of file diff --git a/eth-bytecode-db-extractors/contracts-lookup/src/blockscout.rs b/eth-bytecode-db-extractors/contracts-lookup/src/blockscout.rs new file mode 100644 index 000000000..7321f2e16 --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup/src/blockscout.rs @@ -0,0 +1,85 @@ +use anyhow::Context; +use blockscout_display_bytes::Bytes; +use governor::{Quota, RateLimiter}; +use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; +use reqwest_rate_limiter::RateLimiterMiddleware; +use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; +use serde::de::DeserializeOwned; +use std::{num::NonZeroU32, str::FromStr}; +use url::Url; + +#[derive(Clone)] +pub struct Client { + base_url: Url, + request_client: ClientWithMiddleware, + api_key: String, +} + +impl Client { + pub fn try_new( + base_url: String, + limit_requests_per_second: u32, + api_key: String, + ) -> anyhow::Result { + let base_url = Url::from_str(&base_url).context("invalid blockscout base url")?; + let max_burst = NonZeroU32::new(limit_requests_per_second) + .ok_or_else(|| anyhow::anyhow!("invalid limit requests per second"))?; + + let rate_limiter = RateLimiter::direct(Quota::per_second(max_burst)); + + let retry_policy = ExponentialBackoff::builder().build_with_max_retries(3); + let client = ClientBuilder::new(reqwest::Client::new()) + .with(RetryTransientMiddleware::new_with_policy(retry_policy)) + .with(RateLimiterMiddleware::new(rate_limiter)) + .build(); + + Ok(Self { + base_url, + request_client: client, + api_key, + }) + } + + pub async fn search_contract(&self, contract_address: Bytes) -> anyhow::Result<()> { + let url = { + let path = format!("/api/v2/import/smart-contracts/{contract_address}"); + let mut url = self.base_url.clone(); + url.set_path(&path); + url + }; + + self.send_request(url, [("x-api-key", &self.api_key)]) + .await + .context("sending request")?; + + Ok(()) + } + + async fn send_request( + &self, + url: Url, + headers: impl IntoIterator, impl AsRef)>, + ) -> anyhow::Result { + let mut request = self.request_client.get(url); + for (header_key, header_value) in headers { + request = request.header(header_key.as_ref(), header_value.as_ref()); + } + + let response = request.send().await.context("sending request failed")?; + + // Continue only in case if request results is success + if !response.status().is_success() { + return Err(anyhow::anyhow!( + "Invalid status code get as a result: {}", + response.status() + )); + } + + let result = response + .text() + .await + .context("deserializing response into string failed")?; + let jd = &mut serde_json::Deserializer::from_str(&result); + serde_path_to_error::deserialize(jd).context("deserializing response failed") + } +} diff --git a/eth-bytecode-db-extractors/contracts-lookup/src/client.rs b/eth-bytecode-db-extractors/contracts-lookup/src/client.rs new file mode 100644 index 000000000..947e151d0 --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup/src/client.rs @@ -0,0 +1,94 @@ +use crate::blockscout; +use anyhow::Context; +use blockscout_display_bytes::Bytes; +use entity::contract_addresses; +use sea_orm::{prelude::Uuid, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; +use std::sync::Arc; + +#[derive(Clone)] +pub struct Client { + pub db_client: Arc, + pub blockscout_client: blockscout::Client, +} + +impl Client { + pub fn try_new( + db_client: DatabaseConnection, + blockscout_url: String, + limit_requests_per_second: u32, + blockscout_api_key: String, + ) -> anyhow::Result { + let blockscout_client = blockscout::Client::try_new( + blockscout_url, + limit_requests_per_second, + blockscout_api_key, + )?; + + let client = Self { + db_client: Arc::new(db_client), + blockscout_client, + }; + + Ok(client) + } +} + +impl Client { + pub async fn lookup_contracts(self) -> anyhow::Result { + let mut processed = 0; + while let Some(contract_address_model) = self.next_contract().await? { + let contract_address = Bytes::from(contract_address_model.contract_address.clone()); + let job_id = contract_address_model.job_id; + + tracing::info!( + contract_address = contract_address.to_string(), + "processing contract" + ); + + job_queue::process_result!( + self.db_client.as_ref(), + self.blockscout_client + .search_contract(contract_address.clone()) + .await, + job_id, + contract_address = contract_address + ); + + self.mark_as_success(job_id, contract_address).await?; + + processed += 1; + } + + Ok(processed) + } + + async fn mark_as_success(&self, job_id: Uuid, contract_address: Bytes) -> anyhow::Result<()> { + job_queue::mark_as_success(self.db_client.as_ref(), job_id, None) + .await + .context(format!( + "saving success details failed for the contract {}", + contract_address, + ))?; + + Ok(()) + } + + async fn next_contract(&self) -> anyhow::Result> { + let next_job_id = job_queue::next_job_id(self.db_client.as_ref()) + .await + .context("querying the next_job_id")?; + + if let Some(job_id) = next_job_id { + let model = contract_addresses::Entity::find() + .filter(contract_addresses::Column::JobId.eq(job_id)) + .one(self.db_client.as_ref()) + .await + .context("querying contract_address model")? + .expect("contract_address model does not exist"); + + return Ok(Some(model)); + } + + Ok(None) + } +} diff --git a/eth-bytecode-db-extractors/contracts-lookup/src/lib.rs b/eth-bytecode-db-extractors/contracts-lookup/src/lib.rs new file mode 100644 index 000000000..65723645e --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup/src/lib.rs @@ -0,0 +1,6 @@ +mod blockscout; +mod client; +mod settings; + +pub use client::Client; +pub use settings::Settings; diff --git a/eth-bytecode-db-extractors/contracts-lookup/src/main.rs b/eth-bytecode-db-extractors/contracts-lookup/src/main.rs new file mode 100644 index 000000000..6680cac6b --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup/src/main.rs @@ -0,0 +1,42 @@ +use anyhow::Context; +use blockscout_service_launcher::{self as launcher, launcher::ConfigSettings}; +use contracts_lookup::{Client, Settings}; +use migration::Migrator; + +const SERVICE_NAME: &str = "contracts-lookup-extractor"; + +#[tokio::main] +async fn main() -> Result<(), anyhow::Error> { + launcher::tracing::init_logs(SERVICE_NAME, &Default::default(), &Default::default()) + .context("tracing initialization")?; + + let settings = Settings::build().context("failed to read config")?; + + let mut connect_options = sea_orm::ConnectOptions::new(&settings.database_url); + connect_options.sqlx_logging_level(tracing::log::LevelFilter::Debug); + let db_connection = launcher::database::initialize_postgres::( + connect_options, + settings.create_database, + settings.run_migrations, + ) + .await?; + + let client = Client::try_new( + db_connection, + settings.blockscout_url, + settings.limit_requests_per_second, + settings.blockscout_api_key, + )?; + + let mut handles = Vec::with_capacity(settings.n_threads); + for _ in 0..settings.n_threads { + let client = client.clone(); + let handle = tokio::spawn(client.lookup_contracts()); + handles.push(handle); + } + for result in futures::future::join_all(handles).await { + result.context("join handle")?.context("verify contracts")?; + } + + Ok(()) +} diff --git a/eth-bytecode-db-extractors/contracts-lookup/src/settings.rs b/eth-bytecode-db-extractors/contracts-lookup/src/settings.rs new file mode 100644 index 000000000..00db33b7f --- /dev/null +++ b/eth-bytecode-db-extractors/contracts-lookup/src/settings.rs @@ -0,0 +1,41 @@ +use blockscout_service_launcher::launcher; +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Settings { + pub database_url: String, + #[serde(default)] + pub create_database: bool, + #[serde(default)] + pub run_migrations: bool, + + #[serde(default = "default_blockscout_url")] + pub blockscout_url: String, + pub blockscout_api_key: String, + #[serde(default = "default_limit_requests_per_second")] + pub limit_requests_per_second: u32, + + #[serde(default = "default_n_threads")] + pub n_threads: usize, +} + +impl launcher::ConfigSettings for Settings { + const SERVICE_NAME: &'static str = "CONTRACTS_LOOKUP_EXTRACTOR"; +} + +// fn default_blockscout_url() -> String { +// "https://eth.blockscout.com".to_string() +// } + +fn default_blockscout_url() -> String { + "https://eth-goerli.blockscout.com".to_string() +} + +fn default_limit_requests_per_second() -> u32 { + u32::MAX +} + +fn default_n_threads() -> usize { + 1 +} From 3d5d7038856ec02cc17310b264f3ab93a2c4775c Mon Sep 17 00:00:00 2001 From: Rim Rakhimov Date: Fri, 10 Nov 2023 16:40:58 +0300 Subject: [PATCH 2/4] Update contract_addresses make fetch_block to be nullable --- .../src/m20220101_000001_create_contract_addresses_table.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/eth-bytecode-db-extractors/contracts-lookup-migration/src/m20220101_000001_create_contract_addresses_table.rs b/eth-bytecode-db-extractors/contracts-lookup-migration/src/m20220101_000001_create_contract_addresses_table.rs index 81b289868..2da4f7475 100644 --- a/eth-bytecode-db-extractors/contracts-lookup-migration/src/m20220101_000001_create_contract_addresses_table.rs +++ b/eth-bytecode-db-extractors/contracts-lookup-migration/src/m20220101_000001_create_contract_addresses_table.rs @@ -10,7 +10,7 @@ impl MigrationTrait for Migration { CREATE TABLE "contract_addresses" ( "contract_address" bytea NOT NULL, - "fetched_coin_balance_block_number" bigint NOT NULL, + "fetched_coin_balance_block_number" bigint, "inserted_at" timestamp NOT NULL, "updated_at" timestamp NOT NULL, @@ -23,9 +23,7 @@ impl MigrationTrait for Migration { let create_job_queue_connection_statements = job_queue::migration::create_job_queue_connection_statements("contract_addresses"); - let mut statements = vec![ - create_contract_addresses_table, - ]; + let mut statements = vec![create_contract_addresses_table]; statements.extend( create_job_queue_connection_statements .iter() From 4281a6a02e1839ea4628b15f9d6f1f8c73b7d7fb Mon Sep 17 00:00:00 2001 From: Rim Rakhimov Date: Fri, 10 Nov 2023 17:35:20 +0300 Subject: [PATCH 3/4] Add search_contract response structure --- .../contracts-lookup/src/blockscout.rs | 16 +++++++++++----- .../contracts-lookup/src/client.rs | 14 ++++++++++---- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/eth-bytecode-db-extractors/contracts-lookup/src/blockscout.rs b/eth-bytecode-db-extractors/contracts-lookup/src/blockscout.rs index 7321f2e16..a0a13e2e6 100644 --- a/eth-bytecode-db-extractors/contracts-lookup/src/blockscout.rs +++ b/eth-bytecode-db-extractors/contracts-lookup/src/blockscout.rs @@ -4,10 +4,15 @@ use governor::{Quota, RateLimiter}; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use reqwest_rate_limiter::RateLimiterMiddleware; use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; -use serde::de::DeserializeOwned; +use serde::{de::DeserializeOwned, Deserialize}; use std::{num::NonZeroU32, str::FromStr}; use url::Url; +#[derive(Clone, Debug, Deserialize)] +pub struct SearchContractResponse { + pub message: Option, +} + #[derive(Clone)] pub struct Client { base_url: Url, @@ -40,7 +45,10 @@ impl Client { }) } - pub async fn search_contract(&self, contract_address: Bytes) -> anyhow::Result<()> { + pub async fn search_contract( + &self, + contract_address: Bytes, + ) -> anyhow::Result { let url = { let path = format!("/api/v2/import/smart-contracts/{contract_address}"); let mut url = self.base_url.clone(); @@ -50,9 +58,7 @@ impl Client { self.send_request(url, [("x-api-key", &self.api_key)]) .await - .context("sending request")?; - - Ok(()) + .context("sending request") } async fn send_request( diff --git a/eth-bytecode-db-extractors/contracts-lookup/src/client.rs b/eth-bytecode-db-extractors/contracts-lookup/src/client.rs index 947e151d0..05e460fbe 100644 --- a/eth-bytecode-db-extractors/contracts-lookup/src/client.rs +++ b/eth-bytecode-db-extractors/contracts-lookup/src/client.rs @@ -45,7 +45,7 @@ impl Client { "processing contract" ); - job_queue::process_result!( + let response = job_queue::process_result!( self.db_client.as_ref(), self.blockscout_client .search_contract(contract_address.clone()) @@ -54,7 +54,8 @@ impl Client { contract_address = contract_address ); - self.mark_as_success(job_id, contract_address).await?; + self.mark_as_success(job_id, contract_address, response.message) + .await?; processed += 1; } @@ -62,8 +63,13 @@ impl Client { Ok(processed) } - async fn mark_as_success(&self, job_id: Uuid, contract_address: Bytes) -> anyhow::Result<()> { - job_queue::mark_as_success(self.db_client.as_ref(), job_id, None) + async fn mark_as_success( + &self, + job_id: Uuid, + contract_address: Bytes, + message: Option, + ) -> anyhow::Result<()> { + job_queue::mark_as_success(self.db_client.as_ref(), job_id, message) .await .context(format!( "saving success details failed for the contract {}", From d5c5cdafe8670b50339ef966e51ad3dbf3f2976a Mon Sep 17 00:00:00 2001 From: Rim Rakhimov Date: Fri, 2 Feb 2024 21:05:50 +0400 Subject: [PATCH 4/4] Update entities --- .../contracts-lookup-entity/src/contract_addresses.rs | 4 ++-- eth-bytecode-db-extractors/contracts-lookup-entity/src/lib.rs | 2 +- .../contracts-lookup-entity/src/prelude.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eth-bytecode-db-extractors/contracts-lookup-entity/src/contract_addresses.rs b/eth-bytecode-db-extractors/contracts-lookup-entity/src/contract_addresses.rs index 1fb01fdd2..e9d9f8689 100644 --- a/eth-bytecode-db-extractors/contracts-lookup-entity/src/contract_addresses.rs +++ b/eth-bytecode-db-extractors/contracts-lookup-entity/src/contract_addresses.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.11 use sea_orm::entity::prelude::*; @@ -11,7 +11,7 @@ pub struct Model { column_type = "Binary(BlobSize::Blob(None))" )] pub contract_address: Vec, - pub fetched_coin_balance_block_number: i64, + pub fetched_coin_balance_block_number: Option, pub inserted_at: DateTime, pub updated_at: DateTime, #[sea_orm(column_name = "_job_id")] diff --git a/eth-bytecode-db-extractors/contracts-lookup-entity/src/lib.rs b/eth-bytecode-db-extractors/contracts-lookup-entity/src/lib.rs index 5a9f03355..8b6242eec 100644 --- a/eth-bytecode-db-extractors/contracts-lookup-entity/src/lib.rs +++ b/eth-bytecode-db-extractors/contracts-lookup-entity/src/lib.rs @@ -1,4 +1,4 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.11 pub mod prelude; diff --git a/eth-bytecode-db-extractors/contracts-lookup-entity/src/prelude.rs b/eth-bytecode-db-extractors/contracts-lookup-entity/src/prelude.rs index f9942172a..70e77a646 100644 --- a/eth-bytecode-db-extractors/contracts-lookup-entity/src/prelude.rs +++ b/eth-bytecode-db-extractors/contracts-lookup-entity/src/prelude.rs @@ -1,3 +1,3 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.11 pub use super::{contract_addresses::Entity as ContractAddresses, job_queue::Entity as JobQueue};