From 95ba43423b9390c44f46364276b420a2688650ce Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Thu, 28 Mar 2024 13:21:21 +0100 Subject: [PATCH] feat(auction-server): Use sqlx to persist opportunities and bids (#37) * Insert opportunities and remove dashmap * Remove bidders field from opportunity * Bid table * Refactor bidstore * Commit sqlx files --- .pre-commit-config.yaml | 2 +- .prettierignore | 1 + Tiltfile | 2 +- auction-server/.env.example | 1 + ...87ff51075c63e76f11f9f3dce49154fb04c4c.json | 15 + ...fac94db0b487dc485e767cc9ab13b94e88faf.json | 22 + ...13d26119b14307ceae85414b74db38e4ac625.json | 27 + ...6225abe4d6b134991198c4a58abee97917c44.json | 32 + auction-server/Cargo.lock | 678 +++++++++++++++++- auction-server/Cargo.toml | 4 +- auction-server/README.md | 26 + .../migrations/20240320162754_init.down.sql | 1 + .../migrations/20240320162754_init.up.sql | 13 + .../migrations/20240326063340_bids.down.sql | 2 + .../migrations/20240326063340_bids.up.sql | 15 + auction-server/src/api.rs | 1 - auction-server/src/api/bid.rs | 7 +- auction-server/src/api/opportunity.rs | 89 +-- auction-server/src/auction.rs | 68 +- auction-server/src/config/server.rs | 6 +- auction-server/src/opportunity_adapter.rs | 48 +- auction-server/src/server.rs | 19 +- auction-server/src/state.rs | 243 ++++++- auction-server/src/token_spoof.rs | 3 - 24 files changed, 1130 insertions(+), 195 deletions(-) create mode 100644 .prettierignore create mode 100644 auction-server/.env.example create mode 100644 auction-server/.sqlx/query-23f211b134ef73ab694a59738fd87ff51075c63e76f11f9f3dce49154fb04c4c.json create mode 100644 auction-server/.sqlx/query-b1359e04576e41974c0f4c6178efac94db0b487dc485e767cc9ab13b94e88faf.json create mode 100644 auction-server/.sqlx/query-c1e8edc296507ee7b820457492213d26119b14307ceae85414b74db38e4ac625.json create mode 100644 auction-server/.sqlx/query-def2e2585d56895a2a9a7fe5aff6225abe4d6b134991198c4a58abee97917c44.json create mode 100644 auction-server/migrations/20240320162754_init.down.sql create mode 100644 auction-server/migrations/20240320162754_init.up.sql create mode 100644 auction-server/migrations/20240326063340_bids.down.sql create mode 100644 auction-server/migrations/20240326063340_bids.up.sql diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3629b2c8..262c3d3e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - id: cargo-clippy-auction-server name: Cargo clippy for auction server language: "rust" - entry: cargo +nightly-2023-07-23 clippy --manifest-path ./auction-server/Cargo.toml --tests --fix --allow-dirty --allow-staged -- -D warnings + entry: cargo +stable clippy --manifest-path ./auction-server/Cargo.toml --tests --fix --allow-dirty --allow-staged -- -D warnings pass_filenames: false files: auction-server # Hooks for vault-simulator diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 00000000..20ea39c1 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +**/.sqlx/*.json diff --git a/Tiltfile b/Tiltfile index d5757855..29f8a685 100644 --- a/Tiltfile +++ b/Tiltfile @@ -78,7 +78,7 @@ local_resource( local_resource( "auction-server", - serve_cmd="source ../tilt-resources.env; cargo run -- run --relayer-private-key $RELAYER_PRIVATE_KEY", + serve_cmd="source ../tilt-resources.env; source ./.env; cargo run -- run --database-url $DATABASE_URL --relayer-private-key $RELAYER_PRIVATE_KEY", serve_dir="auction-server", resource_deps=["create-configs"], readiness_probe=probe(period_secs=5, http_get=http_get_action(port=9000)), diff --git a/auction-server/.env.example b/auction-server/.env.example new file mode 100644 index 00000000..f2cbd8d0 --- /dev/null +++ b/auction-server/.env.example @@ -0,0 +1 @@ +DATABASE_URL=postgresql://postgres@localhost/postgres diff --git a/auction-server/.sqlx/query-23f211b134ef73ab694a59738fd87ff51075c63e76f11f9f3dce49154fb04c4c.json b/auction-server/.sqlx/query-23f211b134ef73ab694a59738fd87ff51075c63e76f11f9f3dce49154fb04c4c.json new file mode 100644 index 00000000..75616d7b --- /dev/null +++ b/auction-server/.sqlx/query-23f211b134ef73ab694a59738fd87ff51075c63e76f11f9f3dce49154fb04c4c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE opportunity SET removal_time = $1 WHERE id = $2 AND removal_time IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Timestamp", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "23f211b134ef73ab694a59738fd87ff51075c63e76f11f9f3dce49154fb04c4c" +} diff --git a/auction-server/.sqlx/query-b1359e04576e41974c0f4c6178efac94db0b487dc485e767cc9ab13b94e88faf.json b/auction-server/.sqlx/query-b1359e04576e41974c0f4c6178efac94db0b487dc485e767cc9ab13b94e88faf.json new file mode 100644 index 00000000..b85d5bd1 --- /dev/null +++ b/auction-server/.sqlx/query-b1359e04576e41974c0f4c6178efac94db0b487dc485e767cc9ab13b94e88faf.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO opportunity (id,\n creation_time,\n permission_key,\n chain_id,\n target_contract,\n target_call_value,\n target_calldata,\n sell_tokens,\n buy_tokens) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Timestamp", + "Bytea", + "Text", + "Bytea", + "Numeric", + "Bytea", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "b1359e04576e41974c0f4c6178efac94db0b487dc485e767cc9ab13b94e88faf" +} diff --git a/auction-server/.sqlx/query-c1e8edc296507ee7b820457492213d26119b14307ceae85414b74db38e4ac625.json b/auction-server/.sqlx/query-c1e8edc296507ee7b820457492213d26119b14307ceae85414b74db38e4ac625.json new file mode 100644 index 00000000..7d295b6a --- /dev/null +++ b/auction-server/.sqlx/query-c1e8edc296507ee7b820457492213d26119b14307ceae85414b74db38e4ac625.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE bid SET status = $1, removal_time = $2 WHERE id = $3 AND removal_time IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + { + "Custom": { + "name": "bid_status", + "kind": { + "Enum": [ + "pending", + "lost", + "submitted" + ] + } + } + }, + "Timestamp", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c1e8edc296507ee7b820457492213d26119b14307ceae85414b74db38e4ac625" +} diff --git a/auction-server/.sqlx/query-def2e2585d56895a2a9a7fe5aff6225abe4d6b134991198c4a58abee97917c44.json b/auction-server/.sqlx/query-def2e2585d56895a2a9a7fe5aff6225abe4d6b134991198c4a58abee97917c44.json new file mode 100644 index 00000000..6db68b01 --- /dev/null +++ b/auction-server/.sqlx/query-def2e2585d56895a2a9a7fe5aff6225abe4d6b134991198c4a58abee97917c44.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO bid (id, creation_time, permission_key, chain_id, target_contract, target_calldata, bid_amount, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Timestamp", + "Bytea", + "Text", + "Bytea", + "Bytea", + "Numeric", + { + "Custom": { + "name": "bid_status", + "kind": { + "Enum": [ + "pending", + "lost", + "submitted" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "def2e2585d56895a2a9a7fe5aff6225abe4d6b134991198c4a58abee97917c44" +} diff --git a/auction-server/Cargo.lock b/auction-server/Cargo.lock index c12c3cab..4c993af3 100644 --- a/auction-server/Cargo.lock +++ b/auction-server/Cargo.lock @@ -38,6 +38,19 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.2" @@ -47,6 +60,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "anstream" version = "0.6.5" @@ -160,9 +179,18 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "auction-server" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "async-stream", @@ -170,12 +198,12 @@ dependencies = [ "axum-macros", "axum-streams", "clap", - "dashmap", "ethers", "futures", "serde", "serde_json", "serde_yaml", + "sqlx", "tokio", "tokio-stream", "tower-http", @@ -335,6 +363,17 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" +[[package]] +name = "bigdecimal" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -361,6 +400,9 @@ name = "bitflags" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +dependencies = [ + "serde", +] [[package]] name = "bitvec" @@ -662,6 +704,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.3.2" @@ -694,6 +751,16 @@ dependencies = [ "memoffset", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.17" @@ -740,19 +807,6 @@ dependencies = [ "cipher", ] -[[package]] -name = "dashmap" -version = "5.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" -dependencies = [ - "cfg-if", - "hashbrown", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "data-encoding" version = "2.5.0" @@ -766,6 +820,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" dependencies = [ "const-oid", + "pem-rfc7468", "zeroize", ] @@ -869,6 +924,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dunce" version = "1.0.4" @@ -894,6 +955,9 @@ name = "either" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +dependencies = [ + "serde", +] [[package]] name = "elliptic-curve" @@ -966,6 +1030,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "eth-keystore" version = "0.5.0" @@ -1284,6 +1359,12 @@ dependencies = [ "yansi", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "eyre" version = "0.6.11" @@ -1310,6 +1391,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + [[package]] name = "fixed-hash" version = "0.8.0" @@ -1338,12 +1425,38 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1411,6 +1524,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.29" @@ -1568,6 +1692,10 @@ name = "hashbrown" version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] name = "hashers" @@ -1578,11 +1706,23 @@ dependencies = [ "fxhash", ] +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown", +] + [[package]] name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" @@ -1596,6 +1736,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1810,6 +1959,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -1904,6 +2062,9 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "libc" @@ -1928,6 +2089,17 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.12" @@ -2006,6 +2178,12 @@ dependencies = [ "unicase", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -2026,12 +2204,40 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "new_debug_unreachable" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2053,6 +2259,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -2063,6 +2286,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.17" @@ -2144,6 +2378,50 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.41", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -2216,6 +2494,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + [[package]] name = "path-slash" version = "0.2.1" @@ -2253,6 +2537,15 @@ dependencies = [ "base64 0.13.1", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -2362,6 +2655,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -2743,6 +3047,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rust-embed" version = "6.8.1" @@ -2897,6 +3221,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2939,6 +3272,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.20" @@ -3182,6 +3538,9 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] [[package]] name = "spki" @@ -3193,6 +3552,224 @@ dependencies = [ "der", ] +[[package]] +name = "sqlformat" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +dependencies = [ + "itertools 0.12.1", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" +dependencies = [ + "ahash", + "atoi", + "bigdecimal", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap", + "log", + "memchr", + "native-tls", + "once_cell", + "paste", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "time", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid 1.6.1", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" +dependencies = [ + "atoi", + "base64 0.21.5", + "bigdecimal", + "bitflags 2.4.1", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "time", + "tracing", + "uuid 1.6.1", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" +dependencies = [ + "atoi", + "base64 0.21.5", + "bigdecimal", + "bitflags 2.4.1", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "num-bigint", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "time", + "tracing", + "uuid 1.6.1", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "time", + "tracing", + "url", + "urlencoding", + "uuid 1.6.1", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -3212,6 +3789,17 @@ dependencies = [ "precomputed-hash", ] +[[package]] +name = "stringprep" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +dependencies = [ + "finl_unicode", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "strsim" version = "0.10.0" @@ -3763,12 +4351,24 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + [[package]] name = "unicode-xid" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "unsafe-libyaml" version = "0.2.10" @@ -3798,6 +4398,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -3877,6 +4483,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -3908,6 +4520,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.89" @@ -3990,6 +4608,16 @@ version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +[[package]] +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -4206,6 +4834,26 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.41", +] + [[package]] name = "zeroize" version = "1.7.0" diff --git a/auction-server/Cargo.toml b/auction-server/Cargo.toml index 53a771b2..5f9f1a44 100644 --- a/auction-server/Cargo.toml +++ b/auction-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "auction-server" -version = "0.2.2" +version = "0.2.3" edition = "2021" license-file = "license.txt" @@ -24,4 +24,4 @@ utoipa-swagger-ui = { version = "3.1.4", features = ["axum"] } serde_yaml = "0.9.25" ethers = "2.0.10" axum-macros = "0.4.0" -dashmap = { version = "5.4.0" } +sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-native-tls", "postgres", "time", "uuid", "bigdecimal" ] } diff --git a/auction-server/README.md b/auction-server/README.md index e5128ffa..be3fbeb1 100644 --- a/auction-server/README.md +++ b/auction-server/README.md @@ -7,6 +7,8 @@ Each blockchain is configured in `config.yaml`. This package uses Cargo for building and dependency management. Simply run `cargo build` and `cargo test` to build and test the project. +We use `sqlx` for database operations, so you need to have a PostgreSQL server running locally. +Check the Migration section for more information on how to setup the database. ## Local Development @@ -24,3 +26,27 @@ cargo run -- run --relayer-private-key This command will start the webservice on `localhost:9000`. You can check the documentation of the webservice by visiting `localhost:9000/docs`. + +## DB & Migrations + +sqlx checks the database schema at compile time, so you need to have the database schema up-to-date +before building the project. You can create a `.env` file similar +to the `.env.example` file and set `DATABASE_URL` to the URL of your PostgreSQL database. This file +will be picked up by sqlx-cli and cargo scripts when running the checks. + +In the current folder, install sqlx-cli by running `cargo install sqlx-cli`. +Then, run the following command to apply migrations: + +```bash +sqlx migrate run +``` + +We use revertible migrations to manage the database schema. You can create a new migration by running: + +```bash +sqlx migrate add -r +``` + +Since we don't have a running db instance on CI, we use `cargo sqlx prepare` to generate the necessary +info offline. This command will update the `.sqlx` folder. +You need to commit the changes to this folder when adding or changing the queries. diff --git a/auction-server/migrations/20240320162754_init.down.sql b/auction-server/migrations/20240320162754_init.down.sql new file mode 100644 index 00000000..bf616551 --- /dev/null +++ b/auction-server/migrations/20240320162754_init.down.sql @@ -0,0 +1 @@ +DROP TABLE opportunity; diff --git a/auction-server/migrations/20240320162754_init.up.sql b/auction-server/migrations/20240320162754_init.up.sql new file mode 100644 index 00000000..c7c5e212 --- /dev/null +++ b/auction-server/migrations/20240320162754_init.up.sql @@ -0,0 +1,13 @@ +CREATE TABLE opportunity +( + id UUID PRIMARY KEY, + creation_time TIMESTAMP NOT NULL, + permission_key BYTEA NOT NULL, + chain_id TEXT NOT NULL, + target_contract BYTEA NOT NULL CHECK (LENGTH(target_contract) = 20), + target_call_value NUMERIC(78, 0) NOT NULL, + target_calldata BYTEA NOT NULL, + sell_tokens JSONB NOT NULL, + buy_tokens JSONB NOT NULL, + removal_time TIMESTAMP +); diff --git a/auction-server/migrations/20240326063340_bids.down.sql b/auction-server/migrations/20240326063340_bids.down.sql new file mode 100644 index 00000000..77064366 --- /dev/null +++ b/auction-server/migrations/20240326063340_bids.down.sql @@ -0,0 +1,2 @@ +DROP TABLE bid; +DROP TYPE bid_status; diff --git a/auction-server/migrations/20240326063340_bids.up.sql b/auction-server/migrations/20240326063340_bids.up.sql new file mode 100644 index 00000000..451bb353 --- /dev/null +++ b/auction-server/migrations/20240326063340_bids.up.sql @@ -0,0 +1,15 @@ +CREATE TYPE bid_status AS ENUM ('pending', 'lost', 'submitted'); + +CREATE TABLE bid +( + id UUID PRIMARY KEY, + creation_time TIMESTAMP NOT NULL, + permission_key BYTEA NOT NULL, + chain_id TEXT NOT NULL, + target_contract BYTEA NOT NULL CHECK (LENGTH(target_contract) = 20), + target_calldata BYTEA NOT NULL, + bid_amount NUMERIC(78, 0) NOT NULL, + status bid_status NOT NULL, + auction_id UUID, -- TODO: should be linked to the auction table in the future + removal_time TIMESTAMP -- TODO: should be removed and read from the auction table in the future +); diff --git a/auction-server/src/api.rs b/auction-server/src/api.rs index f69ba188..4aaa91ed 100644 --- a/auction-server/src/api.rs +++ b/auction-server/src/api.rs @@ -134,7 +134,6 @@ pub async fn live() -> Response { (StatusCode::OK, "OK").into_response() } - pub async fn start_api(run_options: RunOptions, store: Arc) -> Result<()> { // Make sure functions included in the paths section have distinct names, otherwise some api generators will fail #[derive(OpenApi)] diff --git a/auction-server/src/api/bid.rs b/auction-server/src/api/bid.rs index 9c6a5e72..460c5450 100644 --- a/auction-server/src/api/bid.rs +++ b/auction-server/src/api/bid.rs @@ -40,7 +40,6 @@ pub struct BidResult { pub id: BidId, } - /// Bid on a specific permission key for a specific chain. /// /// Your bid will be simulated and verified by the server. Depending on the outcome of the auction, a transaction @@ -69,7 +68,6 @@ pub async fn process_bid(store: Arc, bid: Bid) -> Result, } } - /// Query the status of a specific bid. #[utoipa::path(get, path = "/v1/bids/{bid_id}", params(("bid_id"=String, description = "Bid id to query for")), @@ -82,9 +80,8 @@ pub async fn bid_status( State(store): State>, Path(bid_id): Path, ) -> Result, RestError> { - let status = store.bid_status_store.get_status(&bid_id).await; - match status { - Some(status) => Ok(status.into()), + match store.bids.read().await.get(&bid_id) { + Some(bid) => Ok(bid.status.clone().into()), None => Err(RestError::BidNotFound), } } diff --git a/auction-server/src/api/opportunity.rs b/auction-server/src/api/opportunity.rs index b2b310de..daf4c234 100644 --- a/auction-server/src/api/opportunity.rs +++ b/auction-server/src/api/opportunity.rs @@ -33,13 +33,8 @@ use { Deserialize, Serialize, }, - std::{ - sync::Arc, - time::{ - SystemTime, - UNIX_EPOCH, - }, - }, + sqlx::types::time::OffsetDateTime, + std::sync::Arc, utoipa::{ IntoParams, ToResponse, @@ -52,10 +47,10 @@ use { #[derive(Serialize, Deserialize, ToSchema, Clone, ToResponse)] pub struct OpportunityParamsWithMetadata { /// The opportunity unique id - #[schema(example = "obo3ee3e-58cc-4372-a567-0e02b2c3d479", value_type=String)] + #[schema(example = "obo3ee3e-58cc-4372-a567-0e02b2c3d479", value_type = String)] opportunity_id: OpportunityId, /// Creation time of the opportunity - #[schema(example = 1700000000, value_type=i64)] + #[schema(example = 1700000000, value_type = i64)] creation_time: UnixTimestamp, /// opportunity data #[serde(flatten)] @@ -87,9 +82,9 @@ impl From for OpportunityParamsWithMetadata { /// The opportunity will be verified by the server. If the opportunity is valid, it will be stored in the database /// and will be available for bidding. #[utoipa::path(post, path = "/v1/opportunities", request_body = OpportunityParams, responses( - (status = 200, description = "The created opportunity", body = OpportunityParamsWithMetadata), - (status = 400, response = ErrorBodyResponse), - (status = 404, description = "Chain id was not found", body = ErrorBodyResponse), +(status = 200, description = "The created opportunity", body = OpportunityParamsWithMetadata), +(status = 400, response = ErrorBodyResponse), +(status = 404, description = "Chain id was not found", body = ErrorBodyResponse), ),)] pub async fn post_opportunity( State(store): State>, @@ -102,36 +97,23 @@ pub async fn post_opportunity( .ok_or(RestError::InvalidChainId)?; let id = Uuid::new_v4(); + let now_odt = OffsetDateTime::now_utc(); let opportunity = Opportunity { id, - creation_time: SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|_| RestError::BadParameters("Invalid system time".to_string()))? - .as_secs() as UnixTimestamp, + creation_time: now_odt.unix_timestamp() as UnixTimestamp, params: versioned_params.clone(), - bidders: Default::default(), }; verify_opportunity(params.clone(), chain_store, store.relayer.address()) .await .map_err(|e| RestError::InvalidOpportunity(e.to_string()))?; - - let opportunities_map = &store.opportunity_store.opportunities; - if let Some(mut opportunities_existing) = opportunities_map.get_mut(¶ms.permission_key) { - // check if same opportunity exists in the vector - for opportunity_existing in opportunities_existing.iter() { - if opportunity_existing == &opportunity { - return Err(RestError::BadParameters( - "Duplicate opportunity submission".to_string(), - )); - } - } - - opportunities_existing.push(opportunity.clone()); - } else { - opportunities_map.insert(params.permission_key.clone(), vec![opportunity.clone()]); + if store.opportunity_exists(&opportunity).await { + return Err(RestError::BadParameters( + "Duplicate opportunity submission".to_string(), + )); } + store.add_opportunity(opportunity.clone()).await?; store .ws @@ -142,32 +124,33 @@ pub async fn post_opportunity( RestError::TemporarilyUnavailable })?; - tracing::debug!("number of permission keys: {}", opportunities_map.len()); - tracing::debug!( - "number of opportunities for key: {}", - opportunities_map - .get(¶ms.permission_key) - .map(|opps| opps.len()) - .unwrap_or(0) - ); + { + let opportunities_map = &store.opportunity_store.opportunities.read().await; + tracing::debug!("number of permission keys: {}", opportunities_map.len()); + tracing::debug!( + "number of opportunities for key: {}", + opportunities_map + .get(¶ms.permission_key) + .map_or(0, |opps| opps.len()) + ); + } let opportunity_with_metadata: OpportunityParamsWithMetadata = opportunity.into(); Ok(opportunity_with_metadata.into()) } - #[derive(Serialize, Deserialize, IntoParams)] pub struct ChainIdQueryParams { - #[param(example = "sepolia", value_type=Option)] + #[param(example = "sepolia", value_type = Option < String >)] chain_id: Option, } /// Fetch all opportunities ready to be exectued. #[utoipa::path(get, path = "/v1/opportunities", responses( - (status = 200, description = "Array of opportunities ready for bidding", body = Vec), - (status = 400, response = ErrorBodyResponse), - (status = 404, description = "Chain id was not found", body = ErrorBodyResponse), +(status = 200, description = "Array of opportunities ready for bidding", body = Vec < OpportunityParamsWithMetadata >), +(status = 400, response = ErrorBodyResponse), +(status = 404, description = "Chain id was not found", body = ErrorBodyResponse), ), params(ChainIdQueryParams))] pub async fn get_opportunities( @@ -177,9 +160,11 @@ pub async fn get_opportunities( let opportunities: Vec = store .opportunity_store .opportunities + .read() + .await .iter() - .map(|opportunities_key| { - opportunities_key + .map(|(_key, opportunities)| { + opportunities .last() .expect("A permission key vector should have at least one opportunity") .clone() @@ -199,11 +184,11 @@ pub async fn get_opportunities( } /// Bid on opportunity -#[utoipa::path(post, path = "/v1/opportunities/{opportunity_id}/bids", request_body=OpportunityBid, - params(("opportunity_id"=String, description = "Opportunity id to bid on")), responses( - (status = 200, description = "Bid Result", body = BidResult, example = json!({"status": "OK"})), - (status = 400, response = ErrorBodyResponse), - (status = 404, description = "Opportunity or chain id was not found", body = ErrorBodyResponse), +#[utoipa::path(post, path = "/v1/opportunities/{opportunity_id}/bids", request_body = OpportunityBid, +params(("opportunity_id" = String, description = "Opportunity id to bid on")), responses( +(status = 200, description = "Bid Result", body = BidResult, example = json ! ({"status": "OK"})), +(status = 400, response = ErrorBodyResponse), +(status = 404, description = "Opportunity or chain id was not found", body = ErrorBodyResponse), ),)] pub async fn opportunity_bid( State(store): State>, diff --git a/auction-server/src/auction.rs b/auction-server/src/auction.rs index 3f35324c..f5937928 100644 --- a/auction-server/src/auction.rs +++ b/auction-server/src/auction.rs @@ -1,7 +1,10 @@ use { crate::{ api::RestError, - config::EthereumConfig, + config::{ + ChainId, + EthereumConfig, + }, server::{ EXIT_CHECK_INTERVAL, SHOULD_EXIT, @@ -10,6 +13,7 @@ use { BidAmount, BidStatus, BidStatusWithId, + PermissionKey, SimulatedBid, Store, }, @@ -58,6 +62,7 @@ use { Serialize, }, std::{ + collections::HashMap, result, sync::{ atomic::Ordering, @@ -117,13 +122,11 @@ pub fn get_simulation_call( .from(relayer) } - pub enum SimulationError { LogicalError { result: Bytes, reason: String }, ContractError(ContractError>), } - pub fn evaluate_simulation_results(results: Vec) -> Result<(), SimulationError> { let failed_result = results.iter().find(|x| !x.external_success); if let Some(call_status) = failed_result { @@ -134,6 +137,7 @@ pub fn evaluate_simulation_results(results: Vec) -> Result<(), } Ok(()) } + pub async fn simulate_bids( relayer: Address, provider: Provider, @@ -222,14 +226,20 @@ pub async fn run_submission_loop(store: Arc) -> Result<()> { tokio::select! { _ = submission_interval.tick() => { for (chain_id, chain_store) in &store.chains { - let permission_bids = chain_store.bids.read().await.clone(); - // release lock asap + let all_bids = store.get_bids_by_chain_id(chain_id).await; + let bid_by_permission_key:HashMap> = + all_bids.into_iter().fold(HashMap::new(), + |mut acc, bid| { + acc.entry(bid.permission_key.clone()).or_default().push(bid); + acc + }); + tracing::info!( "Chain: {chain_id} Auctions to process {auction_len}", chain_id = chain_id, - auction_len = permission_bids.len() + auction_len = bid_by_permission_key.len() ); - for (permission_key, bids) in permission_bids.iter() { + for (permission_key, bids) in bid_by_permission_key.iter() { let mut cloned_bids = bids.clone(); let permission_key = permission_key.clone(); cloned_bids.sort_by(|a, b| b.bid_amount.cmp(&a.bid_amount)); @@ -256,9 +266,8 @@ pub async fn run_submission_loop(store: Arc) -> Result<()> { true => BidStatus::Submitted(receipt.transaction_hash), false => BidStatus::Lost }; - store.bid_status_store.set_and_broadcast(BidStatusWithId { id: bid.id, bid_status }).await; + store.broadcast_bid_status_and_remove(BidStatusWithId { id: bid.id, bid_status }).await?; } - chain_store.bids.write().await.remove(&permission_key); } None => { tracing::error!("Failed to receive transaction receipt"); @@ -282,19 +291,19 @@ pub async fn run_submission_loop(store: Arc) -> Result<()> { #[derive(Serialize, Deserialize, ToSchema, Clone, Debug)] pub struct Bid { /// The permission key to bid on. - #[schema(example = "0xdeadbeef", value_type=String)] + #[schema(example = "0xdeadbeef", value_type = String)] pub permission_key: Bytes, /// The chain id to bid on. - #[schema(example = "sepolia", value_type=String)] - pub chain_id: String, + #[schema(example = "sepolia", value_type = String)] + pub chain_id: ChainId, /// The contract address to call. - #[schema(example = "0xcA11bde05977b3631167028862bE2a173976CA11",value_type = String)] + #[schema(example = "0xcA11bde05977b3631167028862bE2a173976CA11", value_type = String)] pub target_contract: abi::Address, /// Calldata for the contract call. - #[schema(example = "0xdeadbeef", value_type=String)] + #[schema(example = "0xdeadbeef", value_type = String)] pub target_calldata: Bytes, /// Amount of bid in wei. - #[schema(example = "10", value_type=String)] + #[schema(example = "10", value_type = String)] #[serde(with = "crate::serde::u256")] pub amount: BidAmount, } @@ -335,24 +344,15 @@ pub async fn handle_bid(store: Arc, bid: Bid) -> result::Result) -> Result<()> { while !SHOULD_EXIT.load(Ordering::Acquire) { tokio::select! { _ = submission_interval.tick() => { - let all_opportunities = store.opportunity_store.opportunities.clone(); - for item in all_opportunities.iter() { + let all_opportunities = store.opportunity_store.opportunities.read().await.clone(); + for (_permission_key,opportunities) in all_opportunities.iter() { // check each of the opportunities for this permission key for validity - let mut opps_to_remove = vec![]; - for opportunity in item.value().iter() { + for opportunity in opportunities.iter() { match verify_with_store(opportunity.clone(), &store).await { Ok(_) => {} Err(e) => { - opps_to_remove.push(opportunity.id); tracing::info!( "Removing Opportunity {} with failed verification: {}", opportunity.id, e ); + match store.remove_opportunity(opportunity).await { + Ok(_) => {} + Err(e) => { + tracing::error!("Failed to remove opportunity: {}", e); + } + } } } } - let permission_key = item.key(); - let opportunities_map = &store.opportunity_store.opportunities; - if let Some(mut opportunities) = opportunities_map.get_mut(permission_key) { - opportunities.retain(|x| !opps_to_remove.contains(&x.id)); - if opportunities.is_empty() { - drop(opportunities); - opportunities_map.remove(permission_key); - } - } } } _ = exit_check_interval.tick() => { @@ -416,6 +410,8 @@ pub async fn handle_opportunity_bid( let opportunities = store .opportunity_store .opportunities + .read() + .await .get(&opportunity_bid.permission_key) .ok_or(RestError::OpportunityNotFound)? .clone(); @@ -425,13 +421,6 @@ pub async fn handle_opportunity_bid( .find(|o| o.id == opportunity_id) .ok_or(RestError::OpportunityNotFound)?; - // TODO: move this logic to searcher side - if opportunity.bidders.contains(&opportunity_bid.executor) { - return Err(RestError::BadParameters( - "Executor already bid on this opportunity".to_string(), - )); - } - let OpportunityParams::V1(params) = &opportunity.params; let chain_store = store @@ -459,20 +448,7 @@ pub async fn handle_opportunity_bid( ) .await { - Ok(id) => { - let opportunities = store - .opportunity_store - .opportunities - .get_mut(&opportunity_bid.permission_key); - if let Some(mut opportunities) = opportunities { - let opportunity = opportunities - .iter_mut() - .find(|o| o.id == opportunity_id) - .ok_or(RestError::OpportunityNotFound)?; - opportunity.bidders.insert(opportunity_bid.executor); - } - Ok(id) - } + Ok(id) => Ok(id), Err(e) => match e { RestError::SimulationError { result, reason } => { let parsed = parse_revert_error(&result); diff --git a/auction-server/src/server.rs b/auction-server/src/server.rs index e3a76b41..e602f314 100644 --- a/auction-server/src/server.rs +++ b/auction-server/src/server.rs @@ -10,7 +10,6 @@ use { }, opportunity_adapter::run_verification_loop, state::{ - BidStatusStore, ChainStore, OpportunityStore, Store, @@ -29,6 +28,7 @@ use { signers::Signer, }, futures::future::join_all, + sqlx::postgres::PgPoolOptions, std::{ collections::HashMap, sync::{ @@ -43,6 +43,7 @@ use { }, }; + const NOTIFICATIONS_CHAN_LEN: usize = 1000; pub async fn start_server(run_options: RunOptions) -> anyhow::Result<()> { tokio::spawn(async move { @@ -52,7 +53,6 @@ pub async fn start_server(run_options: RunOptions) -> anyhow::Result<()> { SHOULD_EXIT.store(true, Ordering::Release); }); - let config = Config::load(&run_options.config.config).map_err(|err| { anyhow!( "Failed to load config from file({path}): {:?}", @@ -85,7 +85,6 @@ pub async fn start_server(run_options: RunOptions) -> anyhow::Result<()> { ChainStore { provider, network_id: id, - bids: Default::default(), token_spoof_info: Default::default(), config: chain_config.clone(), }, @@ -98,13 +97,18 @@ pub async fn start_server(run_options: RunOptions) -> anyhow::Result<()> { let (broadcast_sender, broadcast_receiver) = tokio::sync::broadcast::channel(NOTIFICATIONS_CHAN_LEN); + + let pool = PgPoolOptions::new() + .max_connections(10) + .connect(&run_options.server.database_url) + .await + .expect("Server should start with a valid database connection."); let store = Arc::new(Store { + db: pool, + bids: Default::default(), chains: chain_store?, opportunity_store: OpportunityStore::default(), - bid_status_store: BidStatusStore { - bids_status: Default::default(), - event_sender: broadcast_sender.clone(), - }, + event_sender: broadcast_sender.clone(), relayer: wallet, ws: ws::WsState { subscriber_counter: AtomicUsize::new(0), @@ -113,7 +117,6 @@ pub async fn start_server(run_options: RunOptions) -> anyhow::Result<()> { }, }); - let submission_loop = tokio::spawn(run_submission_loop(store.clone())); let verification_loop = tokio::spawn(run_verification_loop(store.clone())); let server_loop = tokio::spawn(api::start_api(run_options, store.clone())); diff --git a/auction-server/src/state.rs b/auction-server/src/state.rs index f9ecae19..0af3b301 100644 --- a/auction-server/src/state.rs +++ b/auction-server/src/state.rs @@ -1,15 +1,17 @@ use { crate::{ - api::ws::{ - UpdateEvent, - WsState, + api::{ + ws::{ + UpdateEvent, + WsState, + }, + RestError, }, config::{ ChainId, EthereumConfig, }, }, - dashmap::DashMap, ethers::{ providers::{ Http, @@ -27,9 +29,22 @@ use { Deserialize, Serialize, }, - std::collections::{ - HashMap, - HashSet, + sqlx::{ + database::HasArguments, + encode::IsNull, + types::{ + time::{ + OffsetDateTime, + PrimitiveDateTime, + }, + BigDecimal, + }, + Postgres, + TypeInfo, + }, + std::{ + collections::HashMap, + str::FromStr, }, tokio::sync::{ broadcast, @@ -51,6 +66,9 @@ pub struct SimulatedBid { pub target_contract: Address, pub target_calldata: Bytes, pub bid_amount: BidAmount, + pub permission_key: PermissionKey, + pub chain_id: ChainId, + pub status: BidStatus, // simulation_time: } @@ -59,10 +77,10 @@ pub type UnixTimestamp = i64; #[derive(Serialize, Deserialize, ToSchema, Clone, PartialEq)] pub struct TokenAmount { /// Token contract address - #[schema(example = "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2",value_type=String)] + #[schema(example = "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", value_type = String)] pub token: ethers::abi::Address, /// Token amount - #[schema(example = "1000", value_type=String)] + #[schema(example = "1000", value_type = String)] #[serde(with = "crate::serde::u256")] pub amount: U256, } @@ -74,19 +92,19 @@ pub struct TokenAmount { #[derive(Serialize, Deserialize, ToSchema, Clone, PartialEq)] pub struct OpportunityParamsV1 { /// The permission key required for successful execution of the opportunity. - #[schema(example = "0xdeadbeefcafe", value_type=String)] + #[schema(example = "0xdeadbeefcafe", value_type = String)] pub permission_key: Bytes, /// The chain id where the opportunity will be executed. - #[schema(example = "sepolia", value_type=String)] + #[schema(example = "sepolia", value_type = String)] pub chain_id: ChainId, /// The contract address to call for execution of the opportunity. - #[schema(example = "0xcA11bde05977b3631167028862bE2a173976CA11", value_type=String)] + #[schema(example = "0xcA11bde05977b3631167028862bE2a173976CA11", value_type = String)] pub target_contract: ethers::abi::Address, /// Calldata for the target contract call. - #[schema(example = "0xdeadbeef", value_type=String)] + #[schema(example = "0xdeadbeef", value_type = String)] pub target_calldata: Bytes, /// The value to send with the contract call. - #[schema(example = "1", value_type=String)] + #[schema(example = "1", value_type = String)] #[serde(with = "crate::serde::u256")] pub target_call_value: U256, @@ -102,12 +120,12 @@ pub enum OpportunityParams { } pub type OpportunityId = Uuid; + #[derive(Clone, PartialEq)] pub struct Opportunity { pub id: OpportunityId, pub creation_time: UnixTimestamp, pub params: OpportunityParams, - pub bidders: HashSet
, } #[derive(Clone)] @@ -124,12 +142,25 @@ pub struct ChainStore { pub network_id: u64, pub config: EthereumConfig, pub token_spoof_info: RwLock>, - pub bids: RwLock>>, } #[derive(Default)] pub struct OpportunityStore { - pub opportunities: DashMap>, + pub opportunities: RwLock>>, +} + +impl OpportunityStore { + pub async fn add_opportunity(&self, opportunity: Opportunity) { + let key = match &opportunity.params { + OpportunityParams::V1(params) => params.permission_key.clone(), + }; + self.opportunities + .write() + .await + .entry(key) + .or_insert_with(Vec::new) + .push(opportunity); + } } pub type BidId = Uuid; @@ -140,12 +171,33 @@ pub enum BidStatus { /// The auction for this bid is pending Pending, /// The bid won the auction and was submitted to the chain in a transaction with the given hash - #[schema(example = "0x103d4fbd777a36311b5161f2062490f761f25b67406badb2bace62bb170aa4e3", value_type=String)] + #[schema(example = "0x103d4fbd777a36311b5161f2062490f761f25b67406badb2bace62bb170aa4e3", value_type = String)] Submitted(H256), /// The bid lost the auction Lost, } +impl sqlx::Encode<'_, sqlx::Postgres> for BidStatus { + fn encode_by_ref(&self, buf: &mut >::ArgumentBuffer) -> IsNull { + let result = match self { + BidStatus::Pending => "pending", + BidStatus::Submitted(_) => "submitted", + BidStatus::Lost => "lost", + }; + <&str as sqlx::Encode>::encode(result, buf) + } +} + +impl sqlx::Type for BidStatus { + fn type_info() -> sqlx::postgres::PgTypeInfo { + sqlx::postgres::PgTypeInfo::with_name("bid_status") + } + + fn compatible(ty: &sqlx::postgres::PgTypeInfo) -> bool { + ty.name() == "bid_status" + } +} + #[derive(Serialize, Clone, ToSchema, ToResponse)] pub struct BidStatusWithId { #[schema(value_type = String)] @@ -153,32 +205,151 @@ pub struct BidStatusWithId { pub bid_status: BidStatus, } -pub struct BidStatusStore { - pub bids_status: RwLock>, - pub event_sender: broadcast::Sender, +pub struct Store { + pub chains: HashMap, + pub bids: RwLock>, + pub event_sender: broadcast::Sender, + pub opportunity_store: OpportunityStore, + pub relayer: LocalWallet, + pub ws: WsState, + pub db: sqlx::PgPool, } -impl BidStatusStore { - pub async fn get_status(&self, id: &BidId) -> Option { - self.bids_status.read().await.get(id).cloned() +impl Store { + pub async fn opportunity_exists(&self, opportunity: &Opportunity) -> bool { + let key = match &opportunity.params { + OpportunityParams::V1(params) => params.permission_key.clone(), + }; + self.opportunity_store + .opportunities + .read() + .await + .get(&key) + .map_or(false, |opps| opps.contains(opportunity)) } - - pub async fn set_and_broadcast(&self, update: BidStatusWithId) { - self.bids_status - .write() + pub async fn add_opportunity(&self, opportunity: Opportunity) -> Result<(), RestError> { + let odt = OffsetDateTime::from_unix_timestamp(opportunity.creation_time) + .expect("creation_time is valid"); + let OpportunityParams::V1(params) = &opportunity.params; + sqlx::query!("INSERT INTO opportunity (id, + creation_time, + permission_key, + chain_id, + target_contract, + target_call_value, + target_calldata, + sell_tokens, + buy_tokens) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + opportunity.id, + PrimitiveDateTime::new(odt.date(), odt.time()), + params.permission_key.to_vec(), + params.chain_id, + ¶ms.target_contract.to_fixed_bytes(), + BigDecimal::from_str(¶ms.target_call_value.to_string()).unwrap(), + params.target_calldata.to_vec(), + serde_json::to_value(¶ms.sell_tokens).unwrap(), + serde_json::to_value(¶ms.buy_tokens).unwrap()) + .execute(&self.db) .await - .insert(update.id, update.bid_status.clone()); + .map_err(|e| { + tracing::error!("DB: Failed to insert opportunity: {}", e); + RestError::TemporarilyUnavailable + })?; + self.opportunity_store.add_opportunity(opportunity).await; + Ok(()) + } + + pub async fn remove_opportunity(&self, opportunity: &Opportunity) -> anyhow::Result<()> { + let key = match &opportunity.params { + OpportunityParams::V1(params) => params.permission_key.clone(), + }; + let mut write_guard = self.opportunity_store.opportunities.write().await; + let entry = write_guard.entry(key.clone()); + if entry + .and_modify(|opps| opps.retain(|o| o != opportunity)) + .or_default() + .is_empty() + { + write_guard.remove(&key); + } + drop(write_guard); + let now = OffsetDateTime::now_utc(); + sqlx::query!( + "UPDATE opportunity SET removal_time = $1 WHERE id = $2 AND removal_time IS NULL", + PrimitiveDateTime::new(now.date(), now.time()), + opportunity.id + ) + .execute(&self.db) + .await?; + Ok(()) + } + + pub async fn add_bid(&self, bid: SimulatedBid) -> Result<(), RestError> { + let bid_id = bid.id; + let now = OffsetDateTime::now_utc(); + sqlx::query!("INSERT INTO bid (id, creation_time, permission_key, chain_id, target_contract, target_calldata, bid_amount, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + bid.id, + PrimitiveDateTime::new(now.date(), now.time()), + bid.permission_key.to_vec(), + bid.chain_id, + &bid.target_contract.to_fixed_bytes(), + bid.target_calldata.to_vec(), + BigDecimal::from_str(&bid.bid_amount.to_string()).unwrap(), + bid.status as _, + ) + .execute(&self.db) + .await.map_err(|e| { + tracing::error!("DB: Failed to insert bid: {}", e); + RestError::TemporarilyUnavailable + })?; + + self.bids.write().await.insert(bid_id, bid.clone()); + self.broadcast_status_update(BidStatusWithId { + id: bid_id, + bid_status: bid.status.clone(), + }); + Ok(()) + } + + pub async fn broadcast_bid_status_and_remove( + &self, + update: BidStatusWithId, + ) -> anyhow::Result<()> { + if update.bid_status == BidStatus::Pending { + return Err(anyhow::anyhow!( + "Bid status cannot remain pending when removing a bid." + )); + } + + let now = OffsetDateTime::now_utc(); + sqlx::query!( + "UPDATE bid SET status = $1, removal_time = $2 WHERE id = $3 AND removal_time IS NULL", + update.bid_status as _, + PrimitiveDateTime::new(now.date(), now.time()), + update.id + ) + .execute(&self.db) + .await?; + + self.bids.write().await.remove(&update.id); + self.broadcast_status_update(update); + Ok(()) + } + + fn broadcast_status_update(&self, update: BidStatusWithId) { match self.event_sender.send(UpdateEvent::BidStatusUpdate(update)) { Ok(_) => (), Err(e) => tracing::error!("Failed to send bid status update: {}", e), }; } -} -pub struct Store { - pub chains: HashMap, - pub bid_status_store: BidStatusStore, - pub opportunity_store: OpportunityStore, - pub relayer: LocalWallet, - pub ws: WsState, + pub async fn get_bids_by_chain_id(&self, chain_id: &ChainId) -> Vec { + self.bids + .read() + .await + .values() + .filter(|bid| bid.chain_id.eq(chain_id)) + .cloned() + .collect() + } } diff --git a/auction-server/src/token_spoof.rs b/auction-server/src/token_spoof.rs index cd574d59..0dc25cb5 100644 --- a/auction-server/src/token_spoof.rs +++ b/auction-server/src/token_spoof.rs @@ -45,7 +45,6 @@ pub fn calculate_balance_storage_key(owner: Address, balance_slot: U256) -> H256 keccak256(Bytes::from(buffer)).into() } - /// Calculate the storage key for the allowance of an spender for an address in an ERC20 token. /// This is used to spoof the allowance /// @@ -70,7 +69,6 @@ pub fn calculate_allowance_storage_key( keccak256(Bytes::from(buffer_allowance)).into() } - const MAX_SLOT_FOR_BRUTEFORCE: i32 = 32; /// Find the balance slot of an ERC20 token that can be used to spoof the balance of an address @@ -103,7 +101,6 @@ async fn find_spoof_balance_slot( Err(anyhow!("Could not find balance slot")) } - /// Find the allowance slot of an ERC20 token that can be used to spoof the allowance of an address /// Returns an error if no slot is found or if the network calls fail ///