From ee81d6af9e06762e2300f7c42021aad140067273 Mon Sep 17 00:00:00 2001 From: Leonardo L Date: Thu, 11 Aug 2022 15:03:10 -0300 Subject: [PATCH] feature: add `block_headers` and `blocks` streams full APIs (w/ checkpoint and reorg handling) (#10) * feature: implement an initial mempool.space websocket client, and library for block-events from #3 * feature: add unit and integration tests to block events from #8 * feat+test: add initial http client and tests * wip(feat): add initial approach for subscribing to blocks from a starting height * wip(feat): fix fn and use a tuple as return type for prev and new block streams * wip(refactor): initial updates for fns refactor and architecture change * wip(feat+refactor): add fn to process candidate BlockHeaders, handle reorg and yield BlockEvent * wip(refactor+docs): extract fns to cache struct, and add documentation for fns, enums and structs * wip(fixes+tests): fixes, improve error handling and add new integration tests * wip(fix+test): fix common ancestor and fork branch bug, add new reorg tests * fix: disconnected and connected block events emitting order for reorg events * chore: simplify cli usage and output * chore: update docs and readme examples, use u32 instead of u64 * fix: remaining cargo.toml conflicts from other branches * feat: add full block stream api, better error handling - use TryStream instead of stream, returning result instead - use `?` operator instead of unwraping and panicking - add new mempool.space endpoints for http client - add features for wss:// and https:// usage - add features for api versioning based on mempool.space backend * refactor: do not use features expect two base_url instead, and improve error handling - refactor and update http client, url usage and methods - refactor and add creation methods for cache struct, use specific method to build initial one - add `get_block_header` method in http client, and refactor lib to use `bitcoin::BlockHeader` instead of custom mempool.space `BlockExtended` struct - refactor lib and websocket to improve error handling, update some `unwrap()` usage and match errors for messages in `WebSocketStream` - remove duplicated/unused deps, and use only necessary features * chore: add and update CHANGELOG.md file * fix(test): docs and integration tests - bump mempool/backend docker container version to v2.4.1 - update api+websocket client to handle genesis blocks without prev_blockhash - update and fix integration tests to new fns and structure --- .github/pull_request_template.md | 30 + CHANGELOG.md | 13 + Cargo.lock | 1292 +++++++++++++++++++++++++++++- Cargo.toml | 33 +- README.md | 104 ++- src/api.rs | 97 +++ src/bin.rs | 83 ++ src/http.rs | 151 ++++ src/lib.rs | 374 +++++++++ src/main.rs | 133 --- src/websocket.rs | 124 +++ tests/integration_tests.rs | 527 ++++++++++++ 12 files changed, 2751 insertions(+), 210 deletions(-) create mode 100644 .github/pull_request_template.md create mode 100644 CHANGELOG.md create mode 100644 src/api.rs create mode 100644 src/bin.rs create mode 100644 src/http.rs create mode 100644 src/lib.rs delete mode 100644 src/main.rs create mode 100644 src/websocket.rs create mode 100644 tests/integration_tests.rs diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..f6446e2 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,30 @@ + + +### Description + + + +### Notes to the reviewers + + + +### Checklists + +#### All Submissions: + +* [ ] I've signed all my commits +* [ ] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md) +* [ ] I ran `cargo fmt` and `cargo clippy` before committing + +#### New Features: + +* [ ] I've added tests for the new feature +* [ ] I've added docs for the new feature +* [ ] I've updated `CHANGELOG.md` + +#### Bugfixes: + +* [ ] This pull request breaks the existing API +* [ ] I've added tests to reproduce the issue which are now passing +* [ ] I'm linking the issue being fixed by this PR \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4f0e35d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] +- Add `block_events::websocket` client, and a `websocket::listen_new_block_headers` fn to get a stream of websocket messages. +- Add `block_events::http` client, and multiple fns to consume data from mempool.space and/or esplora REST APIs. +- Add `block_events::api` with multiple structs and definition for responses, such as: `BlockExtended` and events as `BlockEvent` +- Add `block_events::subscribe_to_block_headers` fn in order to return a `Stream` of `block_events::api::BlockEvent(bitcoin::BlockHeader)`. +- Add `block_events::process_blocks` to handle block reorganization, manage in-memory cache struct `BlockHeadersCache`, and propagate errors to caller. +- Add `block_events::subscribe_to_blocks` fn in order to return a `Stream` of `block_events::api::BlockEvent(bitcoin::Block)`. diff --git a/Cargo.lock b/Cargo.lock index ebe02d9..ca1b539 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,48 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + [[package]] name = "anyhow" version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -25,12 +61,97 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base-x" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc19a4937b4fbd3fe3379793130e42060d10627a360f2127802b10b87e7baf74" + [[package]] name = "base64" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "base64-compat" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a8d4d2746f89841e49230dd26917df1876050f95abafafbe34f47cb534b88d7" +dependencies = [ + "byteorder", +] + +[[package]] +name = "bech32" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9ff0bbfd639f15c74af777d81383cf53efb7c93613f6cab67c6c11e05bbf8b" + +[[package]] +name = "bitcoin" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05bba324e6baf655b882df672453dbbc527bc938cadd27750ae510aaccc3a66a" +dependencies = [ + "base64-compat", + "bech32", + "bitcoin_hashes", + "secp256k1", + "serde", +] + +[[package]] +name = "bitcoin_hashes" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "006cc91e1a1d99819bc5b8214be3555c1f0611b169f527a1fdc54ed1f2b745b0" +dependencies = [ + "serde", +] + +[[package]] +name = "bitcoincore-rpc" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0e67dbf7a9971e7f4276f6089e9e814ce0f624a03216b7d92d00351ae7fb3e" +dependencies = [ + "bitcoincore-rpc-json", + "jsonrpc", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "bitcoincore-rpc-json" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e2ae16202721ba8c3409045681fac790a5ddc791f05731a2df22c0c6bffc0f1" +dependencies = [ + "bitcoin", + "serde", + "serde_json", +] + +[[package]] +name = "bitcoind" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0831b9721892ce845a6acadd111311bee84f9e1cc0c5017b8213ec4437ccdfe2" +dependencies = [ + "bitcoin_hashes", + "bitcoincore-rpc", + "filetime", + "flate2", + "home", + "log", + "tar", + "tempfile", + "ureq", + "which", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -47,20 +168,44 @@ dependencies = [ ] [[package]] -name = "block-explorer-cli" +name = "block-events" version = "0.1.0" dependencies = [ "anyhow", + "async-stream", + "bitcoin", + "bitcoind", "clap", - "futures-util", + "env_logger", + "futures", "log", + "reqwest", "serde", "serde_json", + "serial_test", + "testcontainers", "tokio", "tokio-tungstenite", "url", ] +[[package]] +name = "bollard-stubs" +version = "1.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2f2e73fffe9455141e170fb9c1feb0ac521ec7e7dcd47a7cab72a658490fb8" +dependencies = [ + "chrono", + "serde", + "serde_with", +] + +[[package]] +name = "bumpalo" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" + [[package]] name = "byteorder" version = "1.4.3" @@ -85,6 +230,26 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time 0.1.43", + "winapi", +] + +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + [[package]] name = "clap" version = "3.1.8" @@ -115,6 +280,39 @@ dependencies = [ "syn", ] +[[package]] +name = "const_fn" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" + +[[package]] +name = "cookie" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" +dependencies = [ + "percent-encoding", + "time 0.2.27", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3" +dependencies = [ + "cookie", + "idna", + "log", + "publicsuffix", + "serde", + "serde_json", + "time 0.2.27", + "url", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -140,6 +338,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.3" @@ -150,6 +357,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.10.3" @@ -158,6 +400,41 @@ checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ "block-buffer", "crypto-common", + "subtle", +] + +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_logger" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", ] [[package]] @@ -169,6 +446,30 @@ dependencies = [ "instant", ] +[[package]] +name = "filetime" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi", +] + +[[package]] +name = "flate2" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -200,12 +501,65 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +dependencies = [ + "futures-core", + "futures-sink", +] + [[package]] name = "futures-core" version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" +[[package]] +name = "futures-executor" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" + +[[package]] +name = "futures-macro" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.21" @@ -224,9 +578,13 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -253,6 +611,25 @@ dependencies = [ "wasi 0.10.2+wasi-snapshot-preview1", ] +[[package]] +name = "h2" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.11.2" @@ -263,33 +640,123 @@ checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" name = "heck" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2456aef2e6b6a9784192ae780c0f15bc57df0e918585282325e8c8ac27737654" +dependencies = [ + "winapi", +] + +[[package]] +name = "http" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6330e8a36bd8c859f3fa6d9382911fbb7147ec39807f63b923933a247240b9ba" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "hyper" +version = "0.14.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" dependencies = [ - "libc", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", ] [[package]] -name = "http" -version = "0.2.6" +name = "hyper-tls" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "fnv", - "itoa", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", ] [[package]] -name = "httparse" -version = "1.7.0" +name = "ident_case" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6330e8a36bd8c859f3fa6d9382911fbb7147ec39807f63b923933a247240b9ba" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" @@ -321,12 +788,39 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + [[package]] name = "itoa" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "jsonrpc" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8423b78fc94d12ef1a4a9d13c348c9a78766dda0cc18817adf0faf77e670c8" +dependencies = [ + "base64-compat", + "serde", + "serde_derive", + "serde_json", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -339,6 +833,16 @@ version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb691a747a7ab48abc15c5b42066eaafde10dc427e3b6ee2a1cf43db04c763bd" +[[package]] +name = "lock_api" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.16" @@ -360,6 +864,22 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.8.2" @@ -410,6 +930,25 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.13.1" @@ -422,9 +961,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.10.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" [[package]] name = "openssl" @@ -468,6 +1007,29 @@ dependencies = [ "memchr", ] +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + [[package]] name = "percent-encoding" version = "2.1.0" @@ -522,6 +1084,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.37" @@ -531,6 +1099,25 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "publicsuffix" +version = "1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f" +dependencies = [ + "idna", + "url", +] + +[[package]] +name = "qstring" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d464fae65fff2680baf48019211ce37aaec0c78e9264c84a3e484717f965104e" +dependencies = [ + "percent-encoding", +] + [[package]] name = "quote" version = "1.0.18" @@ -579,6 +1166,23 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -588,6 +1192,86 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustversion" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24c8ad4f0c00e1eb5bc7614d236a7f1300e3dbd76b68cac8e06fb00b015ad8d8" + [[package]] name = "ryu" version = "1.0.9" @@ -604,6 +1288,41 @@ dependencies = [ "winapi", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "secp256k1" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26947345339603ae8395f68e2f3d85a6b0a8ddfe6315818e80b8504415099db0" +dependencies = [ + "secp256k1-sys", + "serde", +] + +[[package]] +name = "secp256k1-sys" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152e20a0fd0519390fc43ab404663af8a0b794273d2a91d60ad4a39f13ffe110" +dependencies = [ + "cc", +] + [[package]] name = "security-framework" version = "2.6.1" @@ -627,6 +1346,21 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "1.0.136" @@ -637,32 +1371,117 @@ dependencies = [ ] [[package]] -name = "serde_derive" -version = "1.0.136" +name = "serde_derive" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" +dependencies = [ + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serial_test" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d19dbfb999a147cedbfe82f042eb9555f5b0fa4ef95ee4570b74349103d9c9f4" +dependencies = [ + "lazy_static", + "log", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb9e2050b2be1d681f8f1c1a528bcfe4e00afa2d8995f713974f5333288659f2" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "sha-1" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" dependencies = [ - "proc-macro2", - "quote", - "syn", + "sha1_smol", ] [[package]] -name = "serde_json" -version = "1.0.79" +name = "sha1_smol" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" -dependencies = [ - "itoa", - "ryu", - "serde", -] +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" [[package]] -name = "sha-1" -version = "0.10.0" +name = "sha2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" dependencies = [ "cfg-if", "cpufeatures", @@ -675,6 +1494,12 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" +[[package]] +name = "smallvec" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" + [[package]] name = "socket2" version = "0.4.4" @@ -685,12 +1510,82 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "standback" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" +dependencies = [ + "version_check", +] + +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + [[package]] name = "syn" version = "1.0.91" @@ -702,6 +1597,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -725,6 +1631,23 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "testcontainers" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e2b1567ca8a2b819ea7b28c92be35d9f76fb9edb214321dcc86eb96023d1f87" +dependencies = [ + "bollard-stubs", + "futures", + "hex", + "hmac", + "log", + "rand", + "serde", + "serde_json", + "sha2", +] + [[package]] name = "textwrap" version = "0.15.0" @@ -751,6 +1674,54 @@ dependencies = [ "syn", ] +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "time" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + [[package]] name = "tinyvec" version = "1.5.1" @@ -768,15 +1739,16 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.17.0" +version = "1.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" dependencies = [ "bytes", "libc", "memchr", "mio", "num_cpus", + "once_cell", "pin-project-lite", "socket2", "tokio-macros", @@ -818,6 +1790,52 @@ dependencies = [ "tungstenite", ] +[[package]] +name = "tokio-util" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b7358be39f2f274f322d2aaed611acc57f382e8eb1e5b48cb9ae30933495ce7" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "tungstenite" version = "0.17.2" @@ -865,6 +1883,31 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8b063c2d59218ae09f22b53c42eaad0d53516457905f5235ca4bc9e99daa71" +dependencies = [ + "base64", + "chunked_transfer", + "cookie", + "cookie_store", + "log", + "once_cell", + "qstring", + "rustls", + "url", + "webpki", + "webpki-roots", +] + [[package]] name = "url" version = "2.2.2" @@ -895,6 +1938,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" @@ -907,6 +1960,112 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + +[[package]] +name = "web-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" +dependencies = [ + "webpki", +] + +[[package]] +name = "which" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" +dependencies = [ + "either", + "lazy_static", + "libc", +] + [[package]] name = "winapi" version = "0.3.9" @@ -937,3 +2096,64 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index d3d9888..81686db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,17 +1,38 @@ [package] -name = "block-explorer-cli" +name = "block-events" version = "0.1.0" edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +authors = ["Leonardo Souza ", "LLFourn "] +repository = "https://github.com/oleonardolima/block-events" +description = "A real-time stream block events library, covering connected and disconnected blocks.\nThis a work in progress project for Summer of Bitcoin 2022." +keywords = ["bitcoin", "blockchain", "blocks", "events", "mempool-space", "stream", "summer-of-bitcoin"] +readme = "README.md" +license = "MIT OR Apache-2.0" [dependencies] anyhow = { version = "1.0" } +async-stream = { version = "0.3.3"} +bitcoin = { version = "0.28", features = ["use-serde", "base64"] } clap = { version = "3.0", features = ["derive"]} -futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] } +env_logger = { version = "0.9.0" } +futures = { version = "0.3" } log = { version = "0.4" } -serde = { version = "1.0.117", features = ["derive"] } +reqwest = { version = "0.11.11" } +serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } -tokio = { version = "1.0.0", features = ["io-util", "io-std", "macros", "net", "rt-multi-thread", "time"] } +tokio = { version = "1.19.2", features = ["macros", "rt-multi-thread"] } tokio-tungstenite = { version = "0.17.1", features = ["connect", "native-tls"]} url = { version = "2.0.0" } + +[dev-dependencies] +testcontainers = { version = "^0.14.0" } +bitcoind = { version = "^0.26.1", features = ["22_0"] } +serial_test = { version = "0.7.0" } + +[lib] +name = "block_events" +path = "src/lib.rs" + +[[bin]] +name = "block-events-cli" +path = "src/bin.rs" diff --git a/README.md b/README.md index 7f025a3..a23f8c7 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,24 @@ -# A terminal block explorer for mempool.space websocket - -A terminal block explorer exposing the features available on [mempool.space websocket API](https://mempool.space/docs/api/websocket). - -Currently available features are: -- All data feed from mempool.space for: blocks, mempool-blocks, live-2h-chart, and stats. -- Subscription for address related data: track-address. +# Real-time stream of block events library + +A library for consuming and subscribing to new block events in real-time from different sources: + - [ ] mempool.space - [WebSocket](https://mempool.space/docs/api/websocket) and [REST](https://mempool.space/docs/api/rest) APIs (under development) + - [ ] bitcoin core RPC `#TODO` + - [ ] bitcoin P2P `#TODO` + +It's useful for projects to get notified for connected and disconnected new blocks, currently using the following as output in async manner: +``` rust +pub enum BlockEvent { + Connected(BlockExtended), + Disconnected((u32, BlockHash)), + Error(), +} +``` -
+Can be also used through command-line interface (CLI), as `block-explorer-cli` and just passing the network: Regtest, Signet, Testnet and Mainnet. +> **NOTE**: The previous implemented track-address feature and other data, such as: mempool-blocks, stats... has been deprecated and it's not refactored yet. ## Requirements: - -To use this CLI you must have rust and cargo installed in your computer, you can check it with: +To use the library as CLI or to contribute you must have rust and cargo installed in your computer, you can check it with: ``` sh # check rust version, it should return its version if installed @@ -18,35 +26,61 @@ rustc --version # check cargo version, it should return its version if installed cargo --version ``` - If you do not have it installed, you can follow this tutorial from [The Rust Programming Language book](https://doc.rust-lang.org/book/ch01-01-installation.html) - -
- -## How to use: - -If you have cargo and rust installed, you can run the following commands: - +## Compiling and using the CLI: +To compile and use it as a command in terminal with no need of cargo, you can use the following command: ``` sh -# mainnet connection is default -cargo run -- track-address -a - -# to use testnet -cargo run -- --endpoint mempool.space/testnet/api track-address -a +# from inside this repo +cargo install --path . ``` - +## Examples: +### Consuming new block events through the CLI: ``` sh -# all feed [blocks, mempool-blocks, live-2h-chart, and stats] for mainnet: -cargo run -- blocks-data - -# or all feed [blocks, mempool-blocks, live-2h-chart, and stats] for testnet: -cargo run -- --endpoint mempool.space/testnet/api blocks-data -``` +# testnet connection is set by default +cargo run -- data-stream --blocks -## Compiling and using: -To compile and use it as a command in terminal with no need of cargo, you can use the following command: - -``` sh -cargo install --path . +# to use regtest, you need to pass it as a parameter +cargo run -- --base-url localhost:8999/testnet/api/v1 data-stream --blocks ``` +### Subscribing and consuming new block events through the lib: +``` rust +use anyhow::{self, Ok}; +use futures::{pin_mut, StreamExt}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + env_logger::init(); + + // for mempool.space testnet network + let http_base_url = "http://mempool.space/testnet/api/"; + let ws_base_url = "wss://mempool.space/testnet"; + + // no checkpoint for this example, but you could use the following one to test it by yourself (in mainnet). + // checkpoint for first BDK Taproot transaction on mainnet (base_url update needed) + // let checkpoint = (709635, bitcoin::BlockHash::from("00000000000000000001f9ee4f69cbc75ce61db5178175c2ad021fe1df5bad8f")); + let checkpoint = None; + + // async fetch the block-events stream through the lib + let block_events = + block_events::subscribe_to_block_headers(http_base_url, ws_base_url, checkpoint).await?; + + // consume and execute your code (current only matching and printing) in async manner for each new block-event + pin_mut!(block_events); + while let Some(block_event) = block_events.next().await { + match block_event? { + block_events::api::BlockEvent::Connected(block_header) => { + println!("[connected][block_header] {:#?}", block_header); + } + block_events::api::BlockEvent::Disconnected((height, block_hash)) => { + println!( + "[disconnected][height: {:#?}][block_hash: {:#?}]", + height, block_hash + ); + } + } + } + Ok(()) +} + +``` \ No newline at end of file diff --git a/src/api.rs b/src/api.rs new file mode 100644 index 0000000..e43adcb --- /dev/null +++ b/src/api.rs @@ -0,0 +1,97 @@ +// Block Events Library +// Written in 2022 by Leonardo Lima <> and Lloyd Fournier <> +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +//! All structs from mempool.space API +//! Also contains the main [`BlockEvent`] + +use bitcoin::{Address, Block, BlockHash, BlockHeader, TxMerkleNode}; + +/// A structure that implements the equivalent `BlockExtended` type from mempool.space, +/// which is expected and parsed as response +#[derive(serde::Deserialize, Clone, Debug, Copy)] +pub struct BlockExtended { + pub id: BlockHash, + pub height: u32, + pub version: i32, + #[serde(alias = "previousblockhash")] + pub prev_blockhash: Option, // None for genesis block + pub merkle_root: TxMerkleNode, + #[serde(alias = "timestamp")] + pub time: u32, + pub bits: u32, + pub nonce: u32, +} + +impl From for BlockHeader { + fn from(extended: BlockExtended) -> Self { + BlockHeader { + version: extended.version, + prev_blockhash: extended.prev_blockhash.unwrap_or_default(), + merkle_root: extended.merkle_root, + time: extended.time, + bits: extended.bits, + nonce: extended.nonce, + } + } +} + +impl From for BlockExtended { + fn from(block: Block) -> Self { + BlockExtended { + id: block.block_hash(), + height: block + .bip34_block_height() + .expect("Given `bitcoin::Block` does not have height encoded as bip34") + as u32, + version: block.header.version, + prev_blockhash: Some(block.header.prev_blockhash), + merkle_root: block.header.merkle_root, + time: block.header.time, + bits: block.header.bits, + nonce: block.header.nonce, + } + } +} + +/// Structure that implements the standard mempool.space WebSocket client response message +#[derive(serde::Deserialize, Debug)] +pub struct MempoolSpaceWebSocketMessage { + pub block: BlockExtended, + // pub mempool_info: MempoolInfo, + // pub da: DifficultyAdjustment, + // pub fees: RecommendedFee, +} + +/// Structure that implements the standard fields for mempool.space WebSocket client message +#[derive(serde::Serialize, Debug)] +pub struct MempoolSpaceWebSocketRequestMessage { + pub action: String, + pub data: Vec, +} + +/// Enum that implements the candidates for first message request for mempool.space WebSocket client +#[allow(dead_code)] +pub enum MempoolSpaceWebSocketRequestData { + /// Used to listen only new blocks + Blocks, + /// Used to subscribe to mempool-blocks events + MempoolBlocks, + /// Used to subscribe to all events related to an address + TrackAddress(Address), +} + +/// Enum that implements the variants for `BlockEvent` +#[derive(Debug, Clone, Copy)] +pub enum BlockEvent { + /// Used when connecting and extending the current active chain being streamed + Connected(T), + /// Used when there is a fork or reorganization event that turns the block stale + /// then it's disconnected from current active chain + Disconnected((u32, BlockHash)), +} diff --git a/src/bin.rs b/src/bin.rs new file mode 100644 index 0000000..d0fd93f --- /dev/null +++ b/src/bin.rs @@ -0,0 +1,83 @@ +use anyhow::Ok; +use clap::{ArgGroup, Parser, Subcommand}; +use futures::{pin_mut, StreamExt}; +use serde::{Deserialize, Serialize}; + +#[derive(Parser)] +#[clap(name = "block-events-cli")] +#[clap(author = "Leonardo Souza , LLFourn ")] +#[clap(version = "0.1.0")] +#[clap( + long_about = "A CLI interface and tool to use with the block-events library. A work in progress project for Summer of Bitcoin 2022." +)] + +struct Cli { + #[clap(subcommand)] + command: Commands, + + #[clap(short, long, default_value = "https://mempool.space/testnet/api")] + http_base_url: String, + + #[clap(short, long, default_value = "wss://mempool.space/testnet/")] + ws_base_url: String, +} + +#[derive(Debug, Subcommand)] +enum Commands { + // track address feature from mempool.space ws + AddressTracking { + #[clap(short, long)] + address: String, + }, + + // subscribe and fetch new blocks related data + #[clap(group(ArgGroup::new("data-stream") + .required(true) + .args(&["blocks", "mempool-blocks"])))] + DataStream { + // new blocks data only + #[clap(long)] + blocks: bool, + + // new mempool-blocks data only + #[deprecated] + #[clap(long)] + mempool_blocks: bool, + }, +} + +#[allow(dead_code)] +#[derive(Serialize, Deserialize, Debug)] +struct BlockDataMessage { + action: String, + data: Vec, +} + +#[allow(dead_code)] +#[derive(Serialize, Deserialize, Debug)] +struct TrackAddressMessage { + track_address: String, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + env_logger::init(); + + let cli = Cli::parse(); + + // async fetch the data stream through the lib + let checkpoint = None; + let block_events = block_events::subscribe_to_block_headers( + cli.http_base_url.as_str(), + cli.ws_base_url.as_str(), + checkpoint, + ) + .await?; + + // consume and execute the code (current matching and printing) in async manner for each new block-event + pin_mut!(block_events); + while let Some(block_event) = block_events.next().await { + println!("{:#?}", block_event); + } + Ok(()) +} diff --git a/src/http.rs b/src/http.rs new file mode 100644 index 0000000..42a61d3 --- /dev/null +++ b/src/http.rs @@ -0,0 +1,151 @@ +// Block Events Library +// Written in 2022 by Leonardo Lima <> and Lloyd Fournier <> +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +//! Http client implementation for mempool.space available endpoints +//! It used `reqwest` async client + +#![allow(unused_imports)] +use std::ops::Deref; + +use bitcoin::{ + consensus::deserialize, hashes::hex::FromHex, Block, BlockHash, BlockHeader, Transaction, Txid, +}; +use reqwest::Client; + +use crate::api::BlockExtended; + +/// Generic HttpClient using `reqwest` +/// +/// This implementation and approach is based on the BDK's esplora client +/// +/// `` +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub struct HttpClient { + /// The base url for building our http rest calls + /// It's expected to have the protocol, domain and initial api path (e.g: ``) + base_url: String, + /// A `reqwest` client with default or selected config + client: Client, + /// The number of concurrency requests the client is allowed to make + concurrency: u8, +} + +impl HttpClient { + /// Creates a new HttpClient, for given base url and concurrency + pub fn new(base_url: &str) -> Self { + HttpClient { + client: Client::new(), + base_url: base_url.to_string(), + concurrency: crate::DEFAULT_CONCURRENT_REQUESTS, + } + } + + /// Get current blockchain height [`u32`], the current tip height + pub async fn get_tip_height(&self) -> anyhow::Result { + let res = self + .client + .get(&format!("{}/blocks/tip/height", self.base_url)) + .send() + .await?; + + Ok(res.error_for_status()?.text().await?.parse()?) + } + + /// Get current blockchain hash [`BlockHash`], the current tip hash + pub async fn get_tip_hash(&self) -> anyhow::Result { + let res = self + .client + .get(&format!("{}/blocks/tip/hash", self.base_url)) + .send() + .await?; + + Ok(res.error_for_status()?.text().await?.parse()?) + } + + /// Get the [`BlockHash`] for given block height + pub async fn get_block_hash(&self, height: u32) -> anyhow::Result { + let res = self + .client + .get(&format!("{}/block-height/{}", self.base_url, height)) + .send() + .await?; + + Ok(res.error_for_status()?.text().await?.parse()?) + } + + /// Get the [`BlockHeader`] for given [`BlockHash`] + pub async fn get_block_header(&self, hash: BlockHash) -> anyhow::Result { + let res = self + .client + .get(&format!("{}/block/{}/header", self.base_url, hash)) + .send() + .await?; + + let raw_header = Vec::::from_hex(res.error_for_status()?.text().await?.as_str())?; + let header: BlockHeader = deserialize(&raw_header)?; + + Ok(header) + } + + /// Get full block in [`BlockExtended`] format, for given [`BlockHash`] + pub async fn get_block(&self, block_hash: BlockHash) -> anyhow::Result { + let res = self + .client + .get(&format!("{}/block/{}", self.base_url, block_hash)) + .send() + .await?; + + Ok(serde_json::from_str( + res.error_for_status()?.text().await?.as_str(), + )?) + } + + /// This only works when using the blockstream.info (esplora) client, it does not work with mempool.space client + /// + /// NOTE: It will be used instead of multiple calls for building blocks as this commit is added in a new mempool.space + /// release: `` + pub async fn get_block_raw(&self, block_hash: BlockHash) -> anyhow::Result { + let res = self + .client + .get(&format!("{}/block/{}/raw", self.base_url, block_hash)) + .send() + .await?; + + let block: Block = deserialize(res.bytes().await?.deref())?; + + Ok(block) + } + + /// Get all transactions ids [`Vec`] for given [`BlockHash`] + pub async fn get_tx_ids(&self, block_hash: BlockHash) -> anyhow::Result> { + let res = self + .client + .get(format!("{}/block/{}/txids", self.base_url, block_hash)) + .send() + .await?; + + let tx_ids: Vec = serde_json::from_str(res.text().await?.as_str())?; + Ok(tx_ids) + } + + /// Get the [`Transaction`] for given transaction hash/id [`Txid`] + pub async fn get_tx(&self, tx_id: Txid) -> anyhow::Result { + let res = self + .client + .get(&format!("{}/tx/{}/hex", self.base_url, tx_id)) + .send() + .await?; + + let raw_tx = Vec::::from_hex(res.error_for_status()?.text().await?.as_str())?; + let tx: Transaction = deserialize(&raw_tx)?; + + Ok(tx) + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..50eb7e8 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,374 @@ +// Block Events Library +// Written in 2022 by Leonardo Lima <> and Lloyd Fournier <> +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +//! # Block Events Library +//! +//! This a simple, concise and lightweight library for subscribing to real-time stream of blocks from multiple sources. +//! +//! It focuses on providing a simple API and [`BlockEvent`] response type for clients to consume +//! any new events or starting from a pre-determined checkpoint. +//! +//! The library produces [`BlockEvent::Connected`] and [`BlockEvent::Disconnected`] events by handling reorganization +//! events and blockchain forks. +//! +//! The library works in an `async` fashion producing a Rust stream of [`BlockEvent`]. +//! +//! It is a project under development during the Summer of Bitcoin'22 @BitcoinDevKit, if you would like to know more +//! please check out the repository, project proposal or reach out. +//! +//! # Examples +//! ## Subscribe to all new block events for mempool.space +//! ``` no_run +//! use anyhow::{self, Ok}; +//! use futures::{pin_mut, StreamExt}; +//! +//! #[tokio::main] +//! async fn main() -> anyhow::Result<()> { +//! env_logger::init(); +//! +//! // for mempool.space testnet network +//! let http_base_url = "http://mempool.space/testnet/api/"; +//! let ws_base_url = "wss://mempool.space/testnet"; +//! +//! // no checkpoint for this example, but you could use the following one to test it by yourself (in mainnet). +//! // checkpoint for first BDK Taproot transaction on mainnet (base_url update needed) +//! // let checkpoint = (709635, bitcoin::BlockHash::from("00000000000000000001f9ee4f69cbc75ce61db5178175c2ad021fe1df5bad8f")); +//! let checkpoint = None; +//! +//! // async fetch the block-events stream through the lib +//! let block_events = +//! block_events::subscribe_to_block_headers(http_base_url, ws_base_url, checkpoint).await?; +//! +//! // consume and execute your code (current only matching and printing) in async manner for each new block-event +//! pin_mut!(block_events); +//! while let Some(block_event) = block_events.next().await { +//! match block_event? { +//! block_events::api::BlockEvent::Connected(block_header) => { +//! println!("[connected][block_header] {:#?}", block_header); +//! } +//! block_events::api::BlockEvent::Disconnected((height, block_hash)) => { +//! println!( +//! "[disconnected][height: {:#?}][block_hash: {:#?}]", +//! height, block_hash +//! ); +//! } +//! } +//! } +//! Ok(()) +//! } +//! ``` + +pub mod api; +pub mod http; +pub mod websocket; + +pub extern crate async_stream; +pub extern crate bitcoin; +pub extern crate tokio; +pub extern crate tokio_tungstenite; + +use std::{collections::HashMap, collections::VecDeque, pin::Pin}; + +use api::{BlockEvent, BlockExtended}; +use futures::{Stream, StreamExt, TryStream}; +use http::HttpClient; + +use anyhow::{anyhow, Ok}; +use async_stream::try_stream; +use bitcoin::{ + blockdata::constants::genesis_block, Block, BlockHash, BlockHeader, Network, Transaction, +}; + +const DEFAULT_CONCURRENT_REQUESTS: u8 = 4; + +/// A simple cache struct to store the all fetched and new blocks in-memory +/// +/// It's used in order to handle reorganization events, and produce both connected and disconnected events +#[derive(Debug, Clone)] +pub struct BlockHeadersCache { + pub tip: BlockHash, + pub active_headers: HashMap, + pub stale_headers: HashMap, +} + +impl BlockHeadersCache { + /// Create a new instance of [`BlockHeadersCache`] for given checkpoint (height: [`u32`], hash: [`BlockHash`]) + /// + /// It creates with the checkpoint block as tip and active_headers + pub async fn new(base_url: &str) -> anyhow::Result { + let http_client = HttpClient::new(base_url); + + let hash = http_client.get_tip_hash().await?; + let header = http_client.get_block_header(hash).await?; + + Ok(BlockHeadersCache { + tip: hash, + active_headers: HashMap::from([(hash, header)]), + stale_headers: HashMap::new(), + }) + } + + /// Create a new instance of [`BlockHeadersCache`] for given [`Network`] + /// + /// It creates with the genesis block for given network as tip and active_headers + pub fn new_with_genesis(network: Network) -> BlockHeadersCache { + let genesis_block = genesis_block(network); + + BlockHeadersCache { + tip: genesis_block.block_hash(), + active_headers: HashMap::from([(genesis_block.block_hash(), genesis_block.header)]), + stale_headers: HashMap::new(), + } + } + + /// Create a new instance of [`BlockHeadersCache`] for given checkpoint (height: [`u32`], hash: [`BlockHash`]) + /// + /// It creates with the checkpoint block as tip and active_headers + pub async fn new_with_checkpoint( + base_url: &str, + checkpoint: (u32, BlockHash), + ) -> anyhow::Result { + let (_, hash) = checkpoint; + + let header = HttpClient::new(base_url).get_block_header(hash).await?; + + Ok(BlockHeadersCache { + tip: hash, + active_headers: HashMap::from([(hash, header)]), + stale_headers: HashMap::new(), + }) + } + + /// Validate if the new [`BlockHeader`] or [`BlockExtended`] candidate is a valid tip + /// + /// Updates the [`BlockHeadersCache`] state, updating the tip, extending the active_headers and returns a boolean + pub fn validate_new_header(&mut self, candidate: BlockHeader) -> bool { + // TODO: (@leonardo.lima) It should check and validate the PoW for the header candidates + if self.tip == candidate.prev_blockhash { + self.tip = candidate.block_hash(); + self.active_headers + .insert(candidate.block_hash(), candidate); + return true; + } + false + } + + /// Find common ancestor for current active chain and the fork chain candidate + /// + /// Updates the [`BlockHeadersCache`] state with fork chain candidates + /// + /// Returns a common ancestor [`BlockHeader`] stored in [`BlockHeadersCache`] and the + /// fork branch chain as a [`VecDeque`] + pub async fn find_or_fetch_common_ancestor( + &self, + http_client: HttpClient, + fork_candidate: BlockHeader, + ) -> anyhow::Result<(BlockHeader, VecDeque)> { + let mut common_ancestor = fork_candidate; + let mut fork_branch: VecDeque = VecDeque::new(); + while !self + .active_headers + .contains_key(&common_ancestor.block_hash()) + { + fork_branch.push_back(common_ancestor); + common_ancestor = http_client + .get_block_header(common_ancestor.prev_blockhash) + .await?; + } + Ok((common_ancestor, fork_branch)) + } + + /// Rollback active chain in [`BlockHeadersCache`] back to passed [`BlockHeader`] + /// + /// Returns all stale, and to be disconnected blocks as a [`VecDeque`] + pub async fn rollback_active_chain( + &mut self, + header: BlockHeader, + ) -> anyhow::Result> { + let mut disconnected = VecDeque::new(); + while header.block_hash() != self.tip { + let (stale_hash, stale_header) = self.active_headers.remove_entry(&self.tip).unwrap(); + disconnected.push_back(stale_header); + + self.stale_headers.insert(stale_hash, stale_header); + self.tip = stale_header.prev_blockhash; + } + Ok(disconnected) + } + + /// Apply fork branch to active chain, and update tip to new [`BlockHeader`] + /// + /// Returns the new tip [`BlockHash`], and the connected block headers as a [`VecDeque`] + pub fn apply_fork_chain( + &mut self, + mut fork_branch: VecDeque, + ) -> anyhow::Result<(BlockHash, VecDeque)> { + let mut connected = VecDeque::new(); + while !fork_branch.is_empty() { + let header = fork_branch.pop_front().unwrap(); + connected.push_back(header); + + self.active_headers.insert(header.block_hash(), header); + self.tip = header.block_hash(); + } + Ok((self.tip, connected)) + } +} + +/// Process all candidates listened from source, it tries to apply the candidate to current active chain cached +/// It handles reorganization and fork if needed +/// Steps: +/// - validates if current candidate is valid as a new tip, if valid extends chain producing [`BlockEvent::Connected`] +/// - otherwise, find common ancestor between branches +/// - rollback current cached active chain +/// - apply forked branch, and produces [`BlockEvent::Disconnected`] for staled blocks and [`BlockEvent::Connected`] +/// for new branch +async fn process_candidates( + base_url: &str, + mut cache: BlockHeadersCache, + mut candidates: Pin>>>, +) -> anyhow::Result>>> { + let http_client = HttpClient::new(base_url); + + let stream = try_stream! { + // TODO: (@leonardo.lima) Do not just propagate the errors, add a retry mechanism instead + while let candidate = candidates.next().await.ok_or(anyhow!("the `bitcoin::BlockHeader` candidate is None"))?? { + // validate if the [`BlockHeader`] candidate is a valid new tip + // yields a [`BlockEvent::Connected()`] variant and continue the iteration + if cache.validate_new_header(candidate) { + yield BlockEvent::Connected(BlockHeader::from(candidate.clone())); + continue + } + + // find common ancestor for current active chain and the forked chain + // fetches forked chain candidates and store in cache + let (common_ancestor, fork_chain) = cache.find_or_fetch_common_ancestor(http_client.clone(), candidate).await?; + + // rollback current active chain, moving blocks to staled field + // yields BlockEvent::Disconnected((u32, BlockHash)) + let mut disconnected: VecDeque = cache.rollback_active_chain(common_ancestor).await?; + while !disconnected.is_empty() { + if let Some(block_header) = disconnected.pop_back() { + let block_ext: BlockExtended = http_client.get_block(block_header.block_hash()).await?; + yield BlockEvent::Disconnected((block_ext.height, block_header.block_hash())); + } + } + + // iterate over forked chain candidates + // update [`Cache`] active_headers field with candidates + let (_, mut connected) = cache.apply_fork_chain(fork_chain)?; + while !connected.is_empty() { + let block = connected.pop_back().unwrap(); + yield BlockEvent::Connected(BlockHeader::from(block.clone())); + } + } + }; + Ok(stream) +} + +/// Subscribe to a real-time stream of [`BlockEvent`], for all new blocks or starting from an optional checkpoint +pub async fn subscribe_to_block_headers( + http_base_url: &str, + ws_base_url: &str, + checkpoint: Option<(u32, BlockHash)>, +) -> anyhow::Result>>>>> { + match checkpoint { + Some(checkpoint) => { + let headers_cache = + BlockHeadersCache::new_with_checkpoint(http_base_url, checkpoint).await?; + let prev_headers = fetch_block_headers(http_base_url, checkpoint).await?; + let new_headers = websocket::listen_new_block_headers(ws_base_url).await?; + let candidates = Box::pin(prev_headers.chain(new_headers)); + Ok(Box::pin( + process_candidates(http_base_url, headers_cache, candidates).await?, + )) + } + None => { + let headers_cache = BlockHeadersCache::new(http_base_url).await?; + let new_header_candidates = + Box::pin(websocket::listen_new_block_headers(ws_base_url).await?); + Ok(Box::pin( + process_candidates(http_base_url, headers_cache, new_header_candidates).await?, + )) + } + } +} + +/// Subscribe to a real-time stream of [`BlockEvent`] of full rust-bitcoin blocks, for new mined blocks or +/// starting from an optional checkpoint (height: u32, hash: BlockHash) +pub async fn subscribe_to_blocks( + http_base_url: &str, + ws_base_url: &str, + checkpoint: Option<(u32, BlockHash)>, +) -> anyhow::Result>>> { + // build and create a http client + let http_client = HttpClient::new(http_base_url); + + // subscribe to block_headers events + let mut header_events = + subscribe_to_block_headers(http_base_url, ws_base_url, checkpoint).await?; + + // iterate through each event for block_headers + let stream = try_stream! { + while let Some(event) = header_events.next().await { + match event? { + BlockEvent::Connected(header) => { + // fetch all transaction ids (Txids) for the block + let tx_ids = http_client.get_tx_ids(header.block_hash()).await?; + + // fetch full transaction and build transaction list Vec + let mut txs: Vec = Vec::new(); + for id in tx_ids { + let tx = http_client.get_tx(id).await?; + txs.push(Transaction::from(tx)); + } + + // yield connected event for full block + yield BlockEvent::Connected(Block { + header: header, + txdata: txs, + }); + }, + // otherwise yield error or the disconnected event + BlockEvent::Disconnected((height, hash)) => yield BlockEvent::Disconnected((height, hash)), + } + } + }; + Ok(stream) +} + +/// Fetch all [`BlockHeader`] starting from the checkpoint ([`u32`], [`BlockHeader`]) up to tip +pub async fn fetch_block_headers( + base_url: &str, + checkpoint: (u32, BlockHash), +) -> anyhow::Result>> { + let http_client = HttpClient::new(base_url); + + // checks if the checkpoint height and hash matches for the current chain + let (ckpt_height, ckpt_hash) = checkpoint; + if ckpt_hash != http_client.get_block_hash(ckpt_height).await? { + return Err(anyhow!( + "The checkpoint passed is invalid, it should exist in the blockchain." + )); + } + + let tip_height = http_client.get_tip_height().await?; + let stream = try_stream! { + for height in ckpt_height..tip_height { + let block_hash = http_client.get_block_hash(height).await?; + let block_header = http_client.get_block_header(block_hash).await?; + yield block_header; + }; + + let tip_hash = http_client.get_tip_hash().await?; + let tip_header = http_client.get_block_header(tip_hash).await?; + yield tip_header; + }; + Ok(stream) +} diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index 0ea03ac..0000000 --- a/src/main.rs +++ /dev/null @@ -1,133 +0,0 @@ -use anyhow::anyhow; -use clap::{Subcommand, Parser}; -use futures_util::{SinkExt, StreamExt}; -use serde::{Deserialize, Serialize}; -use std::{env, time::Duration}; -use tokio_tungstenite::{connect_async_tls_with_config, tungstenite::protocol::Message}; - -#[derive(Parser)] -#[clap(name = "CLI block explorer with mempool.space websocket - WIP")] -#[clap(author = "Leonardo L.")] -#[clap(version = "0.1")] -#[clap(about = "A work in progress CLI block explorer to be used with BDK, consuming data from mempool.space websocket.\n - This an initial competency test for Summer of Bitcoin 2022", long_about = None)] - -struct Cli { - #[clap(subcommand)] - command: Commands, - - #[clap(short, long)] - endpoint: Option, -} - -#[derive(Debug, Subcommand)] -enum Commands { - // track address - TrackAddress { - // address to track - #[clap(short, long)] - address: String, - }, - // fetch all new blocks - BlocksData { - // remove blocks subscription - #[clap(long)] - no_blocks: bool, - - // remove mempool blocks subscription - #[clap(long)] - no_mempool_blocks: bool, - }, -} - -#[allow(dead_code)] -#[derive(Serialize, Deserialize, Debug)] -struct BlockDataMessage { - action: String, - data: Vec, -} - -#[allow(dead_code)] -#[derive(Serialize, Deserialize, Debug)] -struct TrackAddressMessage { - track_address: String, -} - - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let cli = Cli::parse(); - - let req_message = build_request_message(&cli); - - let connect_address = format!( - "wss://{}/v1/ws", - cli.endpoint - .or(env::var("MEMPOOL_ENDPOINT").ok()) - .unwrap_or("mempool.space/api".to_string()) - ); - - let connect_url = url::Url::parse(&connect_address).unwrap(); - let (mut websocket_stream, _ws_res) = connect_async_tls_with_config(connect_url, None, None) - .await - .expect("failed to connect with url"); - println!("websocket handshake successfully completed!"); - - if let Err(_) = websocket_stream.send(Message::text(req_message)).await { - return Err(anyhow!("Failed to send first message to websocket")); - } - - // need to ping every so often to keep websocket alive - let mut pinger = tokio::time::interval(Duration::from_secs(60)); - - loop { - tokio::select! { - message = websocket_stream.next() => { - if let Some(message) = message { - match message? { - Message::Text(text) => { - let obj: serde_json::Value = serde_json::from_str(&text).unwrap(); - println!("{}", serde_json::to_string_pretty(&obj).unwrap()); - }, - Message::Close(_) => { - eprintln!("websocket closing gracefully"); - break; - }, - Message::Binary(_) => { - eprintln!("unexpected binary message"); - break; - }, - _ => { /*ignore*/ } - } - } - } - _ = pinger.tick() => { - websocket_stream.send(Message::Ping(vec![])).await.unwrap() - } - } - } - - Ok(()) -} - -fn build_request_message(cli: &Cli) -> String { - - match &cli.command { - Commands::TrackAddress { address } => { - return serde_json::to_string(&(TrackAddressMessage {track_address: String::from(address)})).unwrap(); - } - Commands::BlocksData { no_blocks, no_mempool_blocks} => { - let mut data = vec![]; - - if !no_mempool_blocks { - data.push(String::from("mempool-blocks")); - } - - if !no_blocks { - data.push(String::from("blocks")); - } - - return serde_json::to_string(&(BlockDataMessage {action: String::from("want"), data: data})).unwrap(); - } - } -} \ No newline at end of file diff --git a/src/websocket.rs b/src/websocket.rs new file mode 100644 index 0000000..970cd54 --- /dev/null +++ b/src/websocket.rs @@ -0,0 +1,124 @@ +// Block Events Library +// Written in 2022 by Leonardo Lima <> and Lloyd Fournier <> +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license +// , at your option. +// You may not use this file except in accordance with one or both of these +// licenses. + +//! WebSocket module for mempool.space +//! It has functions to connect and create a new WebSocket client, and also subscribe for new blocks (BlockHeaders) + +use super::api::{ + MempoolSpaceWebSocketMessage, MempoolSpaceWebSocketRequestData, + MempoolSpaceWebSocketRequestMessage, +}; + +use anyhow::{anyhow, Ok as AnyhowOk}; +use async_stream::try_stream; +use bitcoin::BlockHeader; +use core::result::Result::Ok; +use futures::{SinkExt, StreamExt, TryStream}; +use std::time::Duration; +use tokio::net::TcpStream; +use tokio_tungstenite::tungstenite::protocol::Message; +use tokio_tungstenite::{connect_async_tls_with_config, MaybeTlsStream, WebSocketStream}; + +/// Create a new WebSocket client for given base url and initial message +/// +/// It uses `tokio_tungestenite` crate and produces [`WebSocketStream`] to be handled and treated by caller +async fn websocket_client( + base_url: &str, + message: String, +) -> anyhow::Result>> { + let url = url::Url::parse(format!("{}/api/v1/ws", base_url).as_str())?; + log::info!("starting websocket handshake with url={}", url); + + let (mut websocket_stream, websocket_response) = + connect_async_tls_with_config(url, None, None).await?; + + log::info!("websocket handshake successfully completed!"); + log::info!("handshake completed with response={:?}", websocket_response); + + if (websocket_stream.send(Message::text(&message)).await).is_err() { + log::error!("failed to publish first message to websocket"); + return Err(anyhow!("failed to publish first message to websocket")); + }; + log::info!("published message: {:#?}, successfully!", &message); + AnyhowOk(websocket_stream) +} + +/// Connects to mempool.space WebSocket client and listen to new messages producing a stream of [`BlockHeader`] candidates +pub async fn listen_new_block_headers( + base_url: &str, +) -> anyhow::Result>> { + let init_message = serde_json::to_string(&build_websocket_request_message( + &MempoolSpaceWebSocketRequestData::Blocks, + ))?; + + let mut ws_stream = websocket_client(base_url, init_message).await?; + + // need to ping every so often to keep the websocket connection alive + let mut pinger = tokio::time::interval(Duration::from_secs(60)); + + let stream = try_stream! { + loop { + tokio::select! { + message = ws_stream.next() => { + if let Some(message) = message { + match message { + Ok(message) => match message { + Message::Text(text) => { + let parsed: MempoolSpaceWebSocketMessage = match serde_json::from_str(&text) { + Err(_) => continue, + Ok(parsed) => parsed, + }; + yield BlockHeader::from(parsed.block); + }, + Message::Close(_) => { + eprintln!("websocket closing gracefully"); + break; + }, + Message::Binary(_) => { + eprintln!("unexpected binary message"); + break; + }, + _ => {/* ignore */} + }, + Err(_error) => { /* ignore */}, + } + } + } + _ = pinger.tick() => { + log::info!("pinging to websocket to keep connection alive"); + if (ws_stream.send(Message::Ping(vec![])).await).is_err() { + log::error!("failed to send ping message to websocket"); + continue + } + } + } + } + }; + + AnyhowOk(stream) +} + +fn build_websocket_request_message( + data: &MempoolSpaceWebSocketRequestData, +) -> MempoolSpaceWebSocketRequestMessage { + let mut message = MempoolSpaceWebSocketRequestMessage { + action: String::from("want"), + data: vec![], + }; + + match data { + MempoolSpaceWebSocketRequestData::Blocks => message.data.push(String::from("blocks")), + MempoolSpaceWebSocketRequestData::MempoolBlocks => { + message.data.push(String::from("mempool-blocks")) + } + // FIXME: (@leonardo.lima) fix this track-address to use different struct + MempoolSpaceWebSocketRequestData::TrackAddress(..) => { /* ignore */ } + } + message +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs new file mode 100644 index 0000000..2953016 --- /dev/null +++ b/tests/integration_tests.rs @@ -0,0 +1,527 @@ +use bitcoin::BlockHash; +use bitcoind::{bitcoincore_rpc::RpcApi, BitcoinD}; +use block_events::{api::BlockEvent, http::HttpClient, websocket}; +use futures::{pin_mut, StreamExt}; +use serial_test::serial; +use std::{collections::VecDeque, ops::Deref, time::Duration}; +use testcontainers::{clients, images, images::generic::GenericImage, RunnableImage}; + +const HOST_IP: &str = "127.0.0.1"; + +const MARIADB_NAME: &str = "mariadb"; +const MARIADB_TAG: &str = "10.5.8"; +const MARIADB_READY_CONDITION: &str = "mysqld: ready for connections."; + +const MEMPOOL_BACKEND_NAME: &str = "mempool/backend"; +const MEMPOOL_BACKEND_TAG: &str = "v2.4.1"; +const MEMPOOL_BACKEND_READY_CONDITION: &str = "Mempool Server is running on port 8999"; + +// TODO: (@leonardo.lima) This should be derived instead, should we add it to bitcoind ? +const RPC_AUTH: &str = "mempool:3c417dbc7ccabb51d8e6fedc302288db$ed44e37a937e8706ea51bbc761df76e995fe92feff8751ce85feaea4c4ae80b1"; + +#[cfg(all( + target_os = "macos", + any(target_arch = "x86_64", target_arch = "aarch64") +))] +fn docker_host_address() -> &'static str { + "host.docker.internal" +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64", target_arch = "aarch64"))] +fn docker_host_address() -> &'static &str { + "172.17.0.1" +} + +pub struct MempoolTestClient { + pub bitcoind: BitcoinD, + pub mariadb_database: RunnableImage, + pub mempool_backend: RunnableImage, +} + +impl MempoolTestClient { + fn start_bitcoind(bitcoind_exe: Option) -> BitcoinD { + let bitcoind_exe = bitcoind_exe.unwrap_or(bitcoind::downloaded_exe_path().ok().expect( + "you should provide a bitcoind_exe parameter or specify a bitcoind version feature", + )); + + log::debug!("launching bitcoind [bitcoind_exe {:?}]", bitcoind_exe); + + let mut conf = bitcoind::Conf::default(); + let rpc_auth = format!("-rpcauth={}", RPC_AUTH); + let rpc_bind = format!("-rpcbind=0.0.0.0"); + conf.args.push(rpc_auth.as_str()); + conf.args.push(rpc_bind.as_str()); + conf.args.push("-txindex"); + conf.args.push("-server"); + + let bitcoind = BitcoinD::with_conf(&bitcoind_exe, &conf).unwrap(); + + log::debug!( + "successfully launched bitcoind and generated initial coins [bitcoind_exe {:?}]", + bitcoind_exe + ); + + bitcoind + } + + fn start_database(name: Option<&str>, tag: Option<&str>) -> RunnableImage { + let name = name.unwrap_or(MARIADB_NAME); + let tag = tag.unwrap_or(MARIADB_TAG); + + log::debug!( + "creating image and starting container [name {}] [tag {}]", + name, + tag + ); + + let image = images::generic::GenericImage::new(name, tag).with_wait_for( + testcontainers::core::WaitFor::StdErrMessage { + message: MARIADB_READY_CONDITION.to_string(), + }, + ); + + let image = RunnableImage::from(image) + .with_env_var(("MYSQL_DATABASE", "mempool")) + .with_env_var(("MYSQL_USER", "mempool")) + .with_env_var(("MYSQL_PASSWORD", "mempool")) + .with_env_var(("MYSQL_ROOT_PASSWORD", "mempool")) + .with_mapped_port((3306, 3306)); + + log::debug!( + "successfully created and started container [name {}] [tag {}]", + name, + tag + ); + image + } + + fn start_backend( + name: Option<&str>, + tag: Option<&str>, + core: &BitcoinD, + ) -> RunnableImage { + let name = name.unwrap_or(MEMPOOL_BACKEND_NAME); + let tag = tag.unwrap_or(MEMPOOL_BACKEND_TAG); + + log::debug!( + "creating image and starting container [name {}] [tag {}]", + name, + tag + ); + + let image = images::generic::GenericImage::new(name, tag).with_wait_for( + testcontainers::core::WaitFor::StdErrMessage { + message: MEMPOOL_BACKEND_READY_CONDITION.to_string(), + }, + ); + + let bitcoind_port = core.params.rpc_socket.port().to_string(); + + println!("{}", docker_host_address().to_string()); + + let image = RunnableImage::from(image) + .with_env_var(("MEMPOOL_BACKEND", "none")) + .with_env_var(("DATABASE_HOST", docker_host_address().to_string())) + .with_env_var(("CORE_RPC_HOST", docker_host_address().to_string())) + .with_env_var(("CORE_RPC_PORT", bitcoind_port)) + .with_mapped_port((8999, 8999)); + + log::debug!( + "successfully created and started container [name {}] [tag {}]", + name, + tag + ); + image + } +} + +impl Default for MempoolTestClient { + fn default() -> Self { + let bitcoind = Self::start_bitcoind(None); + let mariadb = Self::start_database(None, None); + let mempool = Self::start_backend(None, None, &bitcoind); + + MempoolTestClient { + bitcoind: bitcoind, + mariadb_database: mariadb, + mempool_backend: mempool, + } + } +} + +#[tokio::test] +#[serial] +async fn test_fetch_tip_height() { + let _ = env_logger::try_init(); + + let MempoolTestClient { + bitcoind, + mariadb_database, + mempool_backend, + } = MempoolTestClient::default(); + + let docker = clients::Cli::docker(); + let _mariadb = docker.run(mariadb_database); + + // there is some small delay between running the docker for mariadb database and the port being really available + std::thread::sleep(Duration::from_millis(5000)); + let mempool = docker.run(mempool_backend); + + let base_url = format!( + "http://{}:{}/api/v1", + HOST_IP, + mempool.get_host_port_ipv4(8999) + ); + + let http_client = HttpClient::new(&base_url); + + // should return the current tip height + for i in 0..5 { + let tip = http_client.get_tip_height().await.unwrap(); + assert_eq!(i, tip); + + // generate new block + let address = bitcoind.client.get_new_address(None, None).unwrap(); + let _ = bitcoind.client.generate_to_address(1, &address).unwrap(); + } +} + +#[tokio::test] +#[serial] +async fn test_fetch_block_hash_by_height() { + let _ = env_logger::try_init(); + let delay = Duration::from_millis(5000); + + let docker = clients::Cli::docker(); + let client = MempoolTestClient::default(); + + let _mariadb = docker.run(client.mariadb_database); + + std::thread::sleep(delay); // there is some delay between running the docker and the port being really available + let mempool = docker.run(client.mempool_backend); + + let rpc_client = &client.bitcoind.client; + let base_url = format!( + "http://{}:{}/api/v1", + HOST_IP, + mempool.get_host_port_ipv4(8999) + ); + let http_client = HttpClient::new(&base_url); + + // should return an error if there is no block created yet for given height + assert!(http_client.get_block_hash(100).await.is_err()); + + // should return block hash for existing block by height + for i in 1..10 { + let gen_hash = rpc_client + .generate_to_address(1, &rpc_client.get_new_address(None, None).unwrap()) + .unwrap(); + + let res_hash = http_client.get_block_hash(i).await.unwrap(); + assert_eq!(gen_hash.first().unwrap(), &res_hash); + } +} + +#[tokio::test] +#[serial] +async fn test_fetch_blocks_for_invalid_checkpoint() { + let _ = env_logger::try_init(); + let delay = Duration::from_millis(5000); + + let docker = clients::Cli::docker(); + let client = MempoolTestClient::default(); + + let _mariadb = docker.run(client.mariadb_database); + std::thread::sleep(delay); // there is some delay between running the docker and the port being really available + + let mempool = docker.run(client.mempool_backend); + + let base_url = format!( + "http://{}:{}/api/v1", + HOST_IP, + mempool.get_host_port_ipv4(8999) + ); + let checkpoint = (0, BlockHash::default()); + let blocks = block_events::fetch_block_headers(&base_url, checkpoint).await; + + // should produce an error for invalid checkpoint + assert!(blocks.is_err()); + + // should produce an error indicating checkpoint as invalid + assert_eq!( + blocks.err().unwrap().to_string(), + "The checkpoint passed is invalid, it should exist in the blockchain." + ); +} + +#[tokio::test] +#[serial] +async fn test_fetch_blocks_for_checkpoint() { + let _ = env_logger::try_init(); + let delay = Duration::from_millis(5000); + + let docker = clients::Cli::docker(); + let client = MempoolTestClient::default(); + + let _mariadb = docker.run(client.mariadb_database); + std::thread::sleep(delay); // there is some delay between running the docker and the port being really available + let mempool = docker.run(client.mempool_backend); + + let rpc_client = &client.bitcoind.client; + + // generate new 20 blocks + let mut gen_blocks = rpc_client + .generate_to_address(20, &rpc_client.get_new_address(None, None).unwrap()) + .unwrap(); + log::debug!("[{:#?}]", gen_blocks); + + let checkpoint = (10, *gen_blocks.get(9).unwrap()); + let base_url = format!( + "http://{}:{}/api/v1", + HOST_IP, + mempool.get_host_port_ipv4(8999) + ); + + let blocks = block_events::fetch_block_headers(&base_url, checkpoint) + .await + .unwrap(); + + pin_mut!(blocks); + // should return all 10 blocks from 10 to 20, as 10 being the checkpoint + for gen_block in &mut gen_blocks[9..] { + let block = blocks.next().await.unwrap().unwrap(); + assert_eq!(gen_block.deref(), &block.block_hash()); + } +} + +#[tokio::test] +#[serial] +async fn test_failure_for_invalid_websocket_url() { + let block_events = + websocket::listen_new_block_headers(&format!("ws://{}:{}", HOST_IP, 8999)).await; + + // should return an Err. + assert!(block_events.is_err()); + + // should return connection Err. + assert_eq!( + block_events.err().unwrap().to_string(), + "IO error: Connection refused (os error 61)" + ); +} + +#[tokio::test] +#[serial] +async fn test_block_events_stream() { + let _ = env_logger::try_init(); + + let MempoolTestClient { + bitcoind, + mariadb_database, + mempool_backend, + } = MempoolTestClient::default(); + + let docker = clients::Cli::docker(); + let _mariadb = docker.run(mariadb_database); + + // there is some small delay between running the docker for mariadb database and the port being really available + std::thread::sleep(Duration::from_millis(5000)); + let mempool = docker.run(mempool_backend); + + let http_base_url = format!( + "http://{}:{}/api/v1", + HOST_IP, + mempool.get_host_port_ipv4(8999) + ); + let ws_base_url = format!("ws://{}:{}", HOST_IP, mempool.get_host_port_ipv4(8999)); + + // get stream of new block-events + let events = block_events::subscribe_to_block_headers(&http_base_url, &ws_base_url, None) + .await + .unwrap(); + + // generate 5 new blocks through bitcoind rpc-client + let address = &bitcoind.client.get_new_address(None, None).unwrap(); + let mut blocks = VecDeque::from(bitcoind.client.generate_to_address(5, address).unwrap()); + + // consume new blocks from block-events stream + pin_mut!(events); + while !blocks.is_empty() { + let block_hash = blocks.pop_front().unwrap(); + let event = events.next().await.unwrap().unwrap(); + + log::debug!("[event][{:#?}]", event); + + // should produce a BlockEvent::Connected result for each block event + assert!(matches!(event, BlockEvent::Connected { .. })); + + // should handle and build the BlockEvent::Connected successfully + let connected = match event { + BlockEvent::Connected(header) => header, + _ => unreachable!("This test is supposed to have only connected blocks, please check why it's generating disconnected and/or errors at the moment."), + }; + + assert_eq!(block_hash, connected.block_hash()); + } +} + +#[tokio::test] +#[serial] +async fn test_block_events_stream_with_checkpoint() { + let _ = env_logger::try_init(); + + let MempoolTestClient { + bitcoind, + mariadb_database, + mempool_backend, + } = MempoolTestClient::default(); + + // generate 10 new blocks through bitcoind rpc-client + let address = &bitcoind.client.get_new_address(None, None).unwrap(); + let blocks = bitcoind.client.generate_to_address(10, address).unwrap(); + + // expected blocks are from the 4th block in the chain (3rd in the new generated blocks) + let mut expected_block_hashes = VecDeque::from(blocks[3..].to_vec()); + + // checkpoint starts in 3rd new block (index 2) + let ckpt_block_hash = *blocks.get(2).unwrap(); + let checkpoint = Some((3, ckpt_block_hash)); + + // start database (mariadb) and backend (mempool/backend) + let docker = clients::Cli::docker(); + let _mariadb = docker.run(mariadb_database); + + // there is some small delay between running the docker for mariadb database and the port being really available + std::thread::sleep(Duration::from_millis(5000)); + let mempool = docker.run(mempool_backend); + + let http_base_url = format!( + "http://{}:{}/api/v1", + HOST_IP, + mempool.get_host_port_ipv4(8999) + ); + let ws_base_url = format!("ws://{}:{}", HOST_IP, mempool.get_host_port_ipv4(8999)); + + // get block-events stream, starting from the checkpoint + let block_events = + block_events::subscribe_to_block_headers(&http_base_url, &ws_base_url, checkpoint) + .await + .unwrap(); + + // consume new blocks from block-events stream + pin_mut!(block_events); + while !expected_block_hashes.is_empty() { + let expected_hash = expected_block_hashes.pop_front().unwrap(); + let block_event = block_events.next().await.unwrap().unwrap(); + + // should produce a BlockEvent::Connected result for each block event + assert!(matches!(block_event, BlockEvent::Connected { .. })); + + // should parse the BlockEvent::Connected successfully + let connected_hash = match block_event { + BlockEvent::Connected(block) => block.block_hash(), + _ => unreachable!("This test is supposed to have only connected blocks, please check why it's generating disconnected and/or errors at the moment."), + }; + + assert_eq!(expected_hash, connected_hash); + } +} + +#[tokio::test] +#[serial] +async fn test_block_events_stream_with_reorg() { + let _ = env_logger::try_init(); + + let MempoolTestClient { + bitcoind, + mariadb_database, + mempool_backend, + } = MempoolTestClient::default(); + + // start database (mariadb) and backend (mempool/backend) + let docker = clients::Cli::docker(); + let _mariadb = docker.run(mariadb_database); + + // there is some small delay between running the docker for mariadb database and the port being really available + std::thread::sleep(Duration::from_millis(5000)); + let mempool = docker.run(mempool_backend); + + let http_base_url = format!( + "http://{}:{}/api/v1", + HOST_IP, + mempool.get_host_port_ipv4(8999) + ); + let ws_base_url = format!("ws://{}:{}", HOST_IP, mempool.get_host_port_ipv4(8999)); + + // get block-events stream + let block_events = block_events::subscribe_to_block_headers(&http_base_url, &ws_base_url, None) + .await + .unwrap(); + + // generate 5 new blocks through bitcoind rpc-client + let address = bitcoind.client.get_new_address(None, None).unwrap(); + let generated_blocks = + VecDeque::from(bitcoind.client.generate_to_address(5, &address).unwrap()); + + let mut new_blocks = generated_blocks.clone(); + + // consume new blocks from block-events stream + pin_mut!(block_events); + while !new_blocks.is_empty() { + let block_hash = new_blocks.pop_front().unwrap(); + let block_event = block_events.next().await.unwrap().unwrap(); + + // should produce a BlockEvent::Connected result for each block event + assert!(matches!(block_event, BlockEvent::Connected { .. })); + + // should parse the BlockEvent::Connected successfully + let connected_block = match block_event { + BlockEvent::Connected(block) => block, + _ => unreachable!("This test is supposed to have only connected blocks, please check why it's generating disconnected and/or errors at the moment."), + }; + assert_eq!(block_hash.to_owned(), connected_block.block_hash()); + } + + // invalidate last 2 blocks + let mut invalidated_blocks = VecDeque::new(); + for block in generated_blocks.range(3..) { + bitcoind.client.invalidate_block(block).unwrap(); + invalidated_blocks.push_front(block); + } + + // generate 2 new blocks + let address = bitcoind.client.get_new_address(None, None).unwrap(); + let mut new_blocks = VecDeque::from(bitcoind.client.generate_to_address(3, &address).unwrap()); + + // should disconnect invalidated blocks + while !invalidated_blocks.is_empty() { + let invalidated = invalidated_blocks.pop_back().unwrap(); + let block_event = block_events.next().await.unwrap().unwrap(); + + // should produce a BlockEvent::Connected result for each block event + assert!(matches!(block_event, BlockEvent::Disconnected(..))); + + // should parse the BlockEvent::Connected successfully + let disconnected = match block_event { + BlockEvent::Disconnected((_, hash)) => hash, + _ => unreachable!("This test is supposed to have only connected blocks, please check why it's generating disconnected and/or errors at the moment."), + }; + assert_eq!(invalidated.to_owned(), disconnected); + } + + // should connect the new created blocks + while !new_blocks.is_empty() { + let new_block = new_blocks.pop_front().unwrap(); + let block_event = block_events.next().await.unwrap().unwrap(); + + // should produce a BlockEvent::Connected result for each block event + assert!(matches!(block_event, BlockEvent::Connected { .. })); + + // should parse the BlockEvent::Connected successfully + let connected = match block_event { + BlockEvent::Connected(block) => block.block_hash(), + _ => unreachable!("This test is supposed to have only connected blocks, please check why it's generating disconnected and/or errors at the moment."), + }; + assert_eq!(new_block.to_owned(), connected); + } +}