Add full block data for testing and benchmarking (#263)

- Added the blocks 866,342 and 367,891 to `testdata`, as well as the spent UTXOs in each block
- `get_validation_flags` doesn't need to ask the chainstore for the block hash
- Changed the name of the inner `validate_block` to `validate_block_no_acc`
- Update README.md

README.md

@@ -23,6 +23,7 @@ If you want to use `libfloresta` to build your own Bitcoin application, you can
     - [Wallet](#wallet)
   - [Running the tests](#running-the-tests)
     - [Requirements](#requirements)
+  - [Running Benchmarks](#running-benchmarks)
   - [Fuzzing](#fuzzing)
   - [Contributing](#contributing)
   - [Using Nix](#using-nix)
@@ -220,18 +221,40 @@ Once you have a transaction cached in your watch-only, you can use either the rp
 cargo build
 ```
 
-There's a set of unit tests that you can run with
+There's a set of tests that you can run with:
 ```bash
 cargo test
 ```
 
-There's also a set of functional tests that you can run with
+For the full test suite, including long-running tests, use:
+
+```bash
+cargo test --release
+```
+
+There's also a set of functional tests that you can run with:
 
 ```bash
 pip3 install -r tests/requirements.txt
 python tests/run_tests.py
 ```
 
+### Running Benchmarks
+
+Floresta uses `criterion.rs` for benchmarking. You can run the default set of benchmarks with:
+
+```bash
+cargo bench
+```
+
+By default, benchmarks that are resource-intensive are excluded to allow for quicker testing. If you'd like to include all benchmarks, use the following command:
+
+```bash
+EXPENSIVE_BENCHES=1 cargo bench
+```
+
+> **Note**: Running with `EXPENSIVE_BENCHES=1` enables the full benchmark suite, which will take several minutes to complete.
+
 ### Fuzzing
 
 This project uses `cargo-fuzz` (libfuzzer) for fuzzing, you can run a fuzz target with:

crates/floresta-chain/benches/chain_state_bench.rs

@@ -1,13 +1,18 @@
 use std::collections::HashMap;
+use std::fs::File;
 use std::io::Cursor;
 
 use bitcoin::block::Header as BlockHeader;
 use bitcoin::consensus::deserialize;
 use bitcoin::consensus::Decodable;
 use bitcoin::Block;
+use bitcoin::OutPoint;
+use bitcoin::TxOut;
 use criterion::criterion_group;
 use criterion::criterion_main;
+use criterion::BatchSize;
 use criterion::Criterion;
+use criterion::SamplingMode;
 use floresta_chain::pruned_utreexo::UpdatableChainstate;
 use floresta_chain::AssumeValidArg;
 use floresta_chain::ChainState;
@@ -27,7 +32,7 @@ fn read_blocks_txt() -> Vec<Block> {
     blocks
 }
 
-pub fn setup_test_chain<'a>(
+fn setup_test_chain<'a>(
     network: Network,
     assume_valid_arg: AssumeValidArg,
 ) -> ChainState<KvChainStore<'a>> {
@@ -36,6 +41,31 @@ pub fn setup_test_chain<'a>(
     ChainState::new(chainstore, network, assume_valid_arg)
 }
 
+fn decode_block_and_inputs(
+    block_file: File,
+    stxos_file: File,
+) -> (Block, HashMap<OutPoint, TxOut>) {
+    let block_bytes = zstd::decode_all(block_file).unwrap();
+    let block: Block = deserialize(&block_bytes).unwrap();
+
+    // Get txos spent in the block
+    let stxos_bytes = zstd::decode_all(stxos_file).unwrap();
+    let mut stxos: Vec<TxOut> =
+        serde_json::from_slice(&stxos_bytes).expect("Failed to deserialize JSON");
+
+    let inputs = block
+        .txdata
+        .iter()
+        .skip(1) // Skip the coinbase transaction
+        .flat_map(|tx| &tx.input)
+        .map(|txin| (txin.previous_output, stxos.remove(0)))
+        .collect();
+
+    assert!(stxos.is_empty(), "Moved all stxos to the inputs map");
+
+    (block, inputs)
+}
+
 fn accept_mainnet_headers_benchmark(c: &mut Criterion) {
     // Accepts the first 10235 mainnet headers
     let file = include_bytes!("../testdata/headers.zst");
@@ -91,10 +121,59 @@ fn connect_blocks_benchmark(c: &mut Criterion) {
     });
 }
 
+fn validate_full_block_benchmark(c: &mut Criterion) {
+    let block_file = File::open("./testdata/block_866342/raw.zst").unwrap();
+    let stxos_file = File::open("./testdata/block_866342/spent_txos.zst").unwrap();
+    let (block, inputs) = decode_block_and_inputs(block_file, stxos_file);
+
+    let chain = setup_test_chain(Network::Bitcoin, AssumeValidArg::Disabled);
+
+    c.bench_function("validate_block_866342", |b| {
+        b.iter_batched(
+            || inputs.clone(),
+            |inputs| chain.validate_block_no_acc(&block, 866342, inputs).unwrap(),
+            BatchSize::LargeInput,
+        )
+    });
+}
+
+fn validate_many_inputs_block_benchmark(c: &mut Criterion) {
+    if std::env::var("EXPENSIVE_BENCHES").is_err() {
+        println!(
+            "validate_many_inputs_block_benchmark ... \x1b[33mskipped\x1b[0m\n\
+            > Set EXPENSIVE_BENCHES=1 to include this benchmark\n"
+        );
+
+        return;
+    }
+
+    let block_file = File::open("./testdata/block_367891/raw.zst").unwrap();
+    let stxos_file = File::open("./testdata/block_367891/spent_txos.zst").unwrap();
+    let (block, inputs) = decode_block_and_inputs(block_file, stxos_file);
+
+    let chain = setup_test_chain(Network::Bitcoin, AssumeValidArg::Disabled);
+
+    // Create a group with the lowest possible sample size, as validating this block is very slow
+    let mut group = c.benchmark_group("validate_block_367891");
+    group.sampling_mode(SamplingMode::Flat);
+    group.sample_size(10);
+
+    group.bench_function("validate_block_367891", |b| {
+        b.iter_batched(
+            || inputs.clone(),
+            |inputs| chain.validate_block_no_acc(&block, 367891, inputs).unwrap(),
+            BatchSize::LargeInput,
+        )
+    });
+    group.finish();
+}
+
 criterion_group!(
     benches,
     accept_mainnet_headers_benchmark,
     accept_headers_benchmark,
-    connect_blocks_benchmark
+    connect_blocks_benchmark,
+    validate_full_block_benchmark,
+    validate_many_inputs_block_benchmark
 );
 criterion_main!(benches);

crates/floresta-chain/src/pruned_utreexo/chain_state.rs

@@ -130,13 +130,9 @@ impl<PersistedState: ChainStore> ChainState<PersistedState> {
     }
     #[cfg(feature = "bitcoinconsensus")]
     /// Returns the validation flags, given the current block height
-    fn get_validation_flags(&self, height: u32) -> c_uint {
+    fn get_validation_flags(&self, height: u32, hash: BlockHash) -> c_uint {
         let chains_params = &read_lock!(self).consensus.parameters;
-        let hash = read_lock!(self)
-            .chainstore
-            .get_block_hash(height)
-            .unwrap()
-            .unwrap();
+
         if let Some(flag) = chains_params.exceptions.get(&hash) {
             return *flag;
         }
@@ -709,7 +705,12 @@ impl<PersistedState: ChainStore> ChainState<PersistedState> {
         }
         last_block.target()
     }
-    fn validate_block(
+    /// Validates the block without checking whether the inputs are present in the UTXO set. This
+    /// function contains the core validation logic.
+    ///
+    /// The methods `BlockchainInterface::validate_block` and `UpdatableChainstate::connect_block`
+    /// call this and additionally verify the inclusion proof (i.e., they perform full validation).
+    pub fn validate_block_no_acc(
         &self,
         block: &Block,
         height: u32,
@@ -737,7 +738,7 @@ impl<PersistedState: ChainStore> ChainState<PersistedState> {
         let subsidy = read_lock!(self).consensus.get_subsidy(height);
         let verify_script = self.verify_script(height);
         #[cfg(feature = "bitcoinconsensus")]
-        let flags = self.get_validation_flags(height);
+        let flags = self.get_validation_flags(height, block.header.block_hash());
         #[cfg(not(feature = "bitcoinconsensus"))]
         let flags = 0;
         Consensus::verify_block_transactions(
@@ -803,7 +804,7 @@ impl<PersistedState: ChainStore> BlockchainInterface for ChainState<PersistedSta
             .get_block_height(&block.block_hash())?
             .ok_or(BlockchainError::BlockNotPresent)?;
 
-        self.validate_block(block, height, inputs)
+        self.validate_block_no_acc(block, height, inputs)
     }
 
     fn get_block_locator_for_tip(&self, tip: BlockHash) -> Result<Vec<BlockHash>, BlockchainError> {
@@ -1050,7 +1051,7 @@ impl<PersistedState: ChainStore> UpdatableChainstate for ChainState<PersistedSta
             return Ok(height);
         }
 
-        self.validate_block(block, height, inputs)?;
+        self.validate_block_no_acc(block, height, inputs)?;
         let acc = Consensus::update_acc(&self.acc(), block, height, proof, del_hashes)?;
 
         self.update_view(height, &block.header, acc)?;
@@ -1278,6 +1279,7 @@ mod test {
     extern crate std;
     use core::str::FromStr;
     use std::format;
+    use std::fs::File;
     use std::io::Cursor;
     use std::vec::Vec;
 
@@ -1287,6 +1289,8 @@ mod test {
     use bitcoin::hashes::hex::FromHex;
     use bitcoin::Block;
     use bitcoin::BlockHash;
+    use bitcoin::OutPoint;
+    use bitcoin::TxOut;
     use rand::Rng;
     use rustreexo::accumulator::proof::Proof;
 
@@ -1301,7 +1305,7 @@ mod test {
     use crate::KvChainStore;
     use crate::Network;
 
-    pub fn setup_test_chain<'a>(
+    fn setup_test_chain<'a>(
         network: Network,
         assume_valid_arg: AssumeValidArg,
     ) -> ChainState<KvChainStore<'a>> {
@@ -1310,6 +1314,68 @@ mod test {
         ChainState::new(chainstore, network, assume_valid_arg)
     }
 
+    fn decode_block_and_inputs(
+        block_file: File,
+        stxos_file: File,
+    ) -> (Block, HashMap<OutPoint, TxOut>) {
+        let block_bytes = zstd::decode_all(block_file).unwrap();
+        let block: Block = deserialize(&block_bytes).unwrap();
+
+        // Get txos spent in the block
+        let stxos_bytes = zstd::decode_all(stxos_file).unwrap();
+        let mut stxos: Vec<TxOut> =
+            serde_json::from_slice(&stxos_bytes).expect("Failed to deserialize JSON");
+
+        let inputs = block
+            .txdata
+            .iter()
+            .skip(1) // Skip the coinbase transaction
+            .flat_map(|tx| &tx.input)
+            .map(|txin| (txin.previous_output, stxos.remove(0)))
+            .collect();
+
+        assert!(stxos.is_empty(), "Moved all stxos to the inputs map");
+
+        (block, inputs)
+    }
+
+    #[test]
+    #[cfg_attr(debug_assertions, ignore = "this test is very slow in debug mode")]
+    fn test_validate_many_inputs_block() {
+        let block_file = File::open("./testdata/block_367891/raw.zst").unwrap();
+        let stxos_file = File::open("./testdata/block_367891/spent_txos.zst").unwrap();
+        let (block, inputs) = decode_block_and_inputs(block_file, stxos_file);
+
+        assert_eq!(
+            block.block_hash(),
+            BlockHash::from_str("000000000000000012ea0ca9579299ec120e3f57e7c309216884872592b29970")
+                .unwrap(),
+        );
+
+        // Check whether the block validation passes or not
+        let chain = setup_test_chain(Network::Bitcoin, AssumeValidArg::Disabled);
+        chain
+            .validate_block_no_acc(&block, 367891, inputs)
+            .expect("Block must be valid");
+    }
+    #[test]
+    fn test_validate_full_block() {
+        let block_file = File::open("./testdata/block_866342/raw.zst").unwrap();
+        let stxos_file = File::open("./testdata/block_866342/spent_txos.zst").unwrap();
+        let (block, inputs) = decode_block_and_inputs(block_file, stxos_file);
+
+        assert_eq!(
+            block.block_hash(),
+            BlockHash::from_str("000000000000000000014ce9ba7c6760053c3c82ce6ab43d60afb101d3c8f1f1")
+                .unwrap(),
+        );
+
+        // Check whether the block validation passes or not
+        let chain = setup_test_chain(Network::Bitcoin, AssumeValidArg::Disabled);
+        chain
+            .validate_block_no_acc(&block, 866342, inputs)
+            .expect("Block must be valid");
+    }
     #[test]
     fn accept_mainnet_headers() {
         // Accepts the first 10235 mainnet headers