Merge branch 'dev' into 'master'

[Tests] Include BTC-Relay integration tests

See merge request interlay/btc-parachain!118

crates/btc-relay/src/lib.rs

@@ -479,15 +479,15 @@ impl<T: Trait> Module<T> {
         <ChainsIndex>::get(chain_id).ok_or(Error::InvalidChainID)
     }
     /// Get the current best block hash
-    fn get_best_block() -> H256Le {
+    pub fn get_best_block() -> H256Le {
         <BestBlock>::get()
     }
     /// Check if a best block hash is set
     fn best_block_exists() -> bool {
         <BestBlock>::exists()
     }
     /// get the best block height
-    fn get_best_block_height() -> u32 {
+    pub fn get_best_block_height() -> u32 {
         <BestBlockHeight>::get()
     }
     /// Get the current chain counter

crates/btc-relay/src/mock.rs

@@ -27,6 +27,7 @@ impl_outer_event! {
 }
 
 pub type AccountId = u64;
+pub const CONFIRMATIONS: u32 = 6;
 
 // For testing the pallet, we construct most of a mock runtime. This means
 // first constructing a configuration type (`Test`) which `impl`s each of the
@@ -80,9 +81,11 @@ impl ExtBuilder {
             .build_storage::<Test>()
             .unwrap();
 
-        GenesisConfig { confirmations: 6 }
-            .assimilate_storage(&mut storage)
-            .unwrap();
+        GenesisConfig {
+            confirmations: CONFIRMATIONS,
+        }
+        .assimilate_storage(&mut storage)
+        .unwrap();
 
         sp_io::TestExternalities::from(storage)
     }

parachain/README.md

@@ -22,6 +22,20 @@ Build Wasm and native code:
 cargo build --release
 ```
 
+## Test
+
+To download the recent 100 Bitcoin blocks run:
+
+```bash
+python ./scripts/fetch_bitcoin_data.py
+```
+
+Execute tests:
+
+```bash
+cargo test --release
+```
+
 ## Run
 
 ### Single Node Development Chain
@@ -74,29 +88,3 @@ cargo run -- \
 ```
 
 Additional CLI usage options are available and may be shown by running `cargo run -- --help`.
-
-## Advanced: Generate Your Own Substrate Node Template
-
-A substrate node template is always based on a certain version of Substrate. You can inspect it by
-opening [Cargo.toml](Cargo.toml) and see the template referred to a specific Substrate commit(
-`rev` field), branch, or version.
-
-You can generate your own Substrate node-template based on a particular Substrate
-version/commit by running following commands:
-
-```bash
-# git clone from the main Substrate repo
-git clone https://github.com/paritytech/substrate.git
-cd substrate
-
-# Switch to a particular branch or commit of the Substrate repo your node-template based on
-git checkout <branch/tag/sha1>
-
-# Run the helper script to generate a node template.
-# This script compiles Substrate and takes a while to complete. It takes a relative file path
-#   from the current dir. to output the compressed node template.
-.maintain/node-template-release.sh ../node-template.tar.gz
-```
-
-Noted though you will likely get faster and more thorough support if you stick with the releases
-provided in this repository.

parachain/runtime/Cargo.toml

@@ -183,6 +183,7 @@ features= ['codec']
 mocktopus = '0.7.0'
 x-core = { path = '../../crates/x-core' }
 hex = '0.4.2'
+serde_json = "1.0"
 
 [features]
 default = ['std']

parachain/runtime/tests/bitcoin_data.rs

@@ -0,0 +1,35 @@
+use serde::Deserialize;
+use std::fs;
+use std::path::PathBuf;
+
+const ERR_FILE_NOT_FOUND: &'static str = "Testdata not found. Please run the python script under the parachain/scripts folder to obtain bitcoin blocks and transactions.";
+const ERR_JSON_FORMAT: &'static str = "JSON was not well-formatted";
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct Block {
+    pub height: u32,
+    pub hash: String,
+    pub raw_header: String,
+    pub test_txs: Vec<Transaction>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct Transaction {
+    pub txid: String,
+    pub raw_merkle_proof: String,
+}
+
+pub fn get_bitcoin_testdata() -> Vec<Block> {
+    let path_str = String::from("./tests/data/bitcoin-testdata.json");
+    let path = PathBuf::from(&path_str);
+    let abs_path = fs::canonicalize(&path).unwrap();
+    let debug_help = abs_path.as_path().to_str().unwrap();
+
+    let error_message = "\n".to_owned() + ERR_FILE_NOT_FOUND + "\n" + debug_help;
+
+    let data = fs::read_to_string(&path_str).expect(&error_message);
+
+    let test_data: Vec<Block> = serde_json::from_str(&data).expect(ERR_JSON_FORMAT);
+
+    test_data
+}

parachain/runtime/tests/mock.rs

@@ -15,6 +15,7 @@ pub use x_core::Error;
 pub const ALICE: [u8; 32] = [0u8; 32];
 pub const BOB: [u8; 32] = [1u8; 32];
 pub const CLAIRE: [u8; 32] = [2u8; 32];
+pub const CONFIRMATIONS: u32 = 6;
 
 pub type BTCRelayCall = btc_relay::Call<Runtime>;
 pub type BTCRelayEvent = btc_relay::Event;
@@ -66,12 +67,11 @@ pub fn force_issue_tokens(
     treasury::Module::<Runtime>::mint(user.into(), tokens);
 }
 
-fn assert_store_main_chain_header_event(height: u32, hash: H256Le) {
+pub fn assert_store_main_chain_header_event(height: u32, hash: H256Le) {
     let store_event = Event::btc_relay(BTCRelayEvent::StoreMainChainHeader(height, hash));
     let events = SystemModule::events();
 
     // store only main chain header
-    // events.iter().for_each(|a| println!("{:?}",a.event));
     assert!(events.iter().any(|a| a.event == store_event));
 }
 
@@ -169,10 +169,14 @@ pub fn generate_transaction_and_mine(
     (tx_id, tx_block_height, bytes_proof, raw_tx)
 }
 
+#[allow(dead_code)]
 pub type SecurityModule = security::Module<Runtime>;
+#[allow(dead_code)]
 pub type SystemModule = system::Module<Runtime>;
 
+#[allow(dead_code)]
 pub type VaultRegistryCall = vault_registry::Call<Runtime>;
+#[allow(dead_code)]
 pub type OracleCall = exchange_rate_oracle::Call<Runtime>;
 
 pub struct ExtBuilder;
@@ -209,9 +213,11 @@ impl ExtBuilder {
         .assimilate_storage(&mut storage)
         .unwrap();
 
-        btc_relay::GenesisConfig { confirmations: 6 }
-            .assimilate_storage(&mut storage)
-            .unwrap();
+        btc_relay::GenesisConfig {
+            confirmations: CONFIRMATIONS,
+        }
+        .assimilate_storage(&mut storage)
+        .unwrap();
 
         vault_registry::GenesisConfig {
             secure_collateral_threshold: 100000,

parachain/runtime/tests/test_btcrelay.rs

@@ -0,0 +1,51 @@
+mod bitcoin_data;
+mod mock;
+
+use bitcoin_data::get_bitcoin_testdata;
+use mock::*;
+
+#[test]
+fn integration_test_submit_block_headers_and_verify_transaction_inclusion() {
+    ExtBuilder::build().execute_with(|| {
+        // load blocks with transactions
+        let test_data = get_bitcoin_testdata();
+
+        let mut init = false;
+        // store all block headers
+        for block in test_data.iter() {
+            let raw_header = RawBlockHeader::from_hex(&block.raw_header).unwrap();
+            if init == false {
+                assert_ok!(BTCRelayCall::initialize(raw_header, block.height)
+                    .dispatch(origin_of(account_of(ALICE))));
+                init = true;
+            } else {
+                assert_ok!(BTCRelayCall::store_block_header(raw_header)
+                    .dispatch(origin_of(account_of(ALICE))));
+
+                assert_store_main_chain_header_event(
+                    block.height,
+                    H256Le::from_hex_be(&block.hash),
+                );
+            }
+        }
+        // verify all transaction that have enough confirmations
+        let current_height = btc_relay::Module::<Runtime>::get_best_block_height();
+        for block in test_data.iter() {
+            if block.height < current_height - CONFIRMATIONS {
+                for tx in &block.test_txs {
+                    let txid = H256Le::from_hex_be(&tx.txid);
+                    let raw_merkle_proof =
+                        hex::decode(&tx.raw_merkle_proof).expect("Error parsing merkle proof");
+                    assert_ok!(BTCRelayCall::verify_transaction_inclusion(
+                        txid,
+                        block.height,
+                        raw_merkle_proof,
+                        CONFIRMATIONS,
+                        false
+                    )
+                    .dispatch(origin_of(account_of(ALICE))));
+                }
+            }
+        }
+    })
+}

parachain/scripts/fetch_bitcoin_data.py

@@ -0,0 +1,131 @@
+import requests
+import json
+import random
+import os
+
+DIRNAME = os.path.dirname(__file__)
+TESTDATA_DIR = os.path.join(DIRNAME, "..", "runtime", "tests", "data")
+TESTDATA_FILE = os.path.join(TESTDATA_DIR, "bitcoin-testdata.json")
+BASE_URL = "https://blockstream.info/api"
+
+def query(uri):
+    url = BASE_URL + uri
+    response = requests.get(url)
+
+    if (response.ok):
+        return response
+    else:
+        response.raise_for_status()
+
+def query_text(uri):
+    response = query(uri)
+    return response.text
+
+def query_json(uri):
+    response = query(uri)
+    return response.json()
+
+def query_binary(uri):
+    url = BASE_URL + uri
+
+    with requests.get(url, stream=True) as response:
+        if (response.ok):
+            # hacky way to get only the 80 block header bytes
+            # the raw block heade endpoint gives the block header
+            # plus the number of txs and the raw txs
+            # see https://github.com/Blockstream/esplora/issues/171
+            if '/block/' in url:
+                raw_header = response.raw.read(80)
+                assert(len(raw_header) == 80)
+                return raw_header.hex()
+            else:
+                return response.content.decode('utf-8')
+        else:
+            response.raise_for_status()
+
+def get_tip_height():
+    uri = "/blocks/tip/height"
+    return query_json(uri)
+
+def get_raw_header(blockhash):
+    uri = "/block/{}/raw".format(blockhash)
+    return query_binary(uri)
+
+def get_block_hash(height):
+    uri = "/block-height/{}".format(height)
+    return query_text(uri)
+
+def get_block_txids(blockhash):
+    uri = "/block/{}/txids".format(blockhash)
+    return query_json(uri)
+
+def get_raw_merkle_proof(txid):
+    uri = "/tx/{}/merkleblock-proof".format(txid)
+    return query_binary(uri)
+
+def get_testdata(number, tip_height):
+    # query number of blocks
+    blocks = []
+    for i in range(tip_height - number, tip_height):
+        blockhash = get_block_hash(i)
+        print("Getting block at height {} with hash {}".format(i, blockhash))
+        raw_header = get_raw_header(blockhash)
+        # get the txids in the block
+        txids = get_block_txids(blockhash)
+        # select two txids randomly for testing
+        test_txids = random.sample(txids, 2)
+        test_txs = []
+        # get the tx merkle proof
+        for txid in test_txids:
+            raw_merkle_proof = get_raw_merkle_proof(txid)
+            tx = {
+                'txid': txid,
+                'raw_merkle_proof': raw_merkle_proof,
+            }
+            test_txs.append(tx)
+
+        block = {
+            'height': i,
+            'hash': blockhash,
+            'raw_header': raw_header,
+            'test_txs': test_txs,
+        }
+        blocks.append(block)
+    return blocks
+
+def overwrite_testdata(blocks):
+    with open(TESTDATA_FILE, 'w', encoding='utf-8') as f:
+        json.dump(blocks, f, ensure_ascii=False, indent=4)
+
+def read_testdata():
+    blocks = []
+    try:
+        with open(TESTDATA_FILE) as data:
+            blocks = json.load(data)
+    except:
+        print("No existing testdata found")
+    return blocks
+
+def main():
+    max_num_blocks = 100
+    number_blocks = max_num_blocks
+    # get current tip of Bitcoin blockchain
+    tip_height = get_tip_height()
+    blocks = read_testdata()
+    if blocks:
+        if blocks[-1]['height'] == tip_height:
+            print("Latest blocks already downloaded")
+            return
+        else:
+            ## download new blocks
+            delta = tip_height - blocks[-1]["height"]
+            number_blocks = delta if delta <= max_num_blocks else max_num_blocks
+
+    new_blocks = get_testdata(number_blocks, tip_height)
+    blocks = blocks + new_blocks
+    # only store max_num_blocks
+    blocks = blocks[-max_num_blocks:]
+    overwrite_testdata(blocks)
+
+if __name__ == "__main__":
+    main()