fix(bridge): support decoding final era1 file & fix broken merkle pro…

…ofs for partial epochs (#1291) * fix(bridge): support decoding final era1 file * fix: broken merkle proof generation for partial epochs (#12)
ethereum · May 21, 2024 · 94a7aea · 94a7aea
1 parent 6dc8422
commit 94a7aea
Show file tree

Hide file tree

Showing 4 changed files with 102 additions and 40 deletions.
diff --git a/portal-bridge/src/types/era1.rs b/portal-bridge/src/types/era1.rs
@@ -42,9 +42,10 @@ impl Era1 {
     /// deserialized era1 object in memory.
     pub fn iter_tuples(raw_era1: Vec<u8>) -> impl Iterator<Item = BlockTuple> {
         let file = E2StoreFile::deserialize(&raw_era1).expect("invalid era1 file");
-        let block_index = BlockIndexEntry::try_from(&file.entries[32770])
-            .expect("invalid block index entry")
-            .block_index;
+        let block_index =
+            BlockIndexEntry::try_from(file.entries.last().expect("missing block index entry"))
+                .expect("invalid block index entry")
+                .block_index;
         (0..block_index.count).map(move |i| {
             let mut entries: [Entry; 4] = Default::default();
             for (j, entry) in entries.iter_mut().enumerate() {
@@ -66,21 +67,25 @@ impl Era1 {
     pub fn deserialize(buf: &[u8]) -> anyhow::Result<Self> {
         let file = E2StoreFile::deserialize(buf)?;
         ensure!(
-            file.entries.len() == ERA1_ENTRY_COUNT,
+            // era1 file #0-1895 || era1 file #1896
+            file.entries.len() == ERA1_ENTRY_COUNT || file.entries.len() == 21451,
             "invalid era1 file: incorrect entry count"
         );
         let version = VersionEntry::try_from(&file.entries[0])?;
+        let block_index =
+            BlockIndexEntry::try_from(file.entries.last().expect("missing block index entry"))?;
         let mut block_tuples = vec![];
-        for count in 0..BLOCK_TUPLE_COUNT {
+        let block_tuple_count = block_index.block_index.count as usize;
+        for count in 0..block_tuple_count {
             let mut entries: [Entry; 4] = Default::default();
             for (i, entry) in entries.iter_mut().enumerate() {
                 *entry = file.entries[count * 4 + i + 1].clone();
             }
             let block_tuple = BlockTuple::try_from(&entries)?;
             block_tuples.push(block_tuple);
         }
-        let accumulator = AccumulatorEntry::try_from(&file.entries[32769])?;
-        let block_index = BlockIndexEntry::try_from(&file.entries[32770])?;
+        let accumulator_index = (block_tuple_count * 4) + 1;
+        let accumulator = AccumulatorEntry::try_from(&file.entries[accumulator_index])?;
         Ok(Self {
             version,
             block_tuples,
@@ -104,7 +109,8 @@ impl Era1 {
         entries.push(block_index_entry);
         let file = E2StoreFile { entries };
         ensure!(
-            file.entries.len() == ERA1_ENTRY_COUNT,
+            // era1 file #0-1895 || era1 file #1896
+            file.entries.len() == ERA1_ENTRY_COUNT || file.entries.len() == 21451,
             "invalid era1 file: incorrect entry count"
         );
         let file_length = file.length();
@@ -398,15 +404,17 @@ impl TryFrom<&Entry> for BlockIndexEntry {
             "invalid block index entry: incorrect header type"
         );
         ensure!(
-            entry.header.length == 65552,
+            // era1 file #0-1895 || era1 file #1896
+            entry.header.length == 65552 || entry.header.length == 42912,
             "invalid block index entry: incorrect header length"
         );
         ensure!(
             entry.header.reserved == 0,
             "invalid block index entry: incorrect header reserved bytes"
         );
         ensure!(
-            entry.value.len() == 65552,
+            // era1 file #0-1895 || era1 file #1896
+            entry.value.len() == 65552 || entry.value.len() == 42912,
             "invalid block index entry: incorrect value length"
         );
         Ok(Self {
@@ -434,7 +442,7 @@ impl TryInto<Entry> for BlockIndexEntry {
 #[derive(Clone, Eq, PartialEq, Debug)]
 struct BlockIndex {
     starting_number: u64,
-    indices: [u64; BLOCK_TUPLE_COUNT],
+    indices: Vec<u64>,
     count: u64,
 }
 
@@ -443,12 +451,13 @@ impl TryFrom<Entry> for BlockIndex {
 
     fn try_from(entry: Entry) -> Result<Self, Self::Error> {
         let starting_number = u64::from_le_bytes(entry.value[0..8].try_into()?);
-        let mut indices = [0u64; BLOCK_TUPLE_COUNT];
+        let block_tuple_count = (entry.value.len() - 16) / 8;
+        let mut indices = vec![0; block_tuple_count];
         for (i, index) in indices.iter_mut().enumerate() {
             *index = u64::from_le_bytes(entry.value[(i * 8 + 8)..(i * 8 + 16)].try_into()?);
         }
         let count = u64::from_le_bytes(
-            entry.value[(BLOCK_TUPLE_COUNT * 8 + 8)..(BLOCK_TUPLE_COUNT * 8 + 16)].try_into()?,
+            entry.value[(block_tuple_count * 8 + 8)..(block_tuple_count * 8 + 16)].try_into()?,
         );
         Ok(Self {
             starting_number,
@@ -478,5 +487,7 @@ mod tests {
         let actual = era1.write().unwrap();
         let expected = fs::read(path).unwrap();
         assert_eq!(expected, actual);
+        let era1_raw_bytes = fs::read(path).unwrap();
+        let _block_tuples: Vec<BlockTuple> = Era1::iter_tuples(era1_raw_bytes).collect();
     }
 }
diff --git a/trin-validation/src/accumulator.rs b/trin-validation/src/accumulator.rs
@@ -1,4 +1,4 @@
-use alloy_primitives::B256;
+use alloy_primitives::{B256, U256};
 use std::path::PathBuf;
 
 use anyhow::anyhow;
@@ -155,37 +155,51 @@ impl PreMergeAccumulator {
         }
 
         // Create a merkle tree from epoch accumulator.
-        // To construct a valid proof for the header hash, we add both the hash and the total
-        // difficulty as individual leaves for each header record. This will ensure that the
-        // total difficulty is included as the first element in the proof.
-        let mut leaves = vec![];
-        // iterate over every header record in the epoch acc
-        for record in epoch_acc.into_iter() {
-            // add the block hash as a leaf
-            leaves.push(record.block_hash);
-            // convert total difficulty to B256
-            let leaf = B256::from_slice(record.total_difficulty.as_le_slice());
-            // add the total difficulty as a leaf
-            leaves.push(leaf);
-        }
+        // To construct a valid proof for the header hash, we add a leaf of
+        // hash(header, total difficulty) for each header record at a depth of 13.
+        // This must be done to support generating valid proofs for partial
+        // epochs, as opposed to adding the header and total difficulty as individual
+        // leaves on a tree of depth 14.
+        //
+        // Then we re-insert the total difficulty as the first element
+        // in the proof to be able to prove the header hash.
+        //
+        // convert total difficulty to B256
+        let header_difficulty = B256::from(header_record.total_difficulty.to_le_bytes());
+        // calculate hash of the header record
+        let header_record_hash = B256::from_slice(&eth2_hashing::hash32_concat(
+            header_record.block_hash.as_slice(),
+            header_difficulty.as_slice(),
+        ));
+        // iterate over every header record in the epoch acc to create the leaves
+        let leaves = epoch_acc
+            .iter()
+            .map(|record| {
+                B256::from_slice(&eth2_hashing::hash32_concat(
+                    record.block_hash.as_slice(),
+                    record.total_difficulty.as_le_slice(),
+                ))
+            })
+            .collect::<Vec<B256>>();
         // Create the merkle tree from leaves
-        let merkle_tree = MerkleTree::create(&leaves, 14);
-
-        // Multiply hr_index by 2 b/c we're now using a depth of 14 rather than the original 13
-        let hr_index = hr_index * 2;
+        let merkle_tree = MerkleTree::create(&leaves, 13);
 
         // Generating the proof for the value at hr_index (leaf)
         let (leaf, mut proof) = merkle_tree
-            .generate_proof(hr_index, 14)
+            .generate_proof(hr_index, 13)
             .map_err(|err| anyhow!("Unable to generate proof for given index: {err:?}"))?;
-        // Validate that the value the proof is for (leaf) is the header hash
-        assert_eq!(leaf, header.hash());
+
+        // Validate that the value the proof is for (leaf) == hash(header, total_difficulty)
+        assert_eq!(leaf, header_record_hash);
+
+        // Re-insert the total difficulty as the first element in the proof
+        proof.insert(0, header_difficulty);
 
         // Add the be encoded EPOCH_SIZE to proof to comply with ssz merkleization spec
         // https://github.com/ethereum/consensus-specs/blob/dev/ssz/merkle-proofs.md#ssz-object-to-index
-        proof.push(B256::from_slice(&hex_decode(
-            "0x0020000000000000000000000000000000000000000000000000000000000000",
-        )?));
+        let epoch_size = B256::from(U256::from(epoch_acc.len()).to_le_bytes());
+        proof.push(epoch_size);
+
         let final_proof: [B256; 15] = proof
             .try_into()
             .map_err(|_| anyhow!("Invalid proof length."))?;

diff --git a/.../assets/epoch_accs/0xe6ebe562c89bc8ecb94dc9b2889a27a816ec05d3d6bd1625acad72227071e721.bin b/.../assets/epoch_accs/0xe6ebe562c89bc8ecb94dc9b2889a27a816ec05d3d6bd1625acad72227071e721.bin
diff --git a/trin-validation/src/header_validator.rs b/trin-validation/src/header_validator.rs
@@ -237,15 +237,18 @@ mod test {
     use alloy_rlp::Decodable;
     use rstest::*;
     use serde_json::{json, Value};
-    use ssz::Decode;
+    use ssz::{Decode, Encode};
     use tokio::sync::mpsc;
     use tree_hash::TreeHash;
 
     use crate::constants::DEFAULT_PRE_MERGE_ACC_HASH;
     use ethportal_api::{
         types::{
-            execution::header_with_proof::{
-                BlockHeaderProof, HeaderWithProof, PreMergeAccumulatorProof, SszNone,
+            execution::{
+                accumulator::EpochAccumulator,
+                header_with_proof::{
+                    BlockHeaderProof, HeaderWithProof, PreMergeAccumulatorProof, SszNone,
+                },
             },
             jsonrpc::{endpoints::HistoryEndpoint, request::HistoryJsonRpcRequest},
         },
@@ -309,6 +312,32 @@ mod test {
         header_validator.validate_header_with_proof(&hwp).unwrap();
     }
 
+    #[rstest]
+    #[case(HEADER_RLP_15_537_392, HWP_TEST_VECTOR_15_537_392, 15_537_392)]
+    #[case(HEADER_RLP_15_537_393, HWP_TEST_VECTOR_15_537_393, 15_537_393)]
+    fn generate_and_verify_header_with_proofs_from_partial_epoch(
+        #[case] header_rlp: &str,
+        #[case] test_vector: &str,
+        #[case] block_number: u64,
+    ) {
+        let header = Header::decode(&mut hex_decode(header_rlp).unwrap().as_slice()).unwrap();
+        assert_eq!(header.number, block_number);
+        let epoch_acc_bytes = fs::read("./src/assets/epoch_accs/0xe6ebe562c89bc8ecb94dc9b2889a27a816ec05d3d6bd1625acad72227071e721.bin").unwrap();
+        let epoch_acc = EpochAccumulator::from_ssz_bytes(&epoch_acc_bytes).unwrap();
+        assert_eq!(epoch_acc.len(), 5362);
+        let proof = PreMergeAccumulator::construct_proof(&header, &epoch_acc).unwrap();
+        assert_eq!(proof.len(), 15);
+        let header_with_proof = HeaderWithProof {
+            header,
+            proof: BlockHeaderProof::PreMergeAccumulatorProof(PreMergeAccumulatorProof { proof }),
+        };
+        HeaderValidator::new()
+            .validate_header_with_proof(&header_with_proof)
+            .unwrap();
+        let encoded_hwp = hex_encode(header_with_proof.as_ssz_bytes());
+        assert_eq!(encoded_hwp, test_vector);
+    }
+
     #[tokio::test]
     async fn invalidate_invalid_proofs() {
         let header_validator = get_mainnet_header_validator();
@@ -554,4 +583,12 @@ mod test {
             }
         }
     }
+
+    const HEADER_RLP_15_537_392: &str = "0xf90218a02f1dc309c7cc0a5a2e3b3dd9315fea0ffbc53c56f9237f3ca11b20de0232f153a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794ea674fdde714fd979de3edf0f56aa9716b898ec8a0fee48a40a2765ab31fcd06ab6956341d13dc2c4b9762f2447aa425bb1c089b30a082864b3a65d1ac1917c426d48915dca0fc966fbf3f30fd051659f35dc3fd9be1a013c10513b52358022f800e2f9f1c50328798427b1b4a1ebbbd20b7417fb9719db90100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff872741c5e4f6c39283ed14f08401c9c3808401c9a028846322c95c8f617369612d65617374322d31763932a02df332ffb74ecd15c9873d3f6153b878e1c514495dfb6e89ad88e574582b02a488232b0043952c93d98508fb17c6ee";
+    const HEADER_RLP_15_537_393: &str = "0xf9021ba02b3ea3cd4befcab070812443affb08bf17a91ce382c714a536ca3cacab82278ba01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794829bd824b016326a401d083b33d092293333a830a04919dafa6ac8becfbbd0c2808f6c9511a057c21e42839caff5dfb6d3ef514951a0dd5eec02b019ff76e359b09bfa19395a2a0e97bc01e70d8d5491e640167c96a8a0baa842cfd552321a9c2450576126311e071680a1258032219c6490b663c1dab8b90100000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000080000000000000000000000000000000000000000000000000200000000000000000008000000000040000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000084000000000010020000000000000000000000000000000000020000000200000000200000000000000000000000000000000000000000400000000000000000000000008727472e1db3626a83ed14f18401c9c3808401c9a205846322c96292e4b883e5bda9e7a59ee4bb99e9b1bc460021a04cbec03dddd4b939730a7fe6048729604d4266e82426d472a2b2024f3cc4043f8862a3ee77461d4fc9850a1a4e5f06";
+
+    // Test vector for ssz encoded header with proof: BlockNumber 15537392
+    const HWP_TEST_VECTOR_15_537_392: &str = "0x0800000023020000f90218a02f1dc309c7cc0a5a2e3b3dd9315fea0ffbc53c56f9237f3ca11b20de0232f153a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794ea674fdde714fd979de3edf0f56aa9716b898ec8a0fee48a40a2765ab31fcd06ab6956341d13dc2c4b9762f2447aa425bb1c089b30a082864b3a65d1ac1917c426d48915dca0fc966fbf3f30fd051659f35dc3fd9be1a013c10513b52358022f800e2f9f1c50328798427b1b4a1ebbbd20b7417fb9719db90100ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff872741c5e4f6c39283ed14f08401c9c3808401c9a028846322c95c8f617369612d65617374322d31763932a02df332ffb74ecd15c9873d3f6153b878e1c514495dfb6e89ad88e574582b02a488232b0043952c93d98508fb17c6ee01eb461cb6348eeed7700c00000000000000000000000000000000000000000000db21cba827f968eeadeee502025f01cbcfcf20e0fafee4ecb5a877bb7ae218f3f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4bdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71c78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c50cb3e7542c17fa65faecc0fd911b9069cddb9e52114cfff2b06d3411267c78e774ec19c2ab8f61bac6da8835c6c00c74a7c07f3804b626d34563458952c589929776e586fafb3d3f001a11c9aaa3986752d147fa90faea0f463261f2ed9b8711529aacd1c137bffadc6bdf388b668118d469207288ca3681c895855c930f10b26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e189cc2a1734369d3d379aa5cedb69c2709ec75233c63f3b84bf9aa3ae048bd8cd6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220f1f11deb5763b3c38cb34c1344a6a02b6ccb1079b746545b7318057a8a5dbd2ff214000000000000000000000000000000000000000000000000000000000000";
+    // Test vector for ssz encoded header with proof: BlockNumber 15537393
+    const HWP_TEST_VECTOR_15_537_393: &str = "0x0800000026020000f9021ba02b3ea3cd4befcab070812443affb08bf17a91ce382c714a536ca3cacab82278ba01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794829bd824b016326a401d083b33d092293333a830a04919dafa6ac8becfbbd0c2808f6c9511a057c21e42839caff5dfb6d3ef514951a0dd5eec02b019ff76e359b09bfa19395a2a0e97bc01e70d8d5491e640167c96a8a0baa842cfd552321a9c2450576126311e071680a1258032219c6490b663c1dab8b90100000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000080000000000000000000000000000000000000000000000000200000000000000000008000000000040000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000084000000000010020000000000000000000000000000000000020000000200000000200000000000000000000000000000000000000000400000000000000000000000008727472e1db3626a83ed14f18401c9c3808401c9a205846322c96292e4b883e5bda9e7a59ee4bb99e9b1bc460021a04cbec03dddd4b939730a7fe6048729604d4266e82426d472a2b2024f3cc4043f8862a3ee77461d4fc9850a1a4e5f060155a9cfd362d515d8700c00000000000000000000000000000000000000000000aabfef675d3157c2cf8964505418cf314ab0ce8f4a8e742ae33fb067d449db39f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4bdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71c78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c50cb3e7542c17fa65faecc0fd911b9069cddb9e52114cfff2b06d3411267c78e774ec19c2ab8f61bac6da8835c6c00c74a7c07f3804b626d34563458952c589929776e586fafb3d3f001a11c9aaa3986752d147fa90faea0f463261f2ed9b8711529aacd1c137bffadc6bdf388b668118d469207288ca3681c895855c930f10b26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e189cc2a1734369d3d379aa5cedb69c2709ec75233c63f3b84bf9aa3ae048bd8cd6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220f1f11deb5763b3c38cb34c1344a6a02b6ccb1079b746545b7318057a8a5dbd2ff214000000000000000000000000000000000000000000000000000000000000";
 }