feat(epoch-sync): adding EpochSyncInfo validation tool (#10146)

Adding tool for testing epoch sync implementation.

Right now only one subcommand is available `validate-epoch-sync-info`.
It constructs `EpochSyncInfo` for every finalised epoch in DB and checks
that `EpochSyncInfo` passes validation.
Currently we only have one validation (kinda) implemented -- checking of
`epoch_sync_data_hash`.

As epoch sync is under feature and that feature includes DB upgrade, the
tool first creates a snapshot of the DB, and then modifies the config to
work with the snapshot.

Cargo.toml

@@ -199,6 +199,7 @@ nearcore = { path = "nearcore" }
 near-crypto = { path = "core/crypto" }
 near-dyn-configs = { path = "core/dyn-configs" }
 near-epoch-manager = { path = "chain/epoch-manager" }
+near-epoch-sync-tool = { path = "tools/epoch-sync"}
 near-flat-storage = { path = "tools/flat-storage" }
 near-fork-network = { path = "tools/fork-network" }
 near-fmt = { path = "utils/fmt" }

chain/chain/src/chain.rs

@@ -5647,7 +5647,7 @@ impl<'a> ChainUpdate<'a> {
             .set_ser(
                 DBCol::EpochSyncInfo,
                 last_block_info.epoch_id().as_ref(),
-                &self.create_epoch_sync_info(last_block_info, next_epoch_first_hash)?,
+                &self.create_epoch_sync_info(last_block_info, next_epoch_first_hash, None)?,
             )
             .map_err(EpochError::from)?;
         self.chain_store_update.merge(store_update);
@@ -5745,12 +5745,14 @@ impl<'a> ChainUpdate<'a> {
 
     /// Data that is necessary to prove Epoch in new Epoch Sync.
     #[cfg(feature = "new_epoch_sync")]
-    fn create_epoch_sync_info(
+    pub fn create_epoch_sync_info(
         &self,
         last_block_info: &BlockInfo,
         next_epoch_first_hash: &CryptoHash,
+        hash_to_prev_hash: Option<&HashMap<CryptoHash, CryptoHash>>,
     ) -> Result<EpochSyncInfo, Error> {
-        let mut all_block_hashes = self.epoch_manager.get_all_epoch_hashes(last_block_info)?;
+        let mut all_block_hashes =
+            self.epoch_manager.get_all_epoch_hashes(last_block_info, hash_to_prev_hash)?;
         all_block_hashes.reverse();
 
         let (headers, headers_to_save) =

chain/chain/src/lib.rs

@@ -1,5 +1,5 @@
 pub use block_processing_utils::{BlockProcessingArtifact, DoneApplyChunkCallback};
-pub use chain::{check_known, collect_receipts, Chain, MAX_ORPHAN_SIZE};
+pub use chain::{check_known, collect_receipts, Chain, ChainUpdate, MAX_ORPHAN_SIZE};
 pub use doomslug::{Doomslug, DoomslugBlockProductionReadiness, DoomslugThresholdMode};
 pub use lightclient::{create_light_client_block_view, get_epoch_block_producers_view};
 pub use near_chain_primitives::{self, Error};

chain/chain/src/test_utils/kv_runtime.rs

@@ -952,6 +952,7 @@ impl EpochManagerAdapter for MockEpochManager {
     fn get_all_epoch_hashes(
         &self,
         _last_block_info: &BlockInfo,
+        _hash_to_prev_hash: Option<&HashMap<CryptoHash, CryptoHash>>,
     ) -> Result<Vec<CryptoHash>, EpochError> {
         Ok(vec![])
     }

chain/epoch-manager/src/adapter.rs

@@ -20,6 +20,8 @@ use near_primitives::version::ProtocolVersion;
 use near_primitives::views::EpochValidatorInfo;
 use near_store::{ShardUId, StoreUpdate};
 use std::cmp::Ordering;
+#[cfg(feature = "new_epoch_sync")]
+use std::collections::HashMap;
 use std::sync::Arc;
 
 /// A trait that abstracts the interface of the EpochManager.
@@ -390,6 +392,7 @@ pub trait EpochManagerAdapter: Send + Sync {
     fn get_all_epoch_hashes(
         &self,
         last_block_info: &BlockInfo,
+        hash_to_prev_hash: Option<&HashMap<CryptoHash, CryptoHash>>,
     ) -> Result<Vec<CryptoHash>, EpochError>;
 }
 
@@ -962,8 +965,14 @@ impl EpochManagerAdapter for EpochManagerHandle {
     fn get_all_epoch_hashes(
         &self,
         last_block_info: &BlockInfo,
+        hash_to_prev_hash: Option<&HashMap<CryptoHash, CryptoHash>>,
     ) -> Result<Vec<CryptoHash>, EpochError> {
         let epoch_manager = self.read();
-        epoch_manager.get_all_epoch_hashes(last_block_info)
+        match hash_to_prev_hash {
+            None => epoch_manager.get_all_epoch_hashes_from_db(last_block_info),
+            Some(hash_to_prev_hash) => {
+                epoch_manager.get_all_epoch_hashes_from_cache(last_block_info, hash_to_prev_hash)
+            }
+        }
     }
 }

chain/epoch-manager/src/lib.rs

@@ -1859,12 +1859,12 @@ impl EpochManager {
     }
 
     #[cfg(feature = "new_epoch_sync")]
-    pub fn get_all_epoch_hashes(
+    pub fn get_all_epoch_hashes_from_db(
         &self,
         last_block_info: &BlockInfo,
     ) -> Result<Vec<CryptoHash>, EpochError> {
         let _span =
-            tracing::debug_span!(target: "epoch_manager", "get_all_epoch_hashes", ?last_block_info)
+            tracing::debug_span!(target: "epoch_manager", "get_all_epoch_hashes_from_db", ?last_block_info)
                 .entered();
 
         let mut result = vec![];
@@ -1891,4 +1891,28 @@ impl EpochManager {
 
         Ok(result)
     }
+
+    #[cfg(feature = "new_epoch_sync")]
+    fn get_all_epoch_hashes_from_cache(
+        &self,
+        last_block_info: &BlockInfo,
+        hash_to_prev_hash: &HashMap<CryptoHash, CryptoHash>,
+    ) -> Result<Vec<CryptoHash>, EpochError> {
+        let _span =
+            tracing::debug_span!(target: "epoch_manager", "get_all_epoch_hashes_from_cache", ?last_block_info)
+                .entered();
+
+        let mut result = vec![];
+        let mut current_hash = *last_block_info.hash();
+        while current_hash != *last_block_info.epoch_first_block() {
+            result.push(current_hash);
+            current_hash = *hash_to_prev_hash
+                .get(&current_hash)
+                .ok_or(EpochError::MissingBlock(current_hash))?;
+        }
+        // First block of an epoch is not covered by the while loop.
+        result.push(current_hash);
+
+        Ok(result)
+    }
 }

neard/Cargo.toml

@@ -43,6 +43,7 @@ near-config-utils.workspace = true
 near-crypto.workspace = true
 near-database-tool.workspace = true
 near-dyn-configs.workspace = true
+near-epoch-sync-tool = { workspace = true, optional = true }
 near-flat-storage.workspace = true
 near-fork-network.workspace = true
 near-jsonrpc-primitives.workspace = true
@@ -74,7 +75,7 @@ rosetta_rpc = ["nearcore/rosetta_rpc"]
 json_rpc = ["nearcore/json_rpc"]
 protocol_feature_fix_staking_threshold = ["nearcore/protocol_feature_fix_staking_threshold"]
 serialize_all_state_changes = ["nearcore/serialize_all_state_changes"]
-new_epoch_sync = ["nearcore/new_epoch_sync"]
+new_epoch_sync = ["nearcore/new_epoch_sync", "dep:near-epoch-sync-tool"]
 
 nightly = [
   "nightly_protocol",

neard/src/cli.rs

@@ -6,6 +6,8 @@ use near_client::ConfigUpdater;
 use near_cold_store_tool::ColdStoreCommand;
 use near_database_tool::commands::DatabaseCommand;
 use near_dyn_configs::{UpdateableConfigLoader, UpdateableConfigLoaderError, UpdateableConfigs};
+#[cfg(feature = "new_epoch_sync")]
+use near_epoch_sync_tool::EpochSyncCommand;
 use near_flat_storage::commands::FlatStorageCommand;
 use near_fork_network::cli::ForkNetworkCommand;
 use near_jsonrpc_primitives::types::light_client::RpcLightClientExecutionProofResponse;
@@ -144,6 +146,10 @@ impl NeardCmd {
             NeardSubCommand::StatePartsDumpCheck(cmd) => {
                 cmd.run()?;
             }
+            #[cfg(feature = "new_epoch_sync")]
+            NeardSubCommand::EpochSync(cmd) => {
+                cmd.run(&home_dir)?;
+            }
         };
         Ok(())
     }
@@ -272,6 +278,10 @@ pub(super) enum NeardSubCommand {
 
     /// Check completeness of dumped state parts of an epoch
     StatePartsDumpCheck(StatePartsDumpCheckCommand),
+
+    #[cfg(feature = "new_epoch_sync")]
+    /// Testing tool for epoch sync
+    EpochSync(EpochSyncCommand),
 }
 
 #[derive(clap::Parser)]

tools/epoch-sync/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "near-epoch-sync-tool"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+license.workspace = true
+publish = false
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow.workspace = true
+clap.workspace = true
+tracing.workspace = true
+
+nearcore = { workspace = true, features = ["new_epoch_sync"] }
+near-chain.workspace = true
+near-chain-configs.workspace = true
+near-epoch-manager.workspace = true
+near-primitives.workspace = true
+near-store.workspace = true