Ledgerdb migrations verified proofs, batch by number, slot by number (#…

…1649)

Co-authored-by: eyusufatik <[email protected]>

crates/batch-prover/src/db_migrations/batch_and_slot_by_number.rs

@@ -0,0 +1,27 @@
+use std::sync::Arc;
+
+use sov_db::ledger_db::migrations::{LedgerMigration, MigrationName, MigrationVersion};
+use sov_db::ledger_db::LedgerDB;
+
+/// Table removal migration
+/// tables BatchByNumber and SlotByNumber are removed
+pub(crate) struct MigrateBatchAndSlotByNumber {}
+
+impl LedgerMigration for MigrateBatchAndSlotByNumber {
+    fn identifier(&self) -> (MigrationName, MigrationVersion) {
+        ("MigrateBatchAndSlotByNumber".to_owned(), 1)
+    }
+
+    fn execute(
+        &self,
+        _ledger_db: Arc<LedgerDB>,
+        tables_to_drop: &mut Vec<String>,
+    ) -> anyhow::Result<()> {
+        let batch_by_number = "BatchByNumber".to_owned();
+        let slot_by_number = "SlotByNumber".to_owned();
+
+        tables_to_drop.push(batch_by_number);
+        tables_to_drop.push(slot_by_number);
+        Ok(())
+    }
+}

crates/batch-prover/src/db_migrations/mod.rs

@@ -2,9 +2,19 @@ use std::sync::OnceLock;
 
 use sov_db::ledger_db::migrations::LedgerMigration;
 
-#[allow(dead_code)]
+use crate::db_migrations::batch_and_slot_by_number::MigrateBatchAndSlotByNumber;
+use crate::db_migrations::verified_proofs::MigrateVerifiedProofsBySlotNumber;
+
+mod batch_and_slot_by_number;
+mod verified_proofs;
+
 pub fn migrations() -> &'static Vec<Box<dyn LedgerMigration + Send + Sync + 'static>> {
     static MIGRATIONS: OnceLock<Vec<Box<dyn LedgerMigration + Send + Sync + 'static>>> =
         OnceLock::new();
-    MIGRATIONS.get_or_init(Vec::new)
+    MIGRATIONS.get_or_init(|| {
+        vec![
+            Box::new(MigrateVerifiedProofsBySlotNumber {}),
+            Box::new(MigrateBatchAndSlotByNumber {}),
+        ]
+    })
 }

crates/batch-prover/src/db_migrations/verified_proofs.rs

@@ -0,0 +1,40 @@
+use std::sync::Arc;
+
+use sov_db::ledger_db::migrations::{LedgerMigration, MigrationName, MigrationVersion};
+use sov_db::ledger_db::LedgerDB;
+
+/// Table name change migration
+/// table name "VerifiedProofsBySlotNumber" is now "VerifiedBatchProofsBySlotNumber"
+pub(crate) struct MigrateVerifiedProofsBySlotNumber {}
+
+// Name of the schema was changed from VerifiedProofsBySlotNumber to VerifiedBatchProofsBySlotNumber
+impl LedgerMigration for MigrateVerifiedProofsBySlotNumber {
+    fn identifier(&self) -> (MigrationName, MigrationVersion) {
+        ("MigrateVerifiedProofsBySlotNumber".to_owned(), 1)
+    }
+
+    fn execute(
+        &self,
+        ledger_db: Arc<LedgerDB>,
+        tables_to_drop: &mut Vec<String>,
+    ) -> anyhow::Result<()> {
+        let from = "VerifiedProofsBySlotNumber";
+        let to = "VerifiedBatchProofsBySlotNumber";
+
+        let migrate_from_handle = ledger_db.get_cf_handle(from)?;
+
+        let migrate_from_iterator = ledger_db.get_iterator_for_cf(migrate_from_handle, None)?;
+
+        let migrate_to_handle = ledger_db.get_cf_handle(to)?;
+
+        // Insert key value pairs from old table to new table
+        for key_value_res in migrate_from_iterator {
+            let (key, value) = key_value_res.unwrap();
+            ledger_db.insert_into_cf_raw(migrate_to_handle, &key, &value)?;
+        }
+        drop(ledger_db);
+
+        tables_to_drop.push(from.to_string());
+        Ok(())
+    }
+}

crates/evm/src/evm/primitive_types.rs

@@ -1,4 +1,4 @@
-use std::ops::Range;
+use std::ops::{Deref, Range};
 
 use alloy_primitives::{Address, BlockNumber, Bloom, Bytes, Sealable, B256, B64, U256};
 use alloy_rlp::bytes::BufMut;
@@ -245,6 +245,21 @@ impl From<Block<AlloyHeader>> for Block<DoNotUseHeader> {
     }
 }
 
+#[derive(Debug, PartialEq, Clone, serde::Serialize, serde::Deserialize)]
+pub(crate) struct DoNotUseSealedBlock {
+    /// Block header.
+    pub(crate) header: SealedHeader<DoNotUseHeader>,
+
+    /// L1 fee rate.
+    pub(crate) l1_fee_rate: u128,
+
+    /// The hash of L1 block that the L2 block corresponds to.  
+    pub(crate) l1_hash: B256,
+
+    /// Transactions in this block.
+    pub(crate) transactions: Range<u64>,
+}
+
 #[derive(Debug, PartialEq, Clone, serde::Serialize, serde::Deserialize)]
 pub(crate) struct SealedBlock {
     /// Block header.
@@ -368,6 +383,20 @@ impl Decodable for SealedBlock {
     }
 }
 
+impl From<DoNotUseSealedBlock> for SealedBlock {
+    fn from(value: DoNotUseSealedBlock) -> Self {
+        let alloy_header = AlloyHeader::from(value.header.deref().clone());
+        let sealed = alloy_header.seal_slow();
+        let (header, seal) = sealed.into_parts();
+        Self {
+            header: SealedHeader::new(header, seal),
+            l1_fee_rate: value.l1_fee_rate,
+            l1_hash: value.l1_hash,
+            transactions: value.transactions,
+        }
+    }
+}
+
 #[derive(
     Debug,
     PartialEq,

crates/evm/src/provider_functions.rs

@@ -1,7 +1,11 @@
 use alloy_primitives::Address;
 use reth_primitives::{Account, SealedHeader};
-use sov_modules_api::{StateMapAccessor, StateVecAccessor, WorkingSet};
+use sov_modules_api::{
+    AccessoryStateVec, AccessoryWorkingSet, StateMapAccessor, StateVecAccessor, WorkingSet,
+};
+use sov_state::codec::{BcsCodec, RlpCodec};
 
+use crate::primitive_types::{DoNotUseSealedBlock, SealedBlock};
 use crate::Evm;
 
 impl<C: sov_modules_api::Context> Evm<C> {
@@ -23,6 +27,26 @@ impl<C: sov_modules_api::Context> Evm<C> {
     pub fn last_sealed_header(&self, working_set: &mut WorkingSet<C::Storage>) -> SealedHeader {
         self.blocks
             .last(&mut working_set.accessory_state())
+            .or_else(|| {
+                //  upgrading from v0.5.7 to v0.6+ requires a codec change
+                // this only applies to the sequencer
+                // which will only query the genesis block and the head block
+                // right after the upgrade
+                let prefix = <AccessoryStateVec<SealedBlock, RlpCodec> as StateVecAccessor<
+                    SealedBlock,
+                    RlpCodec,
+                    AccessoryWorkingSet<C::Storage>,
+                >>::prefix(&self.blocks);
+                let accessor_with_old_codec =
+                    AccessoryStateVec::<DoNotUseSealedBlock, BcsCodec>::with_codec(
+                        prefix.clone(),
+                        BcsCodec,
+                    );
+
+                accessor_with_old_codec
+                    .last(&mut working_set.accessory_state())
+                    .map(Into::into)
+            })
             .unwrap()
             .header
     }

crates/evm/src/query.rs

@@ -1531,6 +1531,30 @@ impl<C: sov_modules_api::Context> Evm<C> {
             Some(BlockNumberOrTag::Earliest) => Ok(Some(
                 self.blocks
                     .get(0, &mut working_set.accessory_state())
+                    .or_else(|| {
+                        // upgrading from v0.5.7 to v0.6+ requires a codec change
+                        // this only applies to the sequencer
+                        // which will only query the genesis block and the head block
+                        // right after the upgrade
+                        let prefix = <sov_modules_api::AccessoryStateVec<
+                            SealedBlock,
+                            sov_state::codec::RlpCodec,
+                        > as StateVecAccessor<
+                            SealedBlock,
+                            sov_state::codec::RlpCodec,
+                            sov_state::storage::AccessoryWorkingSet<C::Storage>,
+                        >>::prefix(&self.blocks);
+                        let accessor_with_old_codec = sov_modules_api::AccessoryStateVec::<
+                            crate::primitive_types::DoNotUseSealedBlock,
+                            sov_state::codec::BcsCodec,
+                        >::with_codec(
+                            prefix.clone(), sov_state::codec::BcsCodec
+                        );
+
+                        accessor_with_old_codec
+                            .get(0, &mut working_set.accessory_state())
+                            .map(Into::into)
+                    })
                     .expect("Genesis block must be set"),
             )),
             Some(BlockNumberOrTag::Latest) => Ok(Some(

crates/fullnode/src/db_migrations/batch_and_slot_by_number.rs

@@ -0,0 +1,27 @@
+use std::sync::Arc;
+
+use sov_db::ledger_db::migrations::{LedgerMigration, MigrationName, MigrationVersion};
+use sov_db::ledger_db::LedgerDB;
+
+/// Table removal migration
+/// tables BatchByNumber and SlotByNumber are removed
+pub(crate) struct MigrateBatchAndSlotByNumber {}
+
+impl LedgerMigration for MigrateBatchAndSlotByNumber {
+    fn identifier(&self) -> (MigrationName, MigrationVersion) {
+        ("MigrateBatchAndSlotByNumber".to_owned(), 1)
+    }
+
+    fn execute(
+        &self,
+        _ledger_db: Arc<LedgerDB>,
+        tables_to_drop: &mut Vec<String>,
+    ) -> anyhow::Result<()> {
+        let batch_by_number = "BatchByNumber".to_owned();
+        let slot_by_number = "SlotByNumber".to_owned();
+
+        tables_to_drop.push(batch_by_number);
+        tables_to_drop.push(slot_by_number);
+        Ok(())
+    }
+}

crates/fullnode/src/db_migrations/mod.rs

@@ -2,12 +2,19 @@ use std::sync::OnceLock;
 
 use sov_db::ledger_db::migrations::LedgerMigration;
 
+use crate::db_migrations::batch_and_slot_by_number::MigrateBatchAndSlotByNumber;
 use crate::db_migrations::verified_proofs::MigrateVerifiedProofsBySlotNumber;
 
+mod batch_and_slot_by_number;
 mod verified_proofs;
 
 pub fn migrations() -> &'static Vec<Box<dyn LedgerMigration + Send + Sync + 'static>> {
     static MIGRATIONS: OnceLock<Vec<Box<dyn LedgerMigration + Send + Sync + 'static>>> =
         OnceLock::new();
-    MIGRATIONS.get_or_init(|| vec![Box::new(MigrateVerifiedProofsBySlotNumber {})])
+    MIGRATIONS.get_or_init(|| {
+        vec![
+            Box::new(MigrateVerifiedProofsBySlotNumber {}),
+            Box::new(MigrateBatchAndSlotByNumber {}),
+        ]
+    })
 }

crates/sequencer/src/db_migrations/batch_and_slot_by_number.rs

@@ -0,0 +1,27 @@
+use std::sync::Arc;
+
+use sov_db::ledger_db::migrations::{LedgerMigration, MigrationName, MigrationVersion};
+use sov_db::ledger_db::LedgerDB;
+
+/// Table removal migration
+/// tables BatchByNumber and SlotByNumber are removed
+pub(crate) struct MigrateBatchAndSlotByNumber {}
+
+impl LedgerMigration for MigrateBatchAndSlotByNumber {
+    fn identifier(&self) -> (MigrationName, MigrationVersion) {
+        ("MigrateBatchAndSlotByNumber".to_owned(), 1)
+    }
+
+    fn execute(
+        &self,
+        _ledger_db: Arc<LedgerDB>,
+        tables_to_drop: &mut Vec<String>,
+    ) -> anyhow::Result<()> {
+        let batch_by_number = "BatchByNumber".to_owned();
+        let slot_by_number = "SlotByNumber".to_owned();
+
+        tables_to_drop.push(batch_by_number);
+        tables_to_drop.push(slot_by_number);
+        Ok(())
+    }
+}

crates/sequencer/src/db_migrations/mod.rs

@@ -2,8 +2,19 @@ use std::sync::OnceLock;
 
 use sov_db::ledger_db::migrations::LedgerMigration;
 
+use crate::db_migrations::batch_and_slot_by_number::MigrateBatchAndSlotByNumber;
+use crate::db_migrations::verified_proofs::MigrateVerifiedProofsBySlotNumber;
+
+mod batch_and_slot_by_number;
+mod verified_proofs;
+
 pub fn migrations() -> &'static Vec<Box<dyn LedgerMigration + Send + Sync + 'static>> {
     static MIGRATIONS: OnceLock<Vec<Box<dyn LedgerMigration + Send + Sync + 'static>>> =
         OnceLock::new();
-    MIGRATIONS.get_or_init(Vec::new)
+    MIGRATIONS.get_or_init(|| {
+        vec![
+            Box::new(MigrateVerifiedProofsBySlotNumber {}),
+            Box::new(MigrateBatchAndSlotByNumber {}),
+        ]
+    })
 }

crates/sequencer/src/db_migrations/verified_proofs.rs

@@ -0,0 +1,40 @@
+use std::sync::Arc;
+
+use sov_db::ledger_db::migrations::{LedgerMigration, MigrationName, MigrationVersion};
+use sov_db::ledger_db::LedgerDB;
+
+/// Table name change migration
+/// table name "VerifiedProofsBySlotNumber" is now "VerifiedBatchProofsBySlotNumber"
+pub(crate) struct MigrateVerifiedProofsBySlotNumber {}
+
+// Name of the schema was changed from VerifiedProofsBySlotNumber to VerifiedBatchProofsBySlotNumber
+impl LedgerMigration for MigrateVerifiedProofsBySlotNumber {
+    fn identifier(&self) -> (MigrationName, MigrationVersion) {
+        ("MigrateVerifiedProofsBySlotNumber".to_owned(), 1)
+    }
+
+    fn execute(
+        &self,
+        ledger_db: Arc<LedgerDB>,
+        tables_to_drop: &mut Vec<String>,
+    ) -> anyhow::Result<()> {
+        let from = "VerifiedProofsBySlotNumber";
+        let to = "VerifiedBatchProofsBySlotNumber";
+
+        let migrate_from_handle = ledger_db.get_cf_handle(from)?;
+
+        let migrate_from_iterator = ledger_db.get_iterator_for_cf(migrate_from_handle, None)?;
+
+        let migrate_to_handle = ledger_db.get_cf_handle(to)?;
+
+        // Insert key value pairs from old table to new table
+        for key_value_res in migrate_from_iterator {
+            let (key, value) = key_value_res.unwrap();
+            ledger_db.insert_into_cf_raw(migrate_to_handle, &key, &value)?;
+        }
+        drop(ledger_db);
+
+        tables_to_drop.push(from.to_string());
+        Ok(())
+    }
+}

crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/migrations/mod.rs

@@ -6,7 +6,7 @@ use std::sync::Arc;
 use anyhow::anyhow;
 use tracing::{debug, error};
 
-use super::migrations::utils::{drop_column_family, list_column_families};
+use super::migrations::utils::{drop_column_families, list_column_families};
 use super::LedgerDB;
 use crate::ledger_db::{SharedLedgerOps, LEDGER_DB_PATH_SUFFIX};
 use crate::rocks_db_config::RocksdbConfig;
@@ -145,16 +145,15 @@ impl<'a> LedgerDBMigrator<'a> {
         drop(new_ledger_db);
 
         // Now that the lock is gone drop the tables that were migrated
-        for table in tables_to_drop {
-            drop_column_family(
-                &RocksdbConfig::new(
-                    temp_db_path.path(),
-                    max_open_files,
-                    Some(all_column_families.clone()),
-                ),
-                &table,
-            )?;
-        }
+
+        drop_column_families(
+            &RocksdbConfig::new(
+                temp_db_path.path(),
+                max_open_files,
+                Some(all_column_families.clone()),
+            ),
+            tables_to_drop,
+        )?;
 
         // Construct a backup path adjacent to original path
         let ledger_path = dbs_path.join(LEDGER_DB_PATH_SUFFIX);