Ec2/scanning

zcash_client_memory/src/error.rs

@@ -1,6 +1,6 @@
 use zcash_keys::keys::{AddressGenerationError, DerivationError};
 use zcash_primitives::transaction::TxId;
-use zcash_protocol::memo;
+use zcash_protocol::{consensus::BlockHeight, memo};
 
 use crate::mem_wallet::AccountId;
 
@@ -36,6 +36,10 @@ pub enum Error {
     CorruptedData(String),
     #[error("An error occurred while processing an account due to a failure in deriving the account's keys: {0}")]
     BadAccountData(String),
+    #[error("Blocks are non sequental")]
+    NonSequentialBlocks,
+    #[error("Invalid scan range start {0}, end {1}: {2}")]
+    InvalidScanRange(BlockHeight, BlockHeight, String),
     #[error("Other error: {0}")]
     Other(String),
 }

zcash_client_memory/src/mem_wallet/mod.rs

@@ -2,6 +2,7 @@
 use core::time;
 use incrementalmerkletree::{Address, Marking, Retention};
 use sapling::NullifierDerivingKey;
+use scanning::ScanQueue;
 use secrecy::{ExposeSecret, SecretVec};
 use shardtree::{error::ShardTreeError, store::memory::MemoryShardStore, ShardTree};
 use std::{
@@ -55,6 +56,7 @@ use zcash_client_backend::{data_api::ORCHARD_SHARD_HEIGHT, wallet::WalletOrchard
 
 use crate::error::Error;
 
+mod scanning;
 mod tables;
 mod wallet_commitment_trees;
 mod wallet_read;
@@ -68,6 +70,12 @@ struct MemoryWalletBlock {
     // Just the transactions that involve an account in this wallet
     transactions: HashSet<TxId>,
     memos: HashMap<NoteId, MemoBytes>,
+    sapling_commitment_tree_size: Option<u32>,
+    sapling_output_count: Option<u32>,
+    #[cfg(feature = "orchard")]
+    orchard_commitment_tree_size: Option<u32>,
+    #[cfg(feature = "orchard")]
+    orchard_action_count: Option<u32>,
 }
 
 pub struct MemoryWalletDb {
@@ -83,6 +91,8 @@ pub struct MemoryWalletDb {
 
     tx_locator: TxLocatorMap,
 
+    scan_queue: ScanQueue,
+
     sapling_tree: ShardTree<
         MemoryShardStore<sapling::Node, BlockHeight>,
         { SAPLING_SHARD_HEIGHT * 2 },
@@ -109,6 +119,7 @@ impl MemoryWalletDb {
             nullifiers: NullifierMap::new(),
             tx_locator: TxLocatorMap::new(),
             receieved_note_spends: ReceievdNoteSpends::new(),
+            scan_queue: ScanQueue::new(),
         }
     }
     fn mark_sapling_note_spent(&mut self, nf: sapling::Nullifier, txid: TxId) -> Result<(), Error> {
@@ -124,9 +135,6 @@ impl MemoryWalletDb {
         Ok(())
     }
 
-    // fn get_account(&self, account_id: AccountId) -> Option<&Account> {
-    //     self.accounts.get(*account_id as usize)
-    // }
     fn get_account_mut(&mut self, account_id: AccountId) -> Option<&mut Account> {
         self.accounts.get_mut(*account_id as usize)
     }

zcash_client_memory/src/mem_wallet/scanning.rs

@@ -0,0 +1,215 @@
+#![allow(unused)]
+use core::time;
+use incrementalmerkletree::{Address, Marking, Position, Retention};
+use sapling::NullifierDerivingKey;
+use secrecy::{ExposeSecret, SecretVec};
+use shardtree::{error::ShardTreeError, store::memory::MemoryShardStore, ShardTree};
+use std::{
+    cell::RefCell,
+    cmp::Ordering,
+    collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet},
+    convert::Infallible,
+    hash::Hash,
+    num::NonZeroU32,
+    ops::{Deref, DerefMut, Range},
+    path::Iter,
+    rc::Rc,
+};
+use zcash_keys::keys::{AddressGenerationError, DerivationError, UnifiedIncomingViewingKey};
+use zip32::{fingerprint::SeedFingerprint, DiversifierIndex, Scope};
+
+use zcash_primitives::{
+    block::{self, BlockHash},
+    consensus::{BlockHeight, Network},
+    transaction::{components::OutPoint, txid, Authorized, Transaction, TransactionData, TxId},
+};
+use zcash_protocol::{
+    memo::{self, Memo, MemoBytes},
+    value::{ZatBalance, Zatoshis},
+    PoolType,
+    ShieldedProtocol::{self, Orchard, Sapling},
+};
+
+use zcash_client_backend::{
+    address::UnifiedAddress,
+    data_api::{
+        chain::ChainState,
+        scanning::{spanning_tree::SpanningTree, ScanPriority},
+        Account as _, AccountPurpose, AccountSource, SeedRelevance, SentTransactionOutput,
+        TransactionDataRequest, TransactionStatus,
+    },
+    keys::{UnifiedAddressRequest, UnifiedFullViewingKey, UnifiedSpendingKey},
+    wallet::{
+        Note, NoteId, Recipient, WalletSaplingOutput, WalletSpend, WalletTransparentOutput,
+        WalletTx,
+    },
+};
+
+use zcash_client_backend::data_api::{
+    chain::CommitmentTreeRoot, scanning::ScanRange, AccountBirthday, BlockMetadata,
+    DecryptedTransaction, NullifierQuery, ScannedBlock, SentTransaction, WalletCommitmentTrees,
+    WalletRead, WalletSummary, WalletWrite, SAPLING_SHARD_HEIGHT,
+};
+
+use super::AccountId;
+
+#[cfg(feature = "transparent-inputs")]
+use {
+    zcash_client_backend::wallet::TransparentAddressMetadata,
+    zcash_primitives::legacy::TransparentAddress,
+};
+
+#[cfg(feature = "orchard")]
+use {
+    zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT,
+    zcash_client_backend::wallet::WalletOrchardOutput,
+};
+
+use crate::error::Error;
+
+/// A queue of scanning ranges. Contains the start and end heights of each range, along with the
+/// priority of scanning that range.
+pub struct ScanQueue(Vec<(BlockHeight, BlockHeight, ScanPriority)>);
+
+impl ScanQueue {
+    pub fn new() -> Self {
+        ScanQueue(Vec::new())
+    }
+
+    pub fn suggest_scan_ranges(&self, min_priority: ScanPriority) -> Vec<ScanRange> {
+        let mut priorities: Vec<_> = self
+            .0
+            .iter()
+            .filter(|(_, _, p)| *p >= min_priority)
+            .collect();
+        priorities.sort_by(|(_, _, a), (_, _, b)| b.cmp(a));
+
+        priorities
+            .into_iter()
+            .map(|(start, end, priority)| {
+                let range = Range {
+                    start: *start,
+                    end: *end,
+                };
+                ScanRange::from_parts(range, *priority)
+            })
+            .collect()
+    }
+    pub fn insert_queue_entries<'a>(
+        &mut self,
+        entries: impl Iterator<Item = &'a ScanRange>,
+    ) -> Result<(), Error> {
+        for entry in entries {
+            if entry.block_range().start >= entry.block_range().end {
+                return Err(Error::InvalidScanRange(
+                    entry.block_range().start,
+                    entry.block_range().end,
+                    "start must be less than end".to_string(),
+                ));
+            }
+
+            for (start, end, _) in &self.0 {
+                if *start == entry.block_range().start || *end == entry.block_range().end {
+                    return Err(Error::InvalidScanRange(
+                        entry.block_range().start,
+                        entry.block_range().end,
+                        "at least part of range is already covered by another range".to_string(),
+                    ));
+                }
+            }
+
+            self.0.push((
+                entry.block_range().start,
+                entry.block_range().end,
+                entry.priority(),
+            ));
+        }
+        Ok(())
+    }
+    pub fn replace_queue_entries(
+        &mut self,
+        query_range: &Range<BlockHeight>,
+        entries: impl Iterator<Item = ScanRange>,
+        force_rescans: bool,
+    ) -> Result<(), Error> {
+        let (to_create, to_delete_ends) = {
+            let mut q_ranges: Vec<_> = self
+                .0
+                .iter()
+                .filter(|(start, end, _)| {
+                    //  Ignore ranges that do not overlap and are not adjacent to the query range.
+                    !(start > &query_range.end || &query_range.start > end)
+                })
+                .collect();
+            q_ranges.sort_by(|(_, end_a, _), (_, end_b, _)| end_a.cmp(end_b));
+
+            // Iterate over the ranges in the scan queue that overlap the range that we have
+            // identified as needing to be fully scanned. For each such range add it to the
+            // spanning tree (these should all be nonoverlapping ranges, but we might coalesce
+            // some in the process).
+            let mut to_create: Option<SpanningTree> = None;
+            let mut to_delete_ends: Vec<BlockHeight> = vec![];
+
+            let mut q_ranges = q_ranges.into_iter();
+            while let Some((start, end, priority)) = q_ranges.next() {
+                let entry = ScanRange::from_parts(
+                    Range {
+                        start: *start,
+                        end: *end,
+                    },
+                    *priority,
+                );
+                to_delete_ends.push(entry.block_range().end);
+                to_create = if let Some(cur) = to_create {
+                    Some(cur.insert(entry, force_rescans))
+                } else {
+                    Some(SpanningTree::Leaf(entry))
+                };
+            }
+
+            // Update the tree that we read from the database, or if we didn't find any ranges
+            // start with the scanned range.
+            for entry in entries {
+                to_create = if let Some(cur) = to_create {
+                    Some(cur.insert(entry, force_rescans))
+                } else {
+                    Some(SpanningTree::Leaf(entry))
+                };
+            }
+            (to_create, to_delete_ends)
+        };
+
+        if let Some(tree) = to_create {
+            self.0.retain(|(_, block_range_end, _)| {
+                // if the block_range_end is equal to any in to_delete_ends, remove it
+                !to_delete_ends.contains(block_range_end)
+            });
+            let scan_ranges = tree.into_vec();
+            self.insert_queue_entries(scan_ranges.iter());
+        }
+        Ok(())
+    }
+}
+
+impl IntoIterator for ScanQueue {
+    type Item = (BlockHeight, BlockHeight, ScanPriority);
+    type IntoIter = <Vec<Self::Item> as IntoIterator>::IntoIter;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.0.into_iter()
+    }
+}
+
+// We deref to slice so that we can reuse the slice impls
+impl Deref for ScanQueue {
+    type Target = [(BlockHeight, BlockHeight, ScanPriority)];
+
+    fn deref(&self) -> &Self::Target {
+        &self.0[..]
+    }
+}
+impl DerefMut for ScanQueue {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0[..]
+    }
+}