From 9bd4f4833ac6f50e82ae6f90dc8036fb06fa01c4 Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Sun, 20 Oct 2024 13:36:14 +0200 Subject: [PATCH 1/4] piecrust: partial clone in contract session --- piecrust/src/store.rs | 14 ++- piecrust/src/store/commit.rs | 221 ++++++++++++++++++++++++++++++++++ piecrust/src/store/session.rs | 39 +++--- piecrust/src/store/tree.rs | 11 ++ 4 files changed, 265 insertions(+), 20 deletions(-) create mode 100644 piecrust/src/store/commit.rs diff --git a/piecrust/src/store.rs b/piecrust/src/store.rs index 03c48a1c..d5ea6101 100644 --- a/piecrust/src/store.rs +++ b/piecrust/src/store.rs @@ -7,6 +7,7 @@ //! A library for dealing with memories in trees. mod bytecode; +mod commit; mod memory; mod metadata; mod module; @@ -28,6 +29,7 @@ use piecrust_uplink::ContractId; use session::ContractDataEntry; use tree::{Hash, NewContractIndex}; +use crate::store::commit::CommitClone; use crate::store::tree::{ position_from_contract, BaseInfo, ContractIndexElement, ContractsMerkle, PageTree, @@ -235,7 +237,11 @@ impl ContractStore { fn session_with_base(&self, base: Option) -> ContractSession { let base_commit = base.and_then(|hash| { - self.commit_store.lock().unwrap().get_commit(&hash).cloned() // todo: clone here + self.commit_store + .lock() + .unwrap() + .get_commit(&hash) + .map(|commit| commit.to_clone()) // todo: clone here }); ContractSession::new( &self.root_dir, @@ -502,6 +508,7 @@ impl Commit { } } + #[allow(dead_code)] pub fn fast_clone<'a>( &self, contract_ids: impl Iterator, @@ -519,6 +526,11 @@ impl Commit { } } + pub fn to_clone(&self) -> CommitClone { + CommitClone::from_commit(self) + } + + #[allow(dead_code)] pub fn inclusion_proofs( mut self, contract_id: &ContractId, diff --git a/piecrust/src/store/commit.rs b/piecrust/src/store/commit.rs new file mode 100644 index 00000000..9840aa48 --- /dev/null +++ b/piecrust/src/store/commit.rs @@ -0,0 +1,221 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// Copyright (c) DUSK NETWORK. All rights reserved. + +use crate::store::tree::{ + position_from_contract, ContractIndexElement, ContractsMerkle, Hash, + NewContractIndex, PageTree, +}; +use crate::store::{Commit, Memory}; +use crate::PageOpening; +use piecrust_uplink::ContractId; +use std::cell::Ref; +use std::collections::BTreeSet; + +#[derive(Debug, Clone)] +pub(crate) struct CommitClone { + index: Option<*const NewContractIndex>, + index2: NewContractIndex, + contracts_merkle: ContractsMerkle, + maybe_hash: Option, +} + +impl CommitClone { + pub fn from_commit(commit: &Commit) -> Self { + Self { + index: Some(&commit.index), + index2: NewContractIndex::new(), + contracts_merkle: commit.contracts_merkle.clone(), + maybe_hash: commit.maybe_hash, + } + } + + pub fn new() -> Self { + Self { + index: None, + index2: NewContractIndex::new(), + contracts_merkle: ContractsMerkle::default(), + maybe_hash: None, + } + } + + pub fn to_commit(&self) -> Commit { + let index = self.index.map(|p| unsafe { p.as_ref().unwrap() }); + let mut commit = match index { + Some(p) => Commit { + index: p.clone(), + contracts_merkle: self.contracts_merkle.clone(), + maybe_hash: self.maybe_hash, + }, + None => Commit { + index: NewContractIndex::new(), + contracts_merkle: self.contracts_merkle.clone(), + maybe_hash: self.maybe_hash, + }, + }; + for (contract_id, element) in self.index2.contracts().iter() { + commit + .index + .insert_contract_index(&contract_id, element.clone()) + } + commit + } + + pub fn fast_clone<'a>( + &self, + contract_ids: impl Iterator, + ) -> Self { + let mut index2 = NewContractIndex::new(); + for contract_id in contract_ids { + if let Some(a) = self.index_get(contract_id) { + index2.insert_contract_index(contract_id, a.clone()); + } + } + Self { + index: None, + index2, + contracts_merkle: self.contracts_merkle.clone(), + maybe_hash: self.maybe_hash, + } + } + + pub fn inclusion_proofs( + mut self, + contract_id: &ContractId, + ) -> Option> { + let contract = self.remove_contract_index(contract_id)?; + + let pos = position_from_contract(contract_id); + + Some(contract.page_indices.into_iter().map(move |page_index| { + let tree_opening = self + .contracts_merkle + .opening(pos) + .expect("There must be a leaf for the contract"); + + let page_opening = contract + .tree + .opening(page_index as u64) + .expect("There must be a leaf for the page"); + + ( + page_index, + PageOpening { + tree: tree_opening, + inner: page_opening, + }, + ) + })) + } + + pub fn insert(&mut self, contract_id: ContractId, memory: &Memory) { + if self.index_get(&contract_id).is_none() { + self.insert_contract_index( + &contract_id, + ContractIndexElement { + tree: PageTree::new(memory.is_64()), + len: 0, + page_indices: BTreeSet::new(), + hash: None, + int_pos: None, + }, + ); + } + let (index2, contracts_merkle) = self.get_mutables(); + let element = index2.get_mut(&contract_id, None).unwrap(); + + element.len = memory.current_len; + + for (dirty_page, _, page_index) in memory.dirty_pages() { + element.page_indices.insert(*page_index); + let hash = Hash::new(dirty_page); + element.tree.insert(*page_index as u64, hash); + } + + let root = element.tree.root(); + let pos = position_from_contract(&contract_id); + let int_pos = contracts_merkle.insert(pos, *root); + element.hash = Some(*root); + element.int_pos = Some(int_pos); + } + + // to satisfy borrow checker + fn get_mutables( + &mut self, + ) -> (&mut NewContractIndex, &mut ContractsMerkle) { + (&mut self.index2, &mut self.contracts_merkle) + } + + // pub fn remove_and_insert(&mut self, contract: ContractId, memory: + // &Memory) { self.remove_contract_index(&contract); + // self.insert(contract, memory); + // } + + pub fn root(&self) -> Ref { + tracing::trace!("calculating root started"); + let ret = self.contracts_merkle.root(); + tracing::trace!("calculating root finished"); + ret + } + + /* + ========================== + */ + + pub fn remove_contract_index( + &mut self, + contract_id: &ContractId, + ) -> Option { + self.index2.contracts_mut().remove(&contract_id) // todo: may not work + } + + pub fn insert_contract_index( + &mut self, + contract_id: &ContractId, + element: ContractIndexElement, + ) { + self.index2.contracts_mut().insert(*contract_id, element); + } + + pub fn index_get( + &self, + contract_id: &ContractId, + ) -> Option<&ContractIndexElement> { + let index = self.index.map(|p| unsafe { p.as_ref().unwrap() }); + match index { + Some(p) => self + .index2 + .get(&contract_id, self.maybe_hash) + .or_else(move || p.get(&contract_id, self.maybe_hash)), + None => self.index2.get(&contract_id, self.maybe_hash), + } + } + + // note, does not allow changing the original + // pub fn index_get_mut( + // &mut self, + // contract_id: &ContractId, + // ) -> Option<&mut ContractIndexElement> { + // self.index2 + // .get_mut(&contract_id, self.maybe_hash) + // } + + pub fn index_contains_key(&self, contract_id: &ContractId) -> bool { + let index = self.index.map(|p| unsafe { p.as_ref().unwrap() }); + match index { + Some(p) => { + self.index2.contains_key(&contract_id) + || p.contains_key(&contract_id) + } + None => self.index2.contains_key(&contract_id), + } + } + + // pub fn index_iter( + // &self, + // ) -> impl Iterator { + // self.index.contracts().iter() + // } +} diff --git a/piecrust/src/store/session.rs b/piecrust/src/store/session.rs index 085db789..ad621a2c 100644 --- a/piecrust/src/store/session.rs +++ b/piecrust/src/store/session.rs @@ -15,11 +15,12 @@ use dusk_wasmtime::Engine; use piecrust_uplink::ContractId; use crate::contract::ContractMetadata; +use crate::store::commit::CommitClone; use crate::store::tree::{Hash, PageOpening}; use crate::store::{ - base_from_path, Bytecode, Call, Commit, Memory, Metadata, Module, - BASE_FILE, BYTECODE_DIR, ELEMENT_FILE, MAIN_DIR, MEMORY_DIR, - METADATA_EXTENSION, OBJECTCODE_EXTENSION, PAGE_SIZE, + base_from_path, Bytecode, Call, Memory, Metadata, Module, BASE_FILE, + BYTECODE_DIR, ELEMENT_FILE, MAIN_DIR, MEMORY_DIR, METADATA_EXTENSION, + OBJECTCODE_EXTENSION, PAGE_SIZE, }; use crate::Error; @@ -45,7 +46,7 @@ pub struct ContractSession { contracts: BTreeMap, engine: Engine, - base: Option, + base: Option, root_dir: PathBuf, call: mpsc::Sender, @@ -65,7 +66,7 @@ impl ContractSession { pub(crate) fn new>( root_dir: P, engine: Engine, - base: Option, + base: Option, call: mpsc::Sender, ) -> Self { Self { @@ -92,7 +93,7 @@ impl ContractSession { .base .as_ref() .map(|c| c.fast_clone(&mut self.contracts.keys())) - .unwrap_or(Commit::new()); + .unwrap_or(CommitClone::new()); for (contract, entry) in &self.contracts { commit.insert(*contract, &entry.memory); } @@ -109,7 +110,7 @@ impl ContractSession { contract: ContractId, ) -> Option> { tracing::trace!("memory_pages called commit cloning"); - let mut commit = self.base.clone().unwrap_or(Commit::new()); + let mut commit = self.base.clone().unwrap_or(CommitClone::new()); for (contract, entry) in &self.contracts { commit.insert(*contract, &entry.memory); } @@ -145,14 +146,16 @@ impl ContractSession { let (replier, receiver) = mpsc::sync_channel(1); let mut contracts = BTreeMap::new(); - let mut base = self.base.as_ref().map(|c| Commit { - index: c.index.clone(), - contracts_merkle: c.contracts_merkle.clone(), - maybe_hash: c.maybe_hash, - }); + let base = self.base.as_ref().map(|c| //Commit { + // index: c.index.clone(), + // contracts_merkle: c.contracts_merkle.clone(), + // maybe_hash: c.maybe_hash, + c.to_commit() /* } */); + + let mut base_clone = base.as_ref().map(|b| b.to_clone()); // todo: wasteful, think of a better way mem::swap(&mut self.contracts, &mut contracts); - mem::swap(&mut self.base, &mut base); + mem::swap(&mut self.base, &mut base_clone); self.call .send(Call::Commit { @@ -249,7 +252,7 @@ impl ContractSession { Vacant(entry) => match &self.base { None => Ok(None), Some(base_commit) => { - match base_commit.index.contains_key(&contract) { + match base_commit.index_contains_key(&contract) { true => { let base_dir = self.root_dir.join(MAIN_DIR); @@ -269,9 +272,7 @@ impl ContractSession { Module::from_file(&self.engine, module_path)?; let metadata = Metadata::from_file(metadata_path)?; - let memory = match base_commit - .index - .get(&contract, base_commit.maybe_hash) + let memory = match base_commit.index_get(&contract) { Some(elem) => { let page_indices = @@ -336,7 +337,7 @@ impl ContractSession { if self.contracts.contains_key(&contract_id) { true } else if let Some(base_commit) = &self.base { - base_commit.index.contains_key(&contract_id) + base_commit.index_contains_key(&contract_id) } else { false } @@ -363,7 +364,7 @@ impl ContractSession { // If the position is already filled in the tree, the contract cannot be // inserted. if let Some(base) = self.base.as_ref() { - if base.index.contains_key(&contract_id) { + if base.index_contains_key(&contract_id) { return Err(io::Error::new( io::ErrorKind::Other, format!("Existing contract '{contract_id}'"), diff --git a/piecrust/src/store/tree.rs b/piecrust/src/store/tree.rs index 2fec620c..69a4dd6a 100644 --- a/piecrust/src/store/tree.rs +++ b/piecrust/src/store/tree.rs @@ -81,6 +81,17 @@ pub struct NewContractIndex { inner_contracts: BTreeMap, } +impl NewContractIndex { + pub fn contracts(&self) -> &BTreeMap { + &self.inner_contracts + } + pub fn contracts_mut( + &mut self, + ) -> &mut BTreeMap { + &mut self.inner_contracts + } +} + #[derive(Debug, Clone, Archive, Deserialize, Serialize)] #[archive_attr(derive(CheckBytes))] pub struct ContractsMerkle { From b234cba77501641908bdf6d68637ae6a5d62a8a5 Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Sun, 20 Oct 2024 13:47:48 +0200 Subject: [PATCH 2/4] piecrust: renamed commit clone to hulk --- piecrust/src/store.rs | 8 ++++---- piecrust/src/store/commit.rs | 4 ++-- piecrust/src/store/session.rs | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/piecrust/src/store.rs b/piecrust/src/store.rs index d5ea6101..23455dd6 100644 --- a/piecrust/src/store.rs +++ b/piecrust/src/store.rs @@ -29,7 +29,7 @@ use piecrust_uplink::ContractId; use session::ContractDataEntry; use tree::{Hash, NewContractIndex}; -use crate::store::commit::CommitClone; +use crate::store::commit::CommitHulk; use crate::store::tree::{ position_from_contract, BaseInfo, ContractIndexElement, ContractsMerkle, PageTree, @@ -241,7 +241,7 @@ impl ContractStore { .lock() .unwrap() .get_commit(&hash) - .map(|commit| commit.to_clone()) // todo: clone here + .map(|commit| commit.to_hulk()) }); ContractSession::new( &self.root_dir, @@ -526,8 +526,8 @@ impl Commit { } } - pub fn to_clone(&self) -> CommitClone { - CommitClone::from_commit(self) + pub fn to_hulk(&self) -> CommitHulk { + CommitHulk::from_commit(self) } #[allow(dead_code)] diff --git a/piecrust/src/store/commit.rs b/piecrust/src/store/commit.rs index 9840aa48..71007adb 100644 --- a/piecrust/src/store/commit.rs +++ b/piecrust/src/store/commit.rs @@ -15,14 +15,14 @@ use std::cell::Ref; use std::collections::BTreeSet; #[derive(Debug, Clone)] -pub(crate) struct CommitClone { +pub(crate) struct CommitHulk { index: Option<*const NewContractIndex>, index2: NewContractIndex, contracts_merkle: ContractsMerkle, maybe_hash: Option, } -impl CommitClone { +impl CommitHulk { pub fn from_commit(commit: &Commit) -> Self { Self { index: Some(&commit.index), diff --git a/piecrust/src/store/session.rs b/piecrust/src/store/session.rs index ad621a2c..ea82c2b4 100644 --- a/piecrust/src/store/session.rs +++ b/piecrust/src/store/session.rs @@ -15,7 +15,7 @@ use dusk_wasmtime::Engine; use piecrust_uplink::ContractId; use crate::contract::ContractMetadata; -use crate::store::commit::CommitClone; +use crate::store::commit::CommitHulk; use crate::store::tree::{Hash, PageOpening}; use crate::store::{ base_from_path, Bytecode, Call, Memory, Metadata, Module, BASE_FILE, @@ -46,7 +46,7 @@ pub struct ContractSession { contracts: BTreeMap, engine: Engine, - base: Option, + base: Option, root_dir: PathBuf, call: mpsc::Sender, @@ -66,7 +66,7 @@ impl ContractSession { pub(crate) fn new>( root_dir: P, engine: Engine, - base: Option, + base: Option, call: mpsc::Sender, ) -> Self { Self { @@ -93,7 +93,7 @@ impl ContractSession { .base .as_ref() .map(|c| c.fast_clone(&mut self.contracts.keys())) - .unwrap_or(CommitClone::new()); + .unwrap_or(CommitHulk::new()); for (contract, entry) in &self.contracts { commit.insert(*contract, &entry.memory); } @@ -110,7 +110,7 @@ impl ContractSession { contract: ContractId, ) -> Option> { tracing::trace!("memory_pages called commit cloning"); - let mut commit = self.base.clone().unwrap_or(CommitClone::new()); + let mut commit = self.base.clone().unwrap_or(CommitHulk::new()); for (contract, entry) in &self.contracts { commit.insert(*contract, &entry.memory); } @@ -152,7 +152,7 @@ impl ContractSession { // maybe_hash: c.maybe_hash, c.to_commit() /* } */); - let mut base_clone = base.as_ref().map(|b| b.to_clone()); // todo: wasteful, think of a better way + let mut base_clone = base.as_ref().map(|b| b.to_hulk()); // todo: wasteful, think of a better way mem::swap(&mut self.contracts, &mut contracts); mem::swap(&mut self.base, &mut base_clone); From 86b63c757cad97186acc9c4d0246a341e84b8ee0 Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Sun, 20 Oct 2024 19:22:44 +0200 Subject: [PATCH 3/4] piecrust: optimized cloning in session --- piecrust/src/store/commit.rs | 33 ++++++++++++++++----------------- piecrust/src/store/session.rs | 9 +-------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/piecrust/src/store/commit.rs b/piecrust/src/store/commit.rs index 71007adb..a3ee13b4 100644 --- a/piecrust/src/store/commit.rs +++ b/piecrust/src/store/commit.rs @@ -43,7 +43,7 @@ impl CommitHulk { pub fn to_commit(&self) -> Commit { let index = self.index.map(|p| unsafe { p.as_ref().unwrap() }); - let mut commit = match index { + match index { Some(p) => Commit { index: p.clone(), contracts_merkle: self.contracts_merkle.clone(), @@ -54,13 +54,12 @@ impl CommitHulk { contracts_merkle: self.contracts_merkle.clone(), maybe_hash: self.maybe_hash, }, - }; - for (contract_id, element) in self.index2.contracts().iter() { - commit - .index - .insert_contract_index(&contract_id, element.clone()) } - commit + // for (contract_id, element) in self.index2.contracts().iter() { + // commit + // .index + // .insert_contract_index(&contract_id, element.clone()) + // } } pub fn fast_clone<'a>( @@ -123,8 +122,8 @@ impl CommitHulk { }, ); } - let (index2, contracts_merkle) = self.get_mutables(); - let element = index2.get_mut(&contract_id, None).unwrap(); + let (index, contracts_merkle) = self.get_mutables(); + let element = index.get_mut(&contract_id, None).unwrap(); element.len = memory.current_len; @@ -161,14 +160,14 @@ impl CommitHulk { } /* - ========================== + index accessors */ pub fn remove_contract_index( &mut self, contract_id: &ContractId, ) -> Option { - self.index2.contracts_mut().remove(&contract_id) // todo: may not work + self.index2.contracts_mut().remove(contract_id) } pub fn insert_contract_index( @@ -187,9 +186,9 @@ impl CommitHulk { match index { Some(p) => self .index2 - .get(&contract_id, self.maybe_hash) - .or_else(move || p.get(&contract_id, self.maybe_hash)), - None => self.index2.get(&contract_id, self.maybe_hash), + .get(contract_id, self.maybe_hash) + .or_else(move || p.get(contract_id, self.maybe_hash)), + None => self.index2.get(contract_id, self.maybe_hash), } } @@ -206,10 +205,10 @@ impl CommitHulk { let index = self.index.map(|p| unsafe { p.as_ref().unwrap() }); match index { Some(p) => { - self.index2.contains_key(&contract_id) - || p.contains_key(&contract_id) + self.index2.contains_key(contract_id) + || p.contains_key(contract_id) } - None => self.index2.contains_key(&contract_id), + None => self.index2.contains_key(contract_id), } } diff --git a/piecrust/src/store/session.rs b/piecrust/src/store/session.rs index ea82c2b4..083726f0 100644 --- a/piecrust/src/store/session.rs +++ b/piecrust/src/store/session.rs @@ -146,16 +146,9 @@ impl ContractSession { let (replier, receiver) = mpsc::sync_channel(1); let mut contracts = BTreeMap::new(); - let base = self.base.as_ref().map(|c| //Commit { - // index: c.index.clone(), - // contracts_merkle: c.contracts_merkle.clone(), - // maybe_hash: c.maybe_hash, - c.to_commit() /* } */); - - let mut base_clone = base.as_ref().map(|b| b.to_hulk()); // todo: wasteful, think of a better way + let base = self.base.as_ref().map(|c| c.to_commit()); mem::swap(&mut self.contracts, &mut contracts); - mem::swap(&mut self.base, &mut base_clone); self.call .send(Call::Commit { From f2be7120952d7c2b507d6c720d6c80c83e4c1b32 Mon Sep 17 00:00:00 2001 From: Milosz Muszynski Date: Thu, 7 Nov 2024 10:33:18 +0100 Subject: [PATCH 4/4] piecrust: removed commented out code --- piecrust/src/store/commit.rs | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/piecrust/src/store/commit.rs b/piecrust/src/store/commit.rs index a3ee13b4..6d6420cd 100644 --- a/piecrust/src/store/commit.rs +++ b/piecrust/src/store/commit.rs @@ -55,11 +55,6 @@ impl CommitHulk { maybe_hash: self.maybe_hash, }, } - // for (contract_id, element) in self.index2.contracts().iter() { - // commit - // .index - // .insert_contract_index(&contract_id, element.clone()) - // } } pub fn fast_clone<'a>( @@ -147,11 +142,6 @@ impl CommitHulk { (&mut self.index2, &mut self.contracts_merkle) } - // pub fn remove_and_insert(&mut self, contract: ContractId, memory: - // &Memory) { self.remove_contract_index(&contract); - // self.insert(contract, memory); - // } - pub fn root(&self) -> Ref { tracing::trace!("calculating root started"); let ret = self.contracts_merkle.root(); @@ -192,15 +182,6 @@ impl CommitHulk { } } - // note, does not allow changing the original - // pub fn index_get_mut( - // &mut self, - // contract_id: &ContractId, - // ) -> Option<&mut ContractIndexElement> { - // self.index2 - // .get_mut(&contract_id, self.maybe_hash) - // } - pub fn index_contains_key(&self, contract_id: &ContractId) -> bool { let index = self.index.map(|p| unsafe { p.as_ref().unwrap() }); match index { @@ -211,10 +192,4 @@ impl CommitHulk { None => self.index2.contains_key(contract_id), } } - - // pub fn index_iter( - // &self, - // ) -> impl Iterator { - // self.index.contracts().iter() - // } }