From 885d8f9eff7006fa0d9e214248bb9d1ca43d2e78 Mon Sep 17 00:00:00 2001 From: th7nder Date: Sat, 2 Nov 2024 14:57:15 +0100 Subject: [PATCH 01/19] feat(polka-storage-provider): add loading PoRep parameters on-startup --- cli/polka-storage-provider/server/src/main.rs | 24 ++++++++++++++++++- lib/polka-storage-proofs/src/porep/mod.rs | 6 ++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/cli/polka-storage-provider/server/src/main.rs b/cli/polka-storage-provider/server/src/main.rs index 379798477..9d3afe674 100644 --- a/cli/polka-storage-provider/server/src/main.rs +++ b/cli/polka-storage-provider/server/src/main.rs @@ -11,6 +11,7 @@ use std::{env::temp_dir, net::SocketAddr, path::PathBuf, sync::Arc, time::Durati use clap::Parser; use pipeline::types::PipelineMessage; +use polka_storage_proofs::porep::{self, PoRepParameters}; use polka_storage_provider_common::rpc::ServerInfo; use primitives_proofs::{RegisteredPoStProof, RegisteredSealProof}; use rand::Rng; @@ -118,6 +119,9 @@ pub enum ServerError { #[error("proof sectors sizes do not match")] SectorSizeMismatch, + #[error("failed to load PoRep parameters from: {0}, because: {1}")] + InvalidPoRepParameters(std::path::PathBuf, porep::PoRepError), + #[error("FromEnv error: {0}")] EnvFilter(#[from] tracing_subscriber::filter::FromEnvError), @@ -194,13 +198,22 @@ pub struct ServerArguments { /// Proof of Spacetime proof type. #[arg(long)] post_proof: RegisteredPoStProof, + + /// Proving Parameters for PoRep proof, corresponding to given `seal_proof` sector size. + /// They are shared across all of the nodes in the network, as the chain stores corresponding Verifying Key parameters. + /// Shared parameters available to get in the [root repo](http://github.com/eigerco/polka-storage/README.md#Parameters). + /// + /// Testing/temporary parameters can be also generated via `polka-storage-provider-client proofs porep-params` command. + /// Note that when you generate keys, for local testnet, + /// **they need to be set** via an extrinsic pallet-proofs::set_porep_verifyingkey. + #[arg(long)] + porep_parameters: PathBuf, } /// A valid server configuration. To be created using [`ServerConfiguration::try_from`]. /// /// The main difference to [`Server`] is that this structure only contains validated and /// ready to use parameters. -#[derive(Debug)] pub struct ServerConfiguration { /// Storage server listen address. upload_listen_address: SocketAddr, @@ -226,6 +239,11 @@ pub struct ServerConfiguration { /// Proof of Spacetime proof type. post_proof: RegisteredPoStProof, + + /// Proving Parameters for PoRep proof + /// For 2KiB sectors they're ~1GiB of data. + /// We may load them on demand someday. + porep_parameters: PoRepParameters, } impl TryFrom for ServerConfiguration { @@ -264,6 +282,9 @@ impl TryFrom for ServerConfiguration { }); std::fs::create_dir_all(&storage_directory)?; + let porep_parameters = porep::load_groth16_parameters(value.porep_parameters.clone()) + .map_err(|e| ServerError::InvalidPoRepParameters(value.porep_parameters, e))?; + Ok(Self { upload_listen_address: value.upload_listen_address, rpc_listen_address: value.rpc_listen_address, @@ -273,6 +294,7 @@ impl TryFrom for ServerConfiguration { storage_directory, seal_proof: value.seal_proof, post_proof: value.post_proof, + porep_parameters, }) } } diff --git a/lib/polka-storage-proofs/src/porep/mod.rs b/lib/polka-storage-proofs/src/porep/mod.rs index da0c34f72..828c4b2e8 100644 --- a/lib/polka-storage-proofs/src/porep/mod.rs +++ b/lib/polka-storage-proofs/src/porep/mod.rs @@ -13,6 +13,8 @@ use storage_proofs_porep::stacked::StackedDrg; use crate::types::Commitment; +pub type PoRepParameters = groth16::MappedParameters; + /// Generates parameters for proving and verifying PoRep. /// It should be called once and then reused across provers and the verifier. /// Verifying Key is only needed for verification (no_std), rest of the params are required for proving (std). @@ -35,9 +37,7 @@ pub fn generate_random_groth16_parameters( /// Loads Groth16 parameters from the specified path. /// Parameters needed to be serialized with [`groth16::Paramters::::write_bytes`]. -pub fn load_groth16_parameters( - path: std::path::PathBuf, -) -> Result, PoRepError> { +pub fn load_groth16_parameters(path: std::path::PathBuf) -> Result { groth16::Parameters::::build_mapped_parameters(path.clone(), false) .map_err(|e| PoRepError::FailedToLoadGrothParameters(path, e)) } From 1fa1784d8889b89437e63f713215089795ef0977 Mon Sep 17 00:00:00 2001 From: th7nder Date: Sun, 3 Nov 2024 11:33:31 +0100 Subject: [PATCH 02/19] feat(polka-storage-provider-server): ProveCommit pipeline step --- Cargo.lock | 1 + cli/polka-storage-provider/server/Cargo.toml | 3 +- cli/polka-storage-provider/server/src/main.rs | 1 + .../server/src/pipeline/mod.rs | 111 +++++++++++++++++- .../server/src/pipeline/types.rs | 12 ++ .../storagext-cli/src/cmd/storage_provider.rs | 19 ++- .../storagext/src/clients/storage_provider.rs | 6 +- lib/polka-storage-proofs/src/porep/sealer.rs | 3 + primitives/commitment/src/lib.rs | 10 ++ 9 files changed, 146 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 311fea06c..c4e1827e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9499,6 +9499,7 @@ dependencies = [ "futures", "jsonrpsee 0.23.2", "mater", + "parity-scale-codec", "polka-storage-proofs", "polka-storage-provider-common", "primitives-commitment", diff --git a/cli/polka-storage-provider/server/Cargo.toml b/cli/polka-storage-provider/server/Cargo.toml index 2f02f2f67..26494dc60 100644 --- a/cli/polka-storage-provider/server/Cargo.toml +++ b/cli/polka-storage-provider/server/Cargo.toml @@ -10,7 +10,7 @@ version = "0.1.0" [dependencies] # "Homegrown" crates mater = { workspace = true } -polka-storage-proofs = { workspace = true, features = ["std"] } +polka-storage-proofs = { workspace = true, features = ["std", "substrate"] } polka-storage-provider-common = { workspace = true } primitives-commitment = { workspace = true, features = ["serde", "std"] } primitives-proofs = { workspace = true, features = ["clap"] } @@ -20,6 +20,7 @@ async-trait = { workspace = true } axum = { workspace = true, features = ["macros", "multipart"] } cid = { workspace = true, features = ["serde", "std"] } clap = { workspace = true, features = ["derive"] } +codec = { workspace = true } futures = { workspace = true } jsonrpsee = { workspace = true, features = ["http-client", "macros", "server", "ws-client"] } rand = { workspace = true } diff --git a/cli/polka-storage-provider/server/src/main.rs b/cli/polka-storage-provider/server/src/main.rs index 9d3afe674..b0dd80bc6 100644 --- a/cli/polka-storage-provider/server/src/main.rs +++ b/cli/polka-storage-provider/server/src/main.rs @@ -417,6 +417,7 @@ impl ServerConfiguration { unsealed_sectors_dir: unsealed_sector_storage_dir, sealed_sectors_dir: sealed_sector_storage_dir, sealing_cache_dir, + porep_parameters: Arc::new(self.porep_parameters), xt_client, xt_keypair: self.multi_pair_signer, pipeline_sender: pipeline_tx, diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index 2a7b98595..3ce02f474 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -3,14 +3,17 @@ pub mod types; use std::{path::PathBuf, sync::Arc}; use polka_storage_proofs::porep::{ - sealer::{prepare_piece, PreCommitOutput, Sealer}, - PoRepError, + sealer::{prepare_piece, PreCommitOutput, Proof, Sealer, SubstrateProof}, + PoRepError, PoRepParameters, }; use polka_storage_provider_common::rpc::ServerInfo; use primitives_commitment::Commitment; use primitives_proofs::{derive_prover_id, SectorNumber}; use storagext::{ - types::{market::DealProposal, storage_provider::SectorPreCommitInfo}, + types::{ + market::DealProposal, + storage_provider::{ProveCommitSector, SectorPreCommitInfo}, + }, StorageProviderClientExt, SystemClientExt, }; use subxt::tx::Signer; @@ -24,13 +27,13 @@ use types::{AddPieceMessage, PipelineMessage, PreCommitMessage}; use self::types::Sector; use crate::{ db::{DBError, DealDB}, - pipeline::types::SectorState, + pipeline::types::{ProveCommitMessage, SectorState}, }; // PLACEHOLDERS!!!!! // TODO(@th7nder,29/10/2024): get from pallet randomness const TICKET: [u8; 32] = [12u8; 32]; -// const SEED: [u8; 32] = [13u8; 32]; +const SEED: [u8; 32] = [13u8; 32]; const SECTOR_EXPIRATION_MARGIN: u64 = 20; #[derive(Debug, thiserror::Error)] @@ -57,6 +60,7 @@ pub struct PipelineState { pub unsealed_sectors_dir: Arc, pub sealed_sectors_dir: Arc, pub sealing_cache_dir: Arc, + pub porep_parameters: Arc, pub xt_client: Arc, pub xt_keypair: storagext::multipair::MultiPairSigner, @@ -145,6 +149,22 @@ fn process( } }); } + PipelineMessage::ProveCommit(ProveCommitMessage { sector_number }) => { + tracker.spawn(async move { + // ProveCommit is not cancellation safe. + match prove_commit(state, sector_number).await { + Ok(_) => { + tracing::info!( + "ProveCommit for sector {} finished successfully.", + sector_number + ) + } + Err(err) => { + tracing::error!(%err, "Failed ProveCommit for Sector: {}", sector_number) + } + } + }); + } } } @@ -260,6 +280,8 @@ async fn precommit( tracing::info!("Created sector's replica: {:?}", sealing_output); sector.state = SectorState::Sealed; + sector.comm_r = Some(Commitment::replica(sealing_output.comm_r)); + sector.comm_d = Some(Commitment::data(sealing_output.comm_d)); state.db.save_sector(§or)?; let current_block = state.xt_client.height(false).await?; @@ -314,5 +336,84 @@ async fn precommit( precommited_sectors ); + state + .pipeline_sender + .send(PipelineMessage::ProveCommit(ProveCommitMessage { + sector_number: sector.sector_number, + }))?; + + Ok(()) +} + +#[tracing::instrument(skip_all, fields(sector_number))] +async fn prove_commit( + state: Arc, + sector_number: SectorNumber, +) -> Result<(), PipelineError> { + tracing::info!("Starting prove commit"); + + let sealer = Sealer::new(state.server_info.seal_proof); + let Some(mut sector) = state.db.get_sector(sector_number)? else { + tracing::error!("Tried to precommit non-existing sector"); + return Err(PipelineError::NotExistentSector); + }; + + let sealing_handle: JoinHandle, _>> = { + let porep_params = state.porep_parameters.clone(); + let prover_id = derive_prover_id(state.xt_keypair.account_id()); + let cache_dir = state.sealing_cache_dir.clone(); + let sealed_path = sector.sealed_path.clone(); + let piece_infos = sector.piece_infos.clone(); + let comm_r = sector.comm_r.unwrap().raw(); + let comm_d = sector.comm_d.unwrap().raw(); + tokio::task::spawn_blocking(move || { + sealer.prove_sector( + porep_params.as_ref(), + cache_dir.as_ref(), + sealed_path, + prover_id, + sector_number, + TICKET, + Some(SEED), + PreCommitOutput { comm_r, comm_d }, + &piece_infos, + ) + }) + }; + let proofs = sealing_handle.await??; + + // We use sector size 2KiB only at this point, which guarantees to have 1 proof, because it has 1 partition in the config. + // That's why `prove_commit` will always generate a 1 proof. + let proof: SubstrateProof = proofs[0] + .clone() + .try_into() + .expect("converstion between rust-fil-proofs and polka-storage-proofs to work"); + let proof = codec::Encode::encode(&proof); + tracing::info!("Proven sector: {}", sector_number); + + sector.state = SectorState::Proven; + state.db.save_sector(§or)?; + + let result = state + .xt_client + .prove_commit_sectors( + &state.xt_keypair, + vec![ProveCommitSector { + sector_number, + proof, + }], + ) + .await?; + + let proven_sectors = result + .events + .find::() + .collect::, _>>()?; + + tracing::info!("Successfully proven sectors on-chain: {:?}", proven_sectors); + + sector.state = SectorState::ProveCommitted; + state.db.save_sector(§or)?; + Ok(()) } diff --git a/cli/polka-storage-provider/server/src/pipeline/types.rs b/cli/polka-storage-provider/server/src/pipeline/types.rs index 96de7e487..8d10e024c 100644 --- a/cli/polka-storage-provider/server/src/pipeline/types.rs +++ b/cli/polka-storage-provider/server/src/pipeline/types.rs @@ -12,6 +12,7 @@ pub enum PipelineMessage { AddPiece(AddPieceMessage), /// Pads, seals the sector and pre-commits it on chain PreCommit(PreCommitMessage), + ProveCommit(ProveCommitMessage), } /// Deal to be added to a sector with its contents. @@ -34,6 +35,12 @@ pub struct PreCommitMessage { pub sector_number: SectorNumber, } +#[derive(Debug)] +pub struct ProveCommitMessage { + /// Number of an existing, pre-committed sector + pub sector_number: SectorNumber, +} + /// Sector State serialized and stored in the RocksDB database /// It is used for tracking the sector lifetime, precommiting and proving. #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] @@ -66,6 +73,8 @@ pub struct Sector { /// Only after pipeline [`PipelineMessage::PreCommit`], /// the file has contents which should not be touched and are used for later steps. pub sealed_path: std::path::PathBuf, + pub comm_r: Option, + pub comm_d: Option, } impl Sector { @@ -89,6 +98,8 @@ impl Sector { deals: vec![], unsealed_path, sealed_path, + comm_r: None, + comm_d: None, }) } } @@ -104,4 +115,5 @@ pub enum SectorState { Precommitted, /// After a PoRep for a sector has been created and publish on-chain. Proven, + ProveCommitted, } diff --git a/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs b/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs index f3927df87..8fcb3f5ba 100644 --- a/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs +++ b/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs @@ -320,17 +320,14 @@ impl StorageProviderCommand { where Client: StorageProviderClientExt, { - let (sector_numbers, prove_commit_sectors): ( - Vec, - Vec, - ) = prove_commit_sectors - .into_iter() - .map(|s| { - let sector_number = s.sector_number; - (sector_number, s.into()) - }) - .unzip(); - + let (sector_numbers, prove_commit_sectors): (Vec, Vec) = + prove_commit_sectors + .into_iter() + .map(|s| { + let sector_number = s.sector_number; + (sector_number, s.into()) + }) + .unzip(); let submission_result = client .prove_commit_sectors( &account_keypair, diff --git a/cli/polka-storage/storagext/src/clients/storage_provider.rs b/cli/polka-storage/storagext/src/clients/storage_provider.rs index 1b5bdf56c..790740d7a 100644 --- a/cli/polka-storage/storagext/src/clients/storage_provider.rs +++ b/cli/polka-storage/storagext/src/clients/storage_provider.rs @@ -13,13 +13,13 @@ use crate::{ bounded_vec::IntoBoundedByteVec, client::SubmissionResult, runtime_types::pallet_storage_provider::{ - proofs::SubmitWindowedPoStParams, sector::ProveCommitSector, - storage_provider::StorageProviderState, + proofs::SubmitWindowedPoStParams, storage_provider::StorageProviderState, }, storage_provider::calls::types::register_storage_provider::PeerId, }, types::storage_provider::{ - FaultDeclaration, RecoveryDeclaration, SectorPreCommitInfo, TerminationDeclaration, + FaultDeclaration, ProveCommitSector, RecoveryDeclaration, SectorPreCommitInfo, + TerminationDeclaration, }, BlockNumber, Currency, PolkaStorageConfig, }; diff --git a/lib/polka-storage-proofs/src/porep/sealer.rs b/lib/polka-storage-proofs/src/porep/sealer.rs index eefc066ca..788becac5 100644 --- a/lib/polka-storage-proofs/src/porep/sealer.rs +++ b/lib/polka-storage-proofs/src/porep/sealer.rs @@ -22,6 +22,9 @@ use crate::{ ZeroPaddingReader, }; +pub type Proof = groth16::Proof; +pub type SubstrateProof = crate::Proof; + /// Prepares an arbitrary piece to be used by [`Sealer::create_sector`]. /// /// It does so by calculating the proper size for the padded reader diff --git a/primitives/commitment/src/lib.rs b/primitives/commitment/src/lib.rs index e62c9aa39..2b0ee0b00 100644 --- a/primitives/commitment/src/lib.rs +++ b/primitives/commitment/src/lib.rs @@ -80,6 +80,16 @@ impl Commitment { Self::new(commitment, CommitmentKind::Piece) } + /// Create a new data commitment. + pub fn data(commitment: [u8; 32]) -> Self { + Self::new(commitment, CommitmentKind::Data) + } + + /// Create a new replica commitment. + pub fn replica(commitment: [u8; 32]) -> Self { + Self::new(commitment, CommitmentKind::Replica) + } + /// Creates a new `Commitment` from bytes of a valid CID. /// Returns an error if the bytes passed do not represent a valid commitment. pub fn from_cid_bytes(bytes: &[u8], kind: CommitmentKind) -> Result { From 3a15e979776c163c2e4ff32c5ca0a82ce49f0140 Mon Sep 17 00:00:00 2001 From: th7nder Date: Sun, 3 Nov 2024 11:56:53 +0100 Subject: [PATCH 03/19] test: prove commit --- .../storagext-cli/src/cmd/storage_provider.rs | 5 +---- examples/random.car | Bin 0 -> 225 bytes examples/random.txt | 1 + examples/rpc_publish.sh | 2 +- examples/test-data-big.car | Bin 0 -> 2032 bytes examples/test-data-small.car | Bin 0 -> 254 bytes examples/test-data-small.txt | 9 +++++++++ 7 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 examples/random.car create mode 100644 examples/random.txt create mode 100644 examples/test-data-big.car create mode 100644 examples/test-data-small.car create mode 100644 examples/test-data-small.txt diff --git a/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs b/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs index 8fcb3f5ba..2ba82f5a5 100644 --- a/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs +++ b/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs @@ -5,10 +5,7 @@ use primitives_proofs::{RegisteredPoStProof, SectorNumber}; use storagext::{ deser::DeserializablePath, multipair::MultiPairSigner, - runtime::{ - runtime_types::pallet_storage_provider::sector::ProveCommitSector as RuntimeProveCommitSector, - SubmissionResult, - }, + runtime:: SubmissionResult, types::storage_provider::{ FaultDeclaration as SxtFaultDeclaration, ProveCommitSector as SxtProveCommitSector, RecoveryDeclaration as SxtRecoveryDeclaration, diff --git a/examples/random.car b/examples/random.car new file mode 100644 index 0000000000000000000000000000000000000000..a832cd1ec049d1db024ec0252e49dc3864050b47 GIT binary patch literal 225 zcmd;Dm|m7zRGgWg$HagJjG_DtC_NiWTP;d0%Fiz;ZoHutp~}D*Dx|EGsy!`S;4o0AjLQq>lYz-&| M2@$f>8l(sa065`84gdfE literal 0 HcmV?d00001 diff --git a/examples/random.txt b/examples/random.txt new file mode 100644 index 000000000..e97d3423b --- /dev/null +++ b/examples/random.txt @@ -0,0 +1 @@ +idontnow \ No newline at end of file diff --git a/examples/rpc_publish.sh b/examples/rpc_publish.sh index 0663bed08..a085aa965 100755 --- a/examples/rpc_publish.sh +++ b/examples/rpc_publish.sh @@ -57,7 +57,7 @@ DEAL_JSON=$( ) SIGNED_DEAL_JSON="$(RUST_LOG=error target/release/polka-storage-provider-client client sign-deal --sr25519-key "$CLIENT" "$DEAL_JSON")" -(RUST_LOG=debug target/release/polka-storage-provider-server --sr25519-key "$PROVIDER" --seal-proof "2KiB" --post-proof "2KiB") & +(RUST_LOG=debug target/release/polka-storage-provider-server --sr25519-key "$PROVIDER" --seal-proof "2KiB" --post-proof "2KiB" --porep-parameters 2KiB.porep.params) & sleep 5 # gives time for the server to start DEAL_CID="$(RUST_LOG=error target/release/polka-storage-provider-client client propose-deal "$DEAL_JSON")" diff --git a/examples/test-data-big.car b/examples/test-data-big.car new file mode 100644 index 0000000000000000000000000000000000000000..0910f83918f18f269e46954009aeb7e3a5916d27 GIT binary patch literal 2032 zcmd;Dm|m7zRGgWg$HagJjG_E;b_in?l(t%wT9lt(Qrvh$D?*ilF;qyQ$*%t3k^3`u z&y1X|&%1Ejab@wu>rZa|ZlCbD;SGDTwH(X}#$9~GC^s}RHZe6bx3Dzi!p+1Zg`}67 zs&Kmn$-+@ZCZj<>?;xONDyP3$qv?g->18y;i3@RR<^`lWf;?s;2P3c^7J^pFAhre+ OgM Date: Mon, 4 Nov 2024 14:55:57 +0100 Subject: [PATCH 04/19] fix: use proper randomness in porep --- .../server/src/pipeline/mod.rs | 39 ++++++++++++++----- .../storagext-cli/src/cmd/storage_provider.rs | 2 +- cli/polka-storage/storagext/src/lib.rs | 2 +- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index 3ce02f474..df9331ab4 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -8,15 +8,19 @@ use polka_storage_proofs::porep::{ }; use polka_storage_provider_common::rpc::ServerInfo; use primitives_commitment::Commitment; -use primitives_proofs::{derive_prover_id, SectorNumber}; +use primitives_proofs::{ + derive_prover_id, + randomness::{draw_randomness, DomainSeparationTag}, + SectorNumber, +}; use storagext::{ types::{ market::DealProposal, storage_provider::{ProveCommitSector, SectorPreCommitInfo}, }, - StorageProviderClientExt, SystemClientExt, + RandomnessClientExt, StorageProviderClientExt, SystemClientExt, }; -use subxt::tx::Signer; +use subxt::{ext::codec::Encode, tx::Signer}; use tokio::{ sync::mpsc::{error::SendError, UnboundedReceiver, UnboundedSender}, task::{JoinError, JoinHandle}, @@ -30,10 +34,6 @@ use crate::{ pipeline::types::{ProveCommitMessage, SectorState}, }; -// PLACEHOLDERS!!!!! -// TODO(@th7nder,29/10/2024): get from pallet randomness -const TICKET: [u8; 32] = [12u8; 32]; -const SEED: [u8; 32] = [13u8; 32]; const SECTOR_EXPIRATION_MARGIN: u64 = 20; #[derive(Debug, thiserror::Error)] @@ -50,6 +50,8 @@ pub enum PipelineError { DBError(#[from] DBError), #[error("sector does not exist")] NotExistentSector, + #[error("precommit scheduled too early, randomness not available")] + NotAvailableRandomness, #[error(transparent)] SendError(#[from] SendError), } @@ -256,7 +258,25 @@ async fn precommit( sector.piece_infos = sealer.pad_sector(§or.piece_infos, sector.occupied_sector_space)?; tracing::debug!("piece_infos: {:?}", sector.piece_infos); - tracing::info!("Padded sector, commencing pre-commit."); + tracing::info!("Padded sector, commencing pre-commit and getting last finalized block"); + + let current_block = state.xt_client.height(true).await?; + tracing::info!("Current block: {current_block}"); + + let Some(digest) = state.xt_client.get_randomness(current_block).await? else { + tracing::error!("Precommit was scheduled too early, when the randomness on-chain was not yet available..."); + return Err(PipelineError::NotAvailableRandomness); + }; + let entropy = state.xt_keypair.account_id().encode(); + // Must match pallet's logic or otherwise proof won't be verified: + // https://github.com/eigerco/polka-storage/blob/af51a9b121c9b02e0bf6f02f5e835091ab46af76/pallets/storage-provider/src/lib.rs#L1539 + let ticket = draw_randomness( + &digest, + DomainSeparationTag::SealRandomness, + current_block, + &entropy, + ); + // TODO(@th7nder,31/10/2024): what happens if some of the process fails? SP will be slashed, and there is no error reporting? what about retries? let sealing_handle: JoinHandle> = { let prover_id = derive_prover_id(state.xt_keypair.account_id()); @@ -271,7 +291,7 @@ async fn precommit( sealed_path, prover_id, sector_number, - TICKET, + ticket, &piece_infos, ) }) @@ -284,7 +304,6 @@ async fn precommit( sector.comm_d = Some(Commitment::data(sealing_output.comm_d)); state.db.save_sector(§or)?; - let current_block = state.xt_client.height(false).await?; tracing::debug!("Precommiting at block: {}", current_block); let result = state diff --git a/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs b/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs index 2ba82f5a5..dd8187470 100644 --- a/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs +++ b/cli/polka-storage/storagext-cli/src/cmd/storage_provider.rs @@ -5,7 +5,7 @@ use primitives_proofs::{RegisteredPoStProof, SectorNumber}; use storagext::{ deser::DeserializablePath, multipair::MultiPairSigner, - runtime:: SubmissionResult, + runtime::SubmissionResult, types::storage_provider::{ FaultDeclaration as SxtFaultDeclaration, ProveCommitSector as SxtProveCommitSector, RecoveryDeclaration as SxtRecoveryDeclaration, diff --git a/cli/polka-storage/storagext/src/lib.rs b/cli/polka-storage/storagext/src/lib.rs index f7feb217d..54186b246 100644 --- a/cli/polka-storage/storagext/src/lib.rs +++ b/cli/polka-storage/storagext/src/lib.rs @@ -8,7 +8,7 @@ pub mod types; pub mod deser; pub use crate::{ - clients::{MarketClientExt, StorageProviderClientExt, SystemClientExt}, + clients::{MarketClientExt, RandomnessClientExt, StorageProviderClientExt, SystemClientExt}, runtime::{bounded_vec::IntoBoundedByteVec, client::Client}, }; From 58a0aaa2df2df05d9c11dcbcd183ed74c42edf3e Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Mon, 4 Nov 2024 16:42:26 +0100 Subject: [PATCH 05/19] chore: remove files from the PR --- examples/random.car | Bin 225 -> 0 bytes examples/random.txt | 1 - examples/test-data-big.car | Bin 2032 -> 0 bytes examples/test-data-small.car | Bin 254 -> 0 bytes examples/test-data-small.txt | 9 --------- 5 files changed, 10 deletions(-) delete mode 100644 examples/random.car delete mode 100644 examples/random.txt delete mode 100644 examples/test-data-big.car delete mode 100644 examples/test-data-small.car delete mode 100644 examples/test-data-small.txt diff --git a/examples/random.car b/examples/random.car deleted file mode 100644 index a832cd1ec049d1db024ec0252e49dc3864050b47..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 225 zcmd;Dm|m7zRGgWg$HagJjG_DtC_NiWTP;d0%Fiz;ZoHutp~}D*Dx|EGsy!`S;4o0AjLQq>lYz-&| M2@$f>8l(sa065`84gdfE diff --git a/examples/random.txt b/examples/random.txt deleted file mode 100644 index e97d3423b..000000000 --- a/examples/random.txt +++ /dev/null @@ -1 +0,0 @@ -idontnow \ No newline at end of file diff --git a/examples/test-data-big.car b/examples/test-data-big.car deleted file mode 100644 index 0910f83918f18f269e46954009aeb7e3a5916d27..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2032 zcmd;Dm|m7zRGgWg$HagJjG_E;b_in?l(t%wT9lt(Qrvh$D?*ilF;qyQ$*%t3k^3`u z&y1X|&%1Ejab@wu>rZa|ZlCbD;SGDTwH(X}#$9~GC^s}RHZe6bx3Dzi!p+1Zg`}67 zs&Kmn$-+@ZCZj<>?;xONDyP3$qv?g->18y;i3@RR<^`lWf;?s;2P3c^7J^pFAhre+ OgM Date: Mon, 4 Nov 2024 19:28:35 +0100 Subject: [PATCH 06/19] fix: make prove commit work somehow --- .../server/src/pipeline/mod.rs | 39 ++++++++++++++++++- .../server/src/pipeline/types.rs | 2 + 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index df9331ab4..2ef5f36db 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -302,6 +302,7 @@ async fn precommit( sector.state = SectorState::Sealed; sector.comm_r = Some(Commitment::replica(sealing_output.comm_r)); sector.comm_d = Some(Commitment::data(sealing_output.comm_d)); + sector.precommit_block = Some(current_block); state.db.save_sector(§or)?; tracing::debug!("Precommiting at block: {}", current_block); @@ -377,6 +378,40 @@ async fn prove_commit( return Err(PipelineError::NotExistentSector); }; + let precommit_block = sector.precommit_block.unwrap(); + let Some(digest) = state.xt_client.get_randomness(precommit_block).await? else { + tracing::error!("Out-of-the-state transition, this SHOULD not happen"); + return Err(PipelineError::NotAvailableRandomness); + }; + let entropy = state.xt_keypair.account_id().encode(); + // Must match pallet's logic or otherwise proof won't be verified: + // https://github.com/eigerco/polka-storage/blob/af51a9b121c9b02e0bf6f02f5e835091ab46af76/pallets/storage-provider/src/lib.rs#L1539 + let ticket = draw_randomness( + &digest, + DomainSeparationTag::SealRandomness, + precommit_block, + &entropy, + ); + + // TODO(@th7nder,04/11/2024): + // https://github.com/eigerco/polka-storage/blob/5edd4194f08f29d769c277577ccbb70bb6ff63bc/runtime/src/configs/mod.rs#L360 + // 10 blocks = 1 minute, only testnet + const PRECOMMIT_CHALLENGE_DELAY: u64 = 10; + let prove_commit_block = precommit_block + PRECOMMIT_CHALLENGE_DELAY; + + tracing::info!("Wait for block {} to get randomness", prove_commit_block); + state.xt_client.wait_for_height(prove_commit_block, true).await?; + let Some(digest) = state.xt_client.get_randomness(prove_commit_block).await? else { + tracing::error!("Randomness for the block not available."); + return Err(PipelineError::NotAvailableRandomness); + }; + let seed = draw_randomness( + &digest, + DomainSeparationTag::InteractiveSealChallengeSeed, + precommit_block, + &entropy, + ); + let sealing_handle: JoinHandle, _>> = { let porep_params = state.porep_parameters.clone(); let prover_id = derive_prover_id(state.xt_keypair.account_id()); @@ -392,8 +427,8 @@ async fn prove_commit( sealed_path, prover_id, sector_number, - TICKET, - Some(SEED), + ticket, + Some(seed), PreCommitOutput { comm_r, comm_d }, &piece_infos, ) diff --git a/cli/polka-storage-provider/server/src/pipeline/types.rs b/cli/polka-storage-provider/server/src/pipeline/types.rs index 8d10e024c..1a3bdb395 100644 --- a/cli/polka-storage-provider/server/src/pipeline/types.rs +++ b/cli/polka-storage-provider/server/src/pipeline/types.rs @@ -75,6 +75,7 @@ pub struct Sector { pub sealed_path: std::path::PathBuf, pub comm_r: Option, pub comm_d: Option, + pub precommit_block: Option, } impl Sector { @@ -100,6 +101,7 @@ impl Sector { sealed_path, comm_r: None, comm_d: None, + precommit_block: None, }) } } From 10df03c506128acc7e2fa949cee14b5ee2d8fe70 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Tue, 5 Nov 2024 10:31:56 +0100 Subject: [PATCH 07/19] fix: prove_commit_block in pipeline.rs --- cli/polka-storage-provider/server/src/pipeline/mod.rs | 2 +- pallets/storage-provider/src/lib.rs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index 2ef5f36db..6f27f28c7 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -408,7 +408,7 @@ async fn prove_commit( let seed = draw_randomness( &digest, DomainSeparationTag::InteractiveSealChallengeSeed, - precommit_block, + prove_commit_block, &entropy, ); diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index aba3a6ee4..ac739a374 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -1513,6 +1513,9 @@ pub mod pallet { // Check if we are too early with the proof submit let interactive_block_number = precommit.pre_commit_block_number + T::PreCommitChallengeDelay::get(); + + // check out this interactive block number cause i'm confused + // would need to fetch it after pre-commit if current_block_number < interactive_block_number { log::error!(target: LOG_TARGET, "too early to prove sector: current_block_number: {current_block_number:?} < interactive_block_number: {interactive_block_number:?}"); return Err(Error::::InvalidProof)?; From 38b0a0ab755aa2d3047e5347bd2bbf1576c695c4 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Tue, 5 Nov 2024 11:05:15 +0100 Subject: [PATCH 08/19] fix: make it work again --- cli/polka-storage-provider/server/src/pipeline/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index 6f27f28c7..bf5114adb 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -456,12 +456,15 @@ async fn prove_commit( sector_number, proof, }], + true, ) - .await?; + .await? + .unwrap(); let proven_sectors = result .events .find::() + .map(|result| result.map_err(|err| subxt::Error::from(err))) .collect::, _>>()?; tracing::info!("Successfully proven sectors on-chain: {:?}", proven_sectors); From 25bb35b4928fd0ebd97727a093b8cb6178abd4a1 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Tue, 5 Nov 2024 13:10:07 +0100 Subject: [PATCH 09/19] fix: make the numbers match between the pallet --- Cargo.lock | 2 ++ cli/artifacts/metadata.scale | Bin 152362 -> 152575 bytes cli/polka-storage-provider/server/Cargo.toml | 1 + .../server/src/pipeline/mod.rs | 29 ++++++++++++------ .../server/src/pipeline/types.rs | 2 ++ .../src/runtime/display/storage_provider.rs | 10 ++++-- pallets/storage-provider/Cargo.toml | 2 ++ pallets/storage-provider/src/lib.rs | 8 +++++ .../src/tests/pre_commit_sectors.rs | 3 ++ 9 files changed, 45 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c4e1827e0..fef7d5510 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8737,6 +8737,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", + "hex", "log", "multihash-codetable", "pallet-balances", @@ -9497,6 +9498,7 @@ dependencies = [ "cid 0.11.1", "clap", "futures", + "hex", "jsonrpsee 0.23.2", "mater", "parity-scale-codec", diff --git a/cli/artifacts/metadata.scale b/cli/artifacts/metadata.scale index dbce6c54bed30b28ff5db7526d7d41a3d3d15b61..62c06839d5f2ac5b9a6be15ce0bd705ea73ac0b7 100644 GIT binary patch delta 3369 zcmZu!eQ;FO6~Di`?>%-U$P$x~1i=?ziQ#jTCIT%fE*LaOLP7%d14uTTmt!hVFr zpqQA9pk^f9h6@!ep+-SP8!de@Bi7i`8SJ1&3^r3!jV*s1Vdr9&fTD?Rhkx1>qD9+==Xag5v_XZ3=~FGZ&;0nwQ98{ z7*Z>}fru7zdm`Qjts>+OR0sV5EgY^?YqbE;Bi`U`0o5B&D@#Jb25qegRJru6(!lB} z497Y@ZkA@qI2jxMVC!4H{$szyfIht%n_wA~ApW)kbM#gX&j3UEt+iOr@wUi#0jWtv zt_Ld03S1j1N;YlWxUOQ&gXINfy4#CWK)?910kgz*FQ)K*DRz0IGxpI#iF`ncwKS%Q z@*VJq0v|DavIAj0WKh*u@ot-__rb%5rPvE?Df-tw?3Qs!uM44p^Gha5%7`SDF($ET zGE%o_9ts{~BPPFC6~+4LqY|@%q7|$UhTXoaqAQ9usiP9^*;rHKsdIY+W!~C=I}#0P zjE#zaN0EigBCP>)saw?m6<5Tz2Fy&oEHP%en=xi(SB#;{9A1rUI@W-ETr*ts#Jjno zW*45A0~rt8qq%)$5qCseHEEm4$ZHc(5N|aiRcvoWHe|8C5%ah#8xD)VHewfSVr>)V zr`u#~*jP^+7q|h(CR&={uuYR0<0BHAA>L@h7f2S*Jqo|LyBRrRX)_WrTezB$otz=Z z!{wM1t_*?Qm?vIoMzPIdz+}iQ>t&+JvzHjtY?;x=BrCgWVR`0grM!Zz6mwdz2>JT@ z7IGUdQ-S!R71rc}N$3Rz#5D^m6yLXE!PFvCzEHNaQbQm`s5%y`8bREw%`z6RDGzB{ znHDj(aP>WyIp&>EEuL$`9b!rwmO~S(+px)^$<`SD zl`dXygCogn*yWa)-!y0hL!9l5TXd}rcUhuiyd~|}jwbz$cC-VJiM*#!e@lyu>iAy7 z-WyH6)#-OPRcRH0V8mO~)Gj`G3Qt-(WQgItND{uiNJfij-HXhec3Db|%NefhlVP8V=oK(HHqybTfodtvRbXIL;QU&s*^iS6YYawUB)QEd8eHnGTqKRGUj&ogxicA zG1&)UXWca7H0=?u?ZcxuAu^vvE_GXlGbO<5ipzE1$ti>WoxbaC!{- z%p~a3CiKBncE&)aS)a*!=h-pdbCY<_n!KrG^*Mu?rsw6@7xR+!70+UW)zU9pMdibk zsE2=rM=&6!bfIw3z;zKC{Qn3I5)?&fa9ntQ7fPlLnVK$WjG zG3pIQ^luK}goTgFX5IPyuPG{*#j@woNk^~md8$-tV%K4~h&Y~{)5SjyW7eE2V;)|W zvBFHqB6t0&`xmQQ=qO(?&@%KDN05S?tK%6<3|8rWIHfJPB2(60BP-2@Lt*sMfpXG< ztWY&fHjY7Dlg*}#RW#u`j@!6RF^ZwM_5x1iOjE{A!z65*LfA71&TM$H!ss)wX(7#! zO%uO6N>${$-%v4|EoL1>E}w1abf#pCY1*m$SUWQ&M3%uJzC20+a*Kz%;fGf&Jcc5X zatyb{$Bo=^z3nrn*~!_gG1Seb{bFqoX6fm_MJupG9C;DBQh|b#VtY5{i2uBZ622s^ zaeEJzOW6upYA)$L$mH4AVGJ0Y|{ynY;0a{bB;CDT7HPBql1Glu9ny0X|# z!-5QjMUAMX$AW&VKJ^6tWJQzss25v#lNn8Y-J9@9RO;|HW{MB~fYrQZLg$ji_8Zl; ziwYg7eBT7mg4l=6skd~PZZaq96c=?YM3+u@8x$N8#qVP6k|PS#d$V0NHD0QJxV?(r zMHRK#H>Kuui+A5e%ARgRBxhVit=4=ukr^lFT+eOwdU_P(8V`hVt#odWkc}26MaIgy zjbWuZ?4cp2r4vd*ZoS**(;^l0d25ryAfX`S zX6&32`%AM+V@eEq;E0-YDnA-~g6MBQo|2_rr8OsTbjpq6*%LId{(#l@~D| zU84OWRm~IP&5I;>Qvc^g{5h9Uk5A=~Q+dox;16N8-kQMoa%AaKr}Gp#+4@{N{}9L% zmuK*jJM!feW_LwHZkVlgu`3#>3x>QAuNDS#iONL26ovY8iF_A59P}wkJO@--ewxe= zpj7WqHlViYM`!XW+~SvaidWKjA$E#y)A-L2)t%{loF=ogFF>bwaxQ;> zy3gnGDodx_C0z6PQA)S3=J9XQBR-qY@3Wqidu549--yQ@e2FM`@cHP|n;iV8g(8!l z#qXnADI&5zqdM7|#hbs@}YY|6svY z(Y%fiqD9|)FMku*r&|m8T|kFey`FEg4@=m_lhhJ3Aq%TJ_1D((mn^hdWyL%bhxF!R z{tT2uio{a$#S_JRk-ofyuZ5M~@hzOuZnl>4uM$ow4kis)MlGXKuc+U|lWR^X2k%~6 zr+Idi)oY$DS}05}%+rc}g-I)K9HX33>`s$%Fa1A+z2QD3H_lNM3`N{NZ=~t0l6$Ag zK`$DQc3wHS*qr@B^VZfy`o-6q_^kN@%E6K{`j2tfYH|7|&Es=>{e#LuFK45c@*#!& E5AK)Ej{pDw delta 3179 zcmZuzZ*WxA6~Di`@4ap?8CD=6e+Voj%RfPPGYJ(WmVnVj8VDqy&<46m9$_V$4ZE8_ zFcbqsNiiMinw&6-5*0K$bc~ulm;p4gIOPM@9SIo6jKwJ*_>k7j$OmRbd*0hkiIAB) z_nv$1xxe2z=iYbT{rT;LcTXm0em(4+Jlv~2Sr_RJrA5vM>>tX&tV%hoL|%_>iL^v# zAnC)`qKF`B9IwWD*aj8Q`C800rf$PwV90pA2EUdFN49KEwddq5dGt{s&XC6kPvoDQ zrx>fBfgj2l<(CFga@Jz698@BsO|wimKOL(@M&zqz2fbT^$)x*Hj$zul3)79LAI&PR z8EZmll(=SLCtp{TG9eU^pdxjnUbVcOX|G%MG^Y_yPQIZCJDl~wKz$(Ot-VRlHew}i z(D_EB@G#hj1sI{98!@MJL_tv~n3I?1S-NalnXjhK8*T{d0%BTWRdvX#h*1SAbWRr)0&3);))eY(!Y-(^W)G&*iakh*$*DZl?13vGK@~z?S40wZ z?7_FVpBkQpUnZ-O(Y0=x){H4K&61Nd)mZ9gk4s%9`J1sbVYZ2psS1}l%`ztK#jujO zz*H_)1vjDC#VwogSW=$)u*f52KOVqR!@VC>koguv=51Syp-^z*0$nkeHC>amwzJj5}#(CI@sfHqp*#yqlzgP-pxEtp(MJ7ks7`6O=8hqVBc_z-p6?JHR$%jv#sfPx zF0iAP`4Al97)z3;-Sn3~VGmAF(qZKCu<|fmi9HsULv$@@-bKF|!@Nwc`>+JNd5e z@4z#N(ntr27es%NpTT?cGsr-kpTXGH#!i&pH)OF~=U|6TZG=mJPFN(bS&o@iMqXFx z``;nc_m(7(Nf{lEe`a00nXfreAG?<8i@&?x#TqflP)86dlCQWc-dj~I6jHArx!F|cz2`Ihv zNDpRk#yJnm(;Tro;?wRbhdw&hgVafWlOt0TJFSFA>ih}DXb(QMqk;N9!0%;)sZTc+ ze~2B5Y|^aX$NL6;!)1!$S3S^jH*U*VNL+W#%9YE}wDCVqv|9c|wl=3uYO?|oZqZI> zPh+7xV&NJ;odzFL2jzZ@Rr`)=a6aKF*tX5bMGrG7`FXLm#xtQd=a^}mGiK|o@%sI2 ze7Cwei5Jz6i$L;P3Y+dv0@cG!6#rm}-7HF8I-x}_wWiWz zpWp}d(9(WH(ogY{lHIHIX=k)^TEBKtyQD?6K~40MX8`G)18}3y6bs2cfDAe^fO*tt zN?#6O4xKh7$ETP(>4JvD!k|}(3&x61(d)oP3xE_qe(_{`%OwJV9 zi4wy|G52gTo==s(k~W{}rPu>vM!)6) z<69{A+lJI(<39^!uF84Hax>!%W2;-vm@*~tN9URbf2AID2DUl%`asooYF{ZILIRyz zDd#Nng`Cwv@2)y$LrAZ7h67H$F2v73r!VZ}rMf`)X?{%V)xN5*x3X4uRyH~Pe8qzg z@@v7qO1f-K+H*AfD>#kPRq|&WZc^zJatM2kg+=lMpw$>ImP^2og=wp0g=1Jjg`DOr z_14zv;o|Ccqh+-`Y2!nhzgEsehf%s#^3%G*_@-2bU{6xfA_bq_Q|sk-`+78&PzG%y zwj0VRExTydcD-r`KQT7z!9bZ_SFHzowXPMGwya*S+NcLZ{21xe91mN%^4)RG8O`Cg zG*9StddL?#r{%^hN&>;Kx7HVK>eq7TkD*uT7qzZM=FGn1Dcx7IJ$#8iDU%u5QLU@A dEF1`WYxJ0Yy() @@ -351,6 +348,10 @@ async fn precommit( .map(|result| result.map_err(|err| subxt::Error::from(err))) .collect::, _>>()?; + sector.precommit_block = Some(precommited_sectors[0].block); + sector.state = SectorState::Precommitted; + state.db.save_sector(§or)?; + tracing::info!( "Successfully pre-commited sectors on-chain: {:?}", precommited_sectors @@ -378,8 +379,12 @@ async fn prove_commit( return Err(PipelineError::NotExistentSector); }; - let precommit_block = sector.precommit_block.unwrap(); - let Some(digest) = state.xt_client.get_randomness(precommit_block).await? else { + let seal_randomness_height = sector.seal_randomness_height.unwrap(); + let Some(digest) = state + .xt_client + .get_randomness(seal_randomness_height) + .await? + else { tracing::error!("Out-of-the-state transition, this SHOULD not happen"); return Err(PipelineError::NotAvailableRandomness); }; @@ -389,7 +394,7 @@ async fn prove_commit( let ticket = draw_randomness( &digest, DomainSeparationTag::SealRandomness, - precommit_block, + seal_randomness_height, &entropy, ); @@ -397,10 +402,13 @@ async fn prove_commit( // https://github.com/eigerco/polka-storage/blob/5edd4194f08f29d769c277577ccbb70bb6ff63bc/runtime/src/configs/mod.rs#L360 // 10 blocks = 1 minute, only testnet const PRECOMMIT_CHALLENGE_DELAY: u64 = 10; - let prove_commit_block = precommit_block + PRECOMMIT_CHALLENGE_DELAY; + let prove_commit_block = sector.precommit_block.unwrap() + PRECOMMIT_CHALLENGE_DELAY; tracing::info!("Wait for block {} to get randomness", prove_commit_block); - state.xt_client.wait_for_height(prove_commit_block, true).await?; + state + .xt_client + .wait_for_height(prove_commit_block, true) + .await?; let Some(digest) = state.xt_client.get_randomness(prove_commit_block).await? else { tracing::error!("Randomness for the block not available."); return Err(PipelineError::NotAvailableRandomness); @@ -412,6 +420,9 @@ async fn prove_commit( &entropy, ); + tracing::info!("Performing prove commit for, seal_randomness_height {}, pre_commit_block: {}, prove_commit_block: {}, entropy: {}, ticket: {}, seed: {}", + seal_randomness_height, sector.precommit_block.unwrap(), prove_commit_block, hex::encode(entropy), hex::encode(ticket), hex::encode(seed)); + let sealing_handle: JoinHandle, _>> = { let porep_params = state.porep_parameters.clone(); let prover_id = derive_prover_id(state.xt_keypair.account_id()); diff --git a/cli/polka-storage-provider/server/src/pipeline/types.rs b/cli/polka-storage-provider/server/src/pipeline/types.rs index 1a3bdb395..31271857d 100644 --- a/cli/polka-storage-provider/server/src/pipeline/types.rs +++ b/cli/polka-storage-provider/server/src/pipeline/types.rs @@ -75,6 +75,7 @@ pub struct Sector { pub sealed_path: std::path::PathBuf, pub comm_r: Option, pub comm_d: Option, + pub seal_randomness_height: Option, pub precommit_block: Option, } @@ -101,6 +102,7 @@ impl Sector { sealed_path, comm_r: None, comm_d: None, + seal_randomness_height: None, precommit_block: None, }) } diff --git a/cli/polka-storage/storagext/src/runtime/display/storage_provider.rs b/cli/polka-storage/storagext/src/runtime/display/storage_provider.rs index 47eebd15a..3b96f2f2c 100644 --- a/cli/polka-storage/storagext/src/runtime/display/storage_provider.rs +++ b/cli/polka-storage/storagext/src/runtime/display/storage_provider.rs @@ -75,9 +75,13 @@ impl std::fmt::Display for Event { "Storage Provider Registered: {{ owner: {}, info: {}, proving_period_start: {} }}", owner, info, proving_period_start, )), - Event::SectorsPreCommitted { owner, sectors } => f.write_fmt(format_args!( - "Sectors Pre-Committed: {{ owner: {}, sector_number: {:?} }}", - owner, sectors, + Event::SectorsPreCommitted { + block, + owner, + sectors, + } => f.write_fmt(format_args!( + "Sectors Pre-Committed: {{ block: {}, owner: {}, sector_number: {:?} }}", + block, owner, sectors, )), Event::SectorsProven { owner, sectors } => f.write_fmt(format_args!( "Sectors Proven: {{ owner: {}, sectors: {:?} }}", diff --git a/pallets/storage-provider/Cargo.toml b/pallets/storage-provider/Cargo.toml index e1d5b109c..4307b8181 100644 --- a/pallets/storage-provider/Cargo.toml +++ b/pallets/storage-provider/Cargo.toml @@ -27,6 +27,8 @@ scale-info = { workspace = true, default-features = false, features = ["derive"] sp-arithmetic = { workspace = true, default-features = false } sp-core = { workspace = true, default-features = false } sp-runtime = { workspace = true, default-features = false } +hex = { workspace = true, default-features = false, features = ["alloc"] } + # Frame deps frame-benchmarking = { workspace = true, default-features = false, optional = true } diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index ac739a374..f817ce101 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -260,6 +260,9 @@ pub mod pallet { }, /// Emitted when a storage provider pre commits some sectors. SectorsPreCommitted { + /// Block at which sectors have been precommitted. + /// It is used for `interactive_randomness` generation in `ProveCommit`. + block: BlockNumberFor, owner: T::AccountId, sectors: BoundedVec>, ConstU32>, @@ -555,6 +558,7 @@ pub mod pallet { })?; Self::deposit_event(Event::SectorsPreCommitted { + block: current_block, owner, sectors: pre_committed_sectors, }); @@ -1510,6 +1514,7 @@ pub mod pallet { let current_block_number = >::block_number(); + // https://github.com/filecoin-project/builtin-actors/blob/a45fb87910bca74d62215b0d58ed90cf78b6c8ff/actors/miner/src/lib.rs#L4865 // Check if we are too early with the proof submit let interactive_block_number = precommit.pre_commit_block_number + T::PreCommitChallengeDelay::get(); @@ -1549,6 +1554,9 @@ pub mod pallet { let prover_id = derive_prover_id(owner); + log::info!("Performing prove commit for, seal_randomness_height {:?}, pre_commit_block: {:?}, prove_commit_block: {:?}, entropy: {}, ticket: {}, seed: {}", + precommit.info.seal_randomness_height, precommit.pre_commit_block_number, interactive_block_number, hex::encode(entropy), hex::encode(randomness), hex::encode(interactive_randomness)); + // Verify the porep proof T::ProofVerification::verify_porep( prover_id, diff --git a/pallets/storage-provider/src/tests/pre_commit_sectors.rs b/pallets/storage-provider/src/tests/pre_commit_sectors.rs index 70fad39bf..44ff6a83b 100644 --- a/pallets/storage-provider/src/tests/pre_commit_sectors.rs +++ b/pallets/storage-provider/src/tests/pre_commit_sectors.rs @@ -50,6 +50,7 @@ fn successfully_precommited() { amount: 1 },), RuntimeEvent::StorageProvider(Event::::SectorsPreCommitted { + block: 1, owner: account(storage_provider), sectors: bounded_vec![sector], }) @@ -97,6 +98,7 @@ fn successfully_precommited_no_deals() { amount: 1 },), RuntimeEvent::StorageProvider(Event::::SectorsPreCommitted { + block: 1, owner: account(storage_provider), sectors: bounded_vec![sector], }) @@ -160,6 +162,7 @@ fn successfully_precommited_batch() { amount: SECTORS_TO_PRECOMMIT },), RuntimeEvent::StorageProvider(Event::::SectorsPreCommitted { + block: 1, owner: account(storage_provider), sectors, }) From 896734e14dd047af48805f00eea535fc7a0b3ba1 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Tue, 5 Nov 2024 13:46:51 +0100 Subject: [PATCH 10/19] fix: debug logging --- cli/polka-storage-provider/server/src/pipeline/mod.rs | 5 +++-- pallets/proofs/src/lib.rs | 1 + pallets/storage-provider/src/lib.rs | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index 5c11d433d..8b398c84c 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -420,12 +420,13 @@ async fn prove_commit( &entropy, ); - tracing::info!("Performing prove commit for, seal_randomness_height {}, pre_commit_block: {}, prove_commit_block: {}, entropy: {}, ticket: {}, seed: {}", + let prover_id = derive_prover_id(state.xt_keypair.account_id()); + tracing::debug!("Performing prove commit for, seal_randomness_height {}, pre_commit_block: {}, prove_commit_block: {}, entropy: {}, ticket: {}, seed: {}", seal_randomness_height, sector.precommit_block.unwrap(), prove_commit_block, hex::encode(entropy), hex::encode(ticket), hex::encode(seed)); + tracing::debug!("Prover Id: {}, Sector Number: {}", hex::encode(prover_id), sector_number); let sealing_handle: JoinHandle, _>> = { let porep_params = state.porep_parameters.clone(); - let prover_id = derive_prover_id(state.xt_keypair.account_id()); let cache_dir = state.sealing_cache_dir.clone(); let sealed_path = sector.sealed_path.clone(); let piece_infos = sector.piece_infos.clone(); diff --git a/pallets/proofs/src/lib.rs b/pallets/proofs/src/lib.rs index 4b17b0412..d7b67de73 100644 --- a/pallets/proofs/src/lib.rs +++ b/pallets/proofs/src/lib.rs @@ -147,6 +147,7 @@ pub mod pallet { let proof_scheme = porep::ProofScheme::setup(seal_proof); let vkey = PoRepVerifyingKey::::get().ok_or(Error::::MissingPoRepVerifyingKey)?; + log::info!(target: LOG_TARGET, "Verifying PoRep proof for sector: {}...", sector); proof_scheme .verify( &comm_r, &comm_d, &prover_id, sector, &ticket, &seed, vkey, &proof, diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index f817ce101..be46dc09b 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -1554,8 +1554,9 @@ pub mod pallet { let prover_id = derive_prover_id(owner); - log::info!("Performing prove commit for, seal_randomness_height {:?}, pre_commit_block: {:?}, prove_commit_block: {:?}, entropy: {}, ticket: {}, seed: {}", + log::debug!(target: LOG_TARGET, "Performing prove commit for, seal_randomness_height {:?}, pre_commit_block: {:?}, prove_commit_block: {:?}, entropy: {}, ticket: {}, seed: {}", precommit.info.seal_randomness_height, precommit.pre_commit_block_number, interactive_block_number, hex::encode(entropy), hex::encode(randomness), hex::encode(interactive_randomness)); + log::debug!(target: LOG_TARGET, "Prover Id: {}, Sector Number: {}", hex::encode(prover_id), precommit.info.sector_number); // Verify the porep proof T::ProofVerification::verify_porep( From 048617841fbe279d8b01668d37c67f5c2ed85fd5 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Tue, 5 Nov 2024 14:31:23 +0100 Subject: [PATCH 11/19] docs: remove a bad comment --- pallets/storage-provider/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/pallets/storage-provider/src/lib.rs b/pallets/storage-provider/src/lib.rs index be46dc09b..538054d7d 100644 --- a/pallets/storage-provider/src/lib.rs +++ b/pallets/storage-provider/src/lib.rs @@ -1519,8 +1519,6 @@ pub mod pallet { let interactive_block_number = precommit.pre_commit_block_number + T::PreCommitChallengeDelay::get(); - // check out this interactive block number cause i'm confused - // would need to fetch it after pre-commit if current_block_number < interactive_block_number { log::error!(target: LOG_TARGET, "too early to prove sector: current_block_number: {current_block_number:?} < interactive_block_number: {interactive_block_number:?}"); return Err(Error::::InvalidProof)?; From 60425933c806351ff64b7ae5d9ea4b965d1df159 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 6 Nov 2024 00:32:39 +0100 Subject: [PATCH 12/19] fix: pr feedback --- cli/polka-storage-provider/server/src/main.rs | 1 - .../server/src/pipeline/mod.rs | 47 ++++++++++++------- .../server/src/pipeline/types.rs | 22 +++++++-- lib/polka-storage-proofs/src/porep/sealer.rs | 8 +++- 4 files changed, 57 insertions(+), 21 deletions(-) diff --git a/cli/polka-storage-provider/server/src/main.rs b/cli/polka-storage-provider/server/src/main.rs index b0dd80bc6..151057aea 100644 --- a/cli/polka-storage-provider/server/src/main.rs +++ b/cli/polka-storage-provider/server/src/main.rs @@ -242,7 +242,6 @@ pub struct ServerConfiguration { /// Proving Parameters for PoRep proof /// For 2KiB sectors they're ~1GiB of data. - /// We may load them on demand someday. porep_parameters: PoRepParameters, } diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index 8b398c84c..0aab241ae 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -3,7 +3,7 @@ pub mod types; use std::{path::PathBuf, sync::Arc}; use polka_storage_proofs::porep::{ - sealer::{prepare_piece, PreCommitOutput, Proof, Sealer, SubstrateProof}, + sealer::{prepare_piece, BlstrsProof, PreCommitOutput, Sealer, SubstrateProof}, PoRepError, PoRepParameters, }; use polka_storage_provider_common::rpc::ServerInfo; @@ -49,9 +49,9 @@ pub enum PipelineError { #[error(transparent)] DBError(#[from] DBError), #[error("sector does not exist")] - NotExistentSector, + SectorNotFound, #[error("precommit scheduled too early, randomness not available")] - NotAvailableRandomness, + RandomnessNotAvailable, #[error(transparent)] SendError(#[from] SendError), } @@ -252,7 +252,7 @@ async fn precommit( let sealer = Sealer::new(state.server_info.seal_proof); let Some(mut sector) = state.db.get_sector(sector_number)? else { tracing::error!("Tried to precommit non-existing sector"); - return Err(PipelineError::NotExistentSector); + return Err(PipelineError::SectorNotFound); }; // Pad sector so CommD can be properly calculated. sector.piece_infos = sealer.pad_sector(§or.piece_infos, sector.occupied_sector_space)?; @@ -265,7 +265,7 @@ async fn precommit( let Some(digest) = state.xt_client.get_randomness(current_block).await? else { tracing::error!("Precommit was scheduled too early, when the randomness on-chain was not yet available..."); - return Err(PipelineError::NotAvailableRandomness); + return Err(PipelineError::RandomnessNotAvailable); }; let entropy = state.xt_keypair.account_id().encode(); // Must match pallet's logic or otherwise proof won't be verified: @@ -376,17 +376,19 @@ async fn prove_commit( let sealer = Sealer::new(state.server_info.seal_proof); let Some(mut sector) = state.db.get_sector(sector_number)? else { tracing::error!("Tried to precommit non-existing sector"); - return Err(PipelineError::NotExistentSector); + return Err(PipelineError::SectorNotFound); }; - let seal_randomness_height = sector.seal_randomness_height.unwrap(); + let seal_randomness_height = sector + .seal_randomness_height + .expect("sector to be sealed before proving using randomness from the chain"); let Some(digest) = state .xt_client .get_randomness(seal_randomness_height) .await? else { tracing::error!("Out-of-the-state transition, this SHOULD not happen"); - return Err(PipelineError::NotAvailableRandomness); + return Err(PipelineError::RandomnessNotAvailable); }; let entropy = state.xt_keypair.account_id().encode(); // Must match pallet's logic or otherwise proof won't be verified: @@ -402,7 +404,10 @@ async fn prove_commit( // https://github.com/eigerco/polka-storage/blob/5edd4194f08f29d769c277577ccbb70bb6ff63bc/runtime/src/configs/mod.rs#L360 // 10 blocks = 1 minute, only testnet const PRECOMMIT_CHALLENGE_DELAY: u64 = 10; - let prove_commit_block = sector.precommit_block.unwrap() + PRECOMMIT_CHALLENGE_DELAY; + let precommit_block = sector + .precommit_block + .expect("sector to be pre-committed on-chain before proving"); + let prove_commit_block = precommit_block + PRECOMMIT_CHALLENGE_DELAY; tracing::info!("Wait for block {} to get randomness", prove_commit_block); state @@ -411,7 +416,7 @@ async fn prove_commit( .await?; let Some(digest) = state.xt_client.get_randomness(prove_commit_block).await? else { tracing::error!("Randomness for the block not available."); - return Err(PipelineError::NotAvailableRandomness); + return Err(PipelineError::RandomnessNotAvailable); }; let seed = draw_randomness( &digest, @@ -422,16 +427,26 @@ async fn prove_commit( let prover_id = derive_prover_id(state.xt_keypair.account_id()); tracing::debug!("Performing prove commit for, seal_randomness_height {}, pre_commit_block: {}, prove_commit_block: {}, entropy: {}, ticket: {}, seed: {}", - seal_randomness_height, sector.precommit_block.unwrap(), prove_commit_block, hex::encode(entropy), hex::encode(ticket), hex::encode(seed)); - tracing::debug!("Prover Id: {}, Sector Number: {}", hex::encode(prover_id), sector_number); + seal_randomness_height, precommit_block, prove_commit_block, hex::encode(entropy), hex::encode(ticket), hex::encode(seed)); + tracing::debug!( + "Prover Id: {}, Sector Number: {}", + hex::encode(prover_id), + sector_number + ); - let sealing_handle: JoinHandle, _>> = { + let sealing_handle: JoinHandle, _>> = { let porep_params = state.porep_parameters.clone(); let cache_dir = state.sealing_cache_dir.clone(); let sealed_path = sector.sealed_path.clone(); let piece_infos = sector.piece_infos.clone(); - let comm_r = sector.comm_r.unwrap().raw(); - let comm_d = sector.comm_d.unwrap().raw(); + let comm_r = sector + .comm_r + .expect("sector to be sealed and it's comm_r set before proving") + .raw(); + let comm_d = sector + .comm_d + .expect("sector to be sealed and it's comm_d set before proving") + .raw(); tokio::task::spawn_blocking(move || { sealer.prove_sector( porep_params.as_ref(), @@ -471,7 +486,7 @@ async fn prove_commit( true, ) .await? - .unwrap(); + .expect("waiting for finalization should always give results"); let proven_sectors = result .events diff --git a/cli/polka-storage-provider/server/src/pipeline/types.rs b/cli/polka-storage-provider/server/src/pipeline/types.rs index 31271857d..8628a5fde 100644 --- a/cli/polka-storage-provider/server/src/pipeline/types.rs +++ b/cli/polka-storage-provider/server/src/pipeline/types.rs @@ -8,10 +8,11 @@ use storagext::types::market::DealProposal; /// Represents a task to be executed on the Storage Provider Pipeline #[derive(Debug)] pub enum PipelineMessage { - /// Adds a deal to a sector selected by the storage provider + /// Adds a deal to a sector selected by the storage provider. AddPiece(AddPieceMessage), - /// Pads, seals the sector and pre-commits it on chain + /// Pads, seals a sector and pre-commits it on-chain. PreCommit(PreCommitMessage), + /// Generates a PoRep for a sector and verifies the proof on-chain. ProveCommit(ProveCommitMessage), } @@ -73,9 +74,23 @@ pub struct Sector { /// Only after pipeline [`PipelineMessage::PreCommit`], /// the file has contents which should not be touched and are used for later steps. pub sealed_path: std::path::PathBuf, + /// CID of the sealed sector. + /// + /// Available at [`SectorState::Sealed`]/[`PipelineMessage::PreCommit`] and later. pub comm_r: Option, + /// CID of the unsealed data of the sector. + /// + /// Available at [`SectorState::Sealed`]/[`PipelineMessage::PreCommit`] and later. pub comm_d: Option, + /// Block at which randomness has been fetched to perform [`PipelineMessage::PreCommit`]. + /// + /// It is used as a randomness seed to create a replica. + /// Available at [`SectorState::Sealed`] and later. pub seal_randomness_height: Option, + /// Block at which the sector was precommitted (extrinsic submitted on-chain). + /// + /// It is used as a randomness seed to create a PoRep. + /// Available at [`SectorState::Precommitted`] and later. pub precommit_block: Option, } @@ -117,7 +132,8 @@ pub enum SectorState { Sealed, /// Sealed sector has been published on-chain, so now the PoRep must be generated for it. Precommitted, - /// After a PoRep for a sector has been created and publish on-chain. + /// After a PoRep for a sector has been generated. Proven, + /// Generated PoRep for a sector has been published and verified on-chain. ProveCommitted, } diff --git a/lib/polka-storage-proofs/src/porep/sealer.rs b/lib/polka-storage-proofs/src/porep/sealer.rs index 788becac5..af9d6e478 100644 --- a/lib/polka-storage-proofs/src/porep/sealer.rs +++ b/lib/polka-storage-proofs/src/porep/sealer.rs @@ -22,7 +22,13 @@ use crate::{ ZeroPaddingReader, }; -pub type Proof = groth16::Proof; +/// Proof using [`blstrs::Bls12`] as finite field elements. +/// +/// It's the output of PoRep and PoSt proof generation and CANNOT be used in no_std. +pub type BlstrsProof = groth16::Proof; +/// Proof using [`bls12_381::Bls12`] as finite field elements. +/// +/// It is used in no_std to verify proofs, converted from [`BlstrsProof`]. pub type SubstrateProof = crate::Proof; /// Prepares an arbitrary piece to be used by [`Sealer::create_sector`]. From 32935073524568f12dad490340ef08c820633c5c Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 6 Nov 2024 00:38:31 +0100 Subject: [PATCH 13/19] refactor: add pipeline operations trait --- .../server/src/pipeline/mod.rs | 127 ++++++++++-------- 1 file changed, 72 insertions(+), 55 deletions(-) diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index 0aab241ae..047b37ed6 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -104,69 +104,86 @@ pub async fn start_pipeline( Ok(()) } -fn process( - tracker: &TaskTracker, - msg: PipelineMessage, - state: Arc, - token: CancellationToken, -) { - match msg { - PipelineMessage::AddPiece(AddPieceMessage { +trait PipelineOperations { + fn add_piece(msg: AddPieceMessage); + fn precommit(msg: PreCommitMessage); + fn prove_commit(msg: ProveCommitMessage); +} + +impl PipelineOperations for TaskTracker { + fn add_piece(&self, msg: AddPieceMessage) { + let AddPieceMessage { deal, published_deal_id, piece_path, piece_cid, - }) => { - tracker.spawn(async move { - tokio::select! { - // AddPiece is cancellation safe, as it can be retried and the state will be fine. - res = add_piece(state, piece_path, piece_cid, deal, published_deal_id) => { - match res { - Ok(_) => tracing::info!("Add Piece for piece {:?}, deal id {}, finished successfully.", piece_cid, published_deal_id), - Err(err) => tracing::error!(%err, "Add Piece for piece {:?}, deal id {}, failed!", piece_cid, published_deal_id), - } - }, - () = token.cancelled() => { - tracing::warn!("AddPiece has been cancelled."); + } = msg; + self.spawn(async move { + tokio::select! { + // AddPiece is cancellation safe, as it can be retried and the state will be fine. + res = add_piece(state, piece_path, piece_cid, deal, published_deal_id) => { + match res { + Ok(_) => tracing::info!("Add Piece for piece {:?}, deal id {}, finished successfully.", piece_cid, published_deal_id), + Err(err) => tracing::error!(%err, "Add Piece for piece {:?}, deal id {}, failed!", piece_cid, published_deal_id), } + }, + () = token.cancelled() => { + tracing::warn!("AddPiece has been cancelled."); } - }); - } - PipelineMessage::PreCommit(PreCommitMessage { sector_number }) => { - tracker.spawn(async move { - // Precommit is not cancellation safe. - // TODO(@th7nder,#501, 04/11/2024): when it's cancelled, it can hang and user will have to wait for it to finish. - // If they don't the state can be corrupted, we could improve that situation. - // One of the ideas is to store state as 'Precommitting' so then we know we can retry that after some time. - match precommit(state, sector_number).await { - Ok(_) => { - tracing::info!( - "Precommit for sector {} finished successfully.", - sector_number - ) - } - Err(err) => { - tracing::error!(%err, "Failed PreCommit for Sector: {}", sector_number) - } + } + }); + } + + fn precommit(msg: PreCommitMessage) { + let PreCommitMessage { sector_number } = msg; + self.spawn(async move { + // Precommit is not cancellation safe. + // TODO(@th7nder,#501, 04/11/2024): when it's cancelled, it can hang and user will have to wait for it to finish. + // If they don't the state can be corrupted, we could improve that situation. + // One of the ideas is to store state as 'Precommitting' so then we know we can retry that after some time. + match precommit(state, sector_number).await { + Ok(_) => { + tracing::info!( + "Precommit for sector {} finished successfully.", + sector_number + ) } - }); - } - PipelineMessage::ProveCommit(ProveCommitMessage { sector_number }) => { - tracker.spawn(async move { - // ProveCommit is not cancellation safe. - match prove_commit(state, sector_number).await { - Ok(_) => { - tracing::info!( - "ProveCommit for sector {} finished successfully.", - sector_number - ) - } - Err(err) => { - tracing::error!(%err, "Failed ProveCommit for Sector: {}", sector_number) - } + Err(err) => { + tracing::error!(%err, "Failed PreCommit for Sector: {}", sector_number) } - }); - } + } + }); + } + + fn prove_commit(&self, msg: ProveCommitMessage) { + let ProveCommitMessage { sector_number } = msg; + self.spawn(async move { + // ProveCommit is not cancellation safe. + match prove_commit(state, sector_number).await { + Ok(_) => { + tracing::info!( + "ProveCommit for sector {} finished successfully.", + sector_number + ) + } + Err(err) => { + tracing::error!(%err, "Failed ProveCommit for Sector: {}", sector_number) + } + } + }); + } +} + +fn process( + tracker: &TaskTracker, + msg: PipelineMessage, + state: Arc, + token: CancellationToken, +) { + match msg { + PipelineMessage::AddPiece(msg) => tracker.add_piece(msg), + PipelineMessage::PreCommit(msg) => tracker.precommit(msg), + PipelineMessage::ProveCommit(msg) => tracker.prove_commit(msg), } } From 95a61146dea334e3ce5cf4dc881fa3906dc3cd83 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 6 Nov 2024 12:10:22 +0100 Subject: [PATCH 14/19] docs: polka-storage-proofs --- .../server/src/pipeline/mod.rs | 18 +++++++++--------- lib/polka-storage-proofs/src/porep/sealer.rs | 3 ++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index 047b37ed6..cd80bfc5a 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -105,13 +105,13 @@ pub async fn start_pipeline( } trait PipelineOperations { - fn add_piece(msg: AddPieceMessage); - fn precommit(msg: PreCommitMessage); - fn prove_commit(msg: ProveCommitMessage); + fn add_piece(&self, state: Arc, msg: AddPieceMessage, token: CancellationToken); + fn precommit(&self, state: Arc, msg: PreCommitMessage); + fn prove_commit(&self, state: Arc, msg: ProveCommitMessage); } impl PipelineOperations for TaskTracker { - fn add_piece(&self, msg: AddPieceMessage) { + fn add_piece(&self, state: Arc, msg: AddPieceMessage, token: CancellationToken) { let AddPieceMessage { deal, published_deal_id, @@ -134,7 +134,7 @@ impl PipelineOperations for TaskTracker { }); } - fn precommit(msg: PreCommitMessage) { + fn precommit(&self, state: Arc, msg: PreCommitMessage) { let PreCommitMessage { sector_number } = msg; self.spawn(async move { // Precommit is not cancellation safe. @@ -155,7 +155,7 @@ impl PipelineOperations for TaskTracker { }); } - fn prove_commit(&self, msg: ProveCommitMessage) { + fn prove_commit(&self, state: Arc, msg: ProveCommitMessage) { let ProveCommitMessage { sector_number } = msg; self.spawn(async move { // ProveCommit is not cancellation safe. @@ -181,9 +181,9 @@ fn process( token: CancellationToken, ) { match msg { - PipelineMessage::AddPiece(msg) => tracker.add_piece(msg), - PipelineMessage::PreCommit(msg) => tracker.precommit(msg), - PipelineMessage::ProveCommit(msg) => tracker.prove_commit(msg), + PipelineMessage::AddPiece(msg) => tracker.add_piece(state.clone(), msg, token.clone()), + PipelineMessage::PreCommit(msg) => tracker.precommit(state.clone(), msg), + PipelineMessage::ProveCommit(msg) => tracker.prove_commit(state.clone(), msg), } } diff --git a/lib/polka-storage-proofs/src/porep/sealer.rs b/lib/polka-storage-proofs/src/porep/sealer.rs index af9d6e478..29a04ba0d 100644 --- a/lib/polka-storage-proofs/src/porep/sealer.rs +++ b/lib/polka-storage-proofs/src/porep/sealer.rs @@ -28,7 +28,8 @@ use crate::{ pub type BlstrsProof = groth16::Proof; /// Proof using [`bls12_381::Bls12`] as finite field elements. /// -/// It is used in no_std to verify proofs, converted from [`BlstrsProof`]. +/// It is reexported so we don't need to add a dependencies in `polka-storage-provider-*` crates. +/// Used to convert into `no_std` version of proof, to call extrinsic to verify proof. pub type SubstrateProof = crate::Proof; /// Prepares an arbitrary piece to be used by [`Sealer::create_sector`]. From 11eb6eeafe48d749734aa0deb7cf0983f7e8479f Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 6 Nov 2024 12:56:02 +0100 Subject: [PATCH 15/19] fix: make save sector generic --- cli/polka-storage-provider/server/src/db.rs | 15 ++++++--------- .../server/src/pipeline/mod.rs | 14 +++++++------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/cli/polka-storage-provider/server/src/db.rs b/cli/polka-storage-provider/server/src/db.rs index 18911ff35..238c25c40 100644 --- a/cli/polka-storage-provider/server/src/db.rs +++ b/cli/polka-storage-provider/server/src/db.rs @@ -5,10 +5,9 @@ use std::{ use primitives_proofs::SectorNumber; use rocksdb::{ColumnFamily, ColumnFamilyDescriptor, Options as DBOptions, DB as RocksDB}; +use serde::{de::DeserializeOwned, Serialize}; use storagext::types::market::{ConversionError, DealProposal}; -use crate::pipeline::types::Sector; - #[derive(Debug, thiserror::Error)] pub enum DBError { #[error(transparent)] @@ -118,25 +117,23 @@ impl DealDB { )?) } - pub fn get_sector(&self, sector_id: SectorNumber) -> Result, DBError> { + pub fn get_sector(&self, sector_number: SectorNumber) -> Result, DBError> { let Some(sector_slice) = self .database - .get_pinned_cf(self.cf_handle(SECTORS_CF), sector_id.to_le_bytes())? + .get_pinned_cf(self.cf_handle(SECTORS_CF), sector_number.to_le_bytes())? else { return Ok(None); }; let sector = serde_json::from_reader(sector_slice.as_ref()) - // SAFETY: this should never fail since the API sets a sector - // if this happens, it means that someone wrote it from a side channel - .expect("invalid content was placed in the database from outside this API"); + .expect("sector type to be equal to the one stored in the database"); Ok(Some(sector)) } - pub fn save_sector(&self, sector: &Sector) -> Result<(), DBError> { + pub fn save_sector(&self, sector_number: SectorNumber, sector: &SectorType) -> Result<(), DBError> { let cf_handle = self.cf_handle(SECTORS_CF); - let key = sector.sector_number.to_le_bytes(); + let key = sector_number.to_le_bytes(); let json = serde_json::to_vec(§or)?; self.database.put_cf(cf_handle, key, json)?; diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index cd80bfc5a..8411b03da 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -240,7 +240,7 @@ async fn add_piece( let sector: Sector = handle.await??; tracing::info!("Finished adding a piece"); - state.db.save_sector(§or)?; + state.db.save_sector(sector.sector_number, §or)?; // TODO(@th7nder,30/10/2024): simplification, as we're always scheduling a precommit just after adding a piece and creating a new sector. // Ideally sector won't be finalized after one piece has been added and the precommit will depend on the start_block? @@ -267,7 +267,7 @@ async fn precommit( tracing::info!("Starting pre-commit"); let sealer = Sealer::new(state.server_info.seal_proof); - let Some(mut sector) = state.db.get_sector(sector_number)? else { + let Some(mut sector) = state.db.get_sector::(sector_number)? else { tracing::error!("Tried to precommit non-existing sector"); return Err(PipelineError::SectorNotFound); }; @@ -320,7 +320,7 @@ async fn precommit( sector.comm_r = Some(Commitment::replica(sealing_output.comm_r)); sector.comm_d = Some(Commitment::data(sealing_output.comm_d)); sector.seal_randomness_height = Some(current_block); - state.db.save_sector(§or)?; + state.db.save_sector(sector.sector_number, §or)?; tracing::debug!("Precommiting at block: {}", current_block); @@ -367,7 +367,7 @@ async fn precommit( sector.precommit_block = Some(precommited_sectors[0].block); sector.state = SectorState::Precommitted; - state.db.save_sector(§or)?; + state.db.save_sector(sector.sector_number, §or)?; tracing::info!( "Successfully pre-commited sectors on-chain: {:?}", @@ -391,7 +391,7 @@ async fn prove_commit( tracing::info!("Starting prove commit"); let sealer = Sealer::new(state.server_info.seal_proof); - let Some(mut sector) = state.db.get_sector(sector_number)? else { + let Some(mut sector) = state.db.get_sector::(sector_number)? else { tracing::error!("Tried to precommit non-existing sector"); return Err(PipelineError::SectorNotFound); }; @@ -490,7 +490,7 @@ async fn prove_commit( tracing::info!("Proven sector: {}", sector_number); sector.state = SectorState::Proven; - state.db.save_sector(§or)?; + state.db.save_sector(sector.sector_number, §or)?; let result = state .xt_client @@ -514,7 +514,7 @@ async fn prove_commit( tracing::info!("Successfully proven sectors on-chain: {:?}", proven_sectors); sector.state = SectorState::ProveCommitted; - state.db.save_sector(§or)?; + state.db.save_sector(sector.sector_number, §or)?; Ok(()) } From c0288678f7854d7913fb1acc9caf9bc137769cc6 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 6 Nov 2024 15:13:51 +0100 Subject: [PATCH 16/19] refactor: sector types --- cli/polka-storage-provider/server/src/db.rs | 16 ++- .../server/src/pipeline/mod.rs | 94 ++++++------ .../server/src/pipeline/types.rs | 136 +++++++++++++----- 3 files changed, 162 insertions(+), 84 deletions(-) diff --git a/cli/polka-storage-provider/server/src/db.rs b/cli/polka-storage-provider/server/src/db.rs index 238c25c40..705a30b3a 100644 --- a/cli/polka-storage-provider/server/src/db.rs +++ b/cli/polka-storage-provider/server/src/db.rs @@ -24,6 +24,9 @@ pub enum DBError { #[error(transparent)] Json(#[from] serde_json::Error), + + #[error("unexpected data when trying to serialize given sector type: {0}")] + InvalidSectorData(serde_json::Error), } const ACCEPTED_DEAL_PROPOSALS_CF: &str = "accepted_deal_proposals"; @@ -117,7 +120,10 @@ impl DealDB { )?) } - pub fn get_sector(&self, sector_number: SectorNumber) -> Result, DBError> { + pub fn get_sector( + &self, + sector_number: SectorNumber, + ) -> Result, DBError> { let Some(sector_slice) = self .database .get_pinned_cf(self.cf_handle(SECTORS_CF), sector_number.to_le_bytes())? @@ -126,12 +132,16 @@ impl DealDB { }; let sector = serde_json::from_reader(sector_slice.as_ref()) - .expect("sector type to be equal to the one stored in the database"); + .map_err(|e| DBError::InvalidSectorData(e))?; Ok(Some(sector)) } - pub fn save_sector(&self, sector_number: SectorNumber, sector: &SectorType) -> Result<(), DBError> { + pub fn save_sector( + &self, + sector_number: SectorNumber, + sector: &SectorType, + ) -> Result<(), DBError> { let cf_handle = self.cf_handle(SECTORS_CF); let key = sector_number.to_le_bytes(); let json = serde_json::to_vec(§or)?; diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index 8411b03da..ea5e2b02e 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -26,15 +26,16 @@ use tokio::{ task::{JoinError, JoinHandle}, }; use tokio_util::{sync::CancellationToken, task::TaskTracker}; -use types::{AddPieceMessage, PipelineMessage, PreCommitMessage}; - -use self::types::Sector; -use crate::{ - db::{DBError, DealDB}, - pipeline::types::{ProveCommitMessage, SectorState}, +use types::{ + AddPieceMessage, PipelineMessage, PreCommitMessage, PreCommittedSector, ProveCommitMessage, + ProvenSector, UnsealedSector, }; +use crate::db::{DBError, DealDB}; + const SECTOR_EXPIRATION_MARGIN: u64 = 20; +/// TODO(@th7nder,[#457, #458], 06/11/2024): Blockchain cannot give randomness until block 82. +const MINIMUM_BLOCK_WITH_RANDOMNESS_AVAILABLE: u64 = 82; #[derive(Debug, thiserror::Error)] pub enum PipelineError { @@ -187,13 +188,14 @@ fn process( } } -async fn find_sector_for_piece(state: &Arc) -> Result { +async fn find_sector_for_piece( + state: &Arc, +) -> Result { // TODO(@th7nder,30/10/2024): simplification, we're always creating a new sector for storing a piece. // It should not work like that, sectors should be filled with pieces according to *some* algorithm. let sector_number = state.db.next_sector_number(); let unsealed_path = state.unsealed_sectors_dir.join(sector_number.to_string()); - let sealed_path = state.sealed_sectors_dir.join(sector_number.to_string()); - let sector = Sector::create_unsealed(sector_number, unsealed_path, sealed_path).await?; + let sector = UnsealedSector::create(sector_number, unsealed_path).await?; Ok(sector) } @@ -216,7 +218,7 @@ async fn add_piece( tracing::info!("Adding a piece..."); let sealer = Sealer::new(state.server_info.seal_proof); - let handle: JoinHandle> = + let handle: JoinHandle> = tokio::task::spawn_blocking(move || { let unsealed_sector = std::fs::File::options() .append(true) @@ -237,7 +239,7 @@ async fn add_piece( Ok(sector) }); - let sector: Sector = handle.await??; + let sector: UnsealedSector = handle.await??; tracing::info!("Finished adding a piece"); state.db.save_sector(sector.sector_number, §or)?; @@ -267,7 +269,7 @@ async fn precommit( tracing::info!("Starting pre-commit"); let sealer = Sealer::new(state.server_info.seal_proof); - let Some(mut sector) = state.db.get_sector::(sector_number)? else { + let Some(mut sector) = state.db.get_sector::(sector_number)? else { tracing::error!("Tried to precommit non-existing sector"); return Err(PipelineError::SectorNotFound); }; @@ -277,9 +279,21 @@ async fn precommit( tracing::info!("Padded sector, commencing pre-commit and getting last finalized block"); - let current_block = state.xt_client.height(true).await?; + let mut current_block = state.xt_client.height(true).await?; tracing::info!("Current block: {current_block}"); + if current_block < MINIMUM_BLOCK_WITH_RANDOMNESS_AVAILABLE { + tracing::info!( + "Waiting for randomness to be available at block: {}", + MINIMUM_BLOCK_WITH_RANDOMNESS_AVAILABLE + ); + state + .xt_client + .wait_for_height(MINIMUM_BLOCK_WITH_RANDOMNESS_AVAILABLE, true) + .await?; + current_block = MINIMUM_BLOCK_WITH_RANDOMNESS_AVAILABLE; + } + let Some(digest) = state.xt_client.get_randomness(current_block).await? else { tracing::error!("Precommit was scheduled too early, when the randomness on-chain was not yet available..."); return Err(PipelineError::RandomnessNotAvailable); @@ -294,12 +308,16 @@ async fn precommit( &entropy, ); + let sealed_path = state.sealed_sectors_dir.join(sector_number.to_string()); + tokio::fs::File::create_new(&sealed_path).await?; + // TODO(@th7nder,31/10/2024): what happens if some of the process fails? SP will be slashed, and there is no error reporting? what about retries? let sealing_handle: JoinHandle> = { let prover_id = derive_prover_id(state.xt_keypair.account_id()); let cache_dir = state.sealing_cache_dir.clone(); let unsealed_path = sector.unsealed_path.clone(); - let sealed_path = sector.sealed_path.clone(); + let sealed_path = sealed_path.clone(); + let piece_infos = sector.piece_infos.clone(); tokio::task::spawn_blocking(move || { sealer.precommit_sector( @@ -316,14 +334,7 @@ async fn precommit( let sealing_output = sealing_handle.await??; tracing::info!("Created sector's replica: {:?}", sealing_output); - sector.state = SectorState::Sealed; - sector.comm_r = Some(Commitment::replica(sealing_output.comm_r)); - sector.comm_d = Some(Commitment::data(sealing_output.comm_d)); - sector.seal_randomness_height = Some(current_block); - state.db.save_sector(sector.sector_number, §or)?; - tracing::debug!("Precommiting at block: {}", current_block); - let result = state .xt_client .pre_commit_sectors( @@ -365,8 +376,15 @@ async fn precommit( .map(|result| result.map_err(|err| subxt::Error::from(err))) .collect::, _>>()?; - sector.precommit_block = Some(precommited_sectors[0].block); - sector.state = SectorState::Precommitted; + let sector = PreCommittedSector::create( + sector, + sealed_path, + Commitment::replica(sealing_output.comm_r), + Commitment::data(sealing_output.comm_d), + current_block, + precommited_sectors[0].block, + ) + .await?; state.db.save_sector(sector.sector_number, §or)?; tracing::info!( @@ -391,20 +409,18 @@ async fn prove_commit( tracing::info!("Starting prove commit"); let sealer = Sealer::new(state.server_info.seal_proof); - let Some(mut sector) = state.db.get_sector::(sector_number)? else { + let Some(sector) = state.db.get_sector::(sector_number)? else { tracing::error!("Tried to precommit non-existing sector"); return Err(PipelineError::SectorNotFound); }; - let seal_randomness_height = sector - .seal_randomness_height - .expect("sector to be sealed before proving using randomness from the chain"); + let seal_randomness_height = sector.seal_randomness_height; let Some(digest) = state .xt_client .get_randomness(seal_randomness_height) .await? else { - tracing::error!("Out-of-the-state transition, this SHOULD not happen"); + tracing::error!("Out-of-the-state transition, this SHOULD NOT happen"); return Err(PipelineError::RandomnessNotAvailable); }; let entropy = state.xt_keypair.account_id().encode(); @@ -421,10 +437,7 @@ async fn prove_commit( // https://github.com/eigerco/polka-storage/blob/5edd4194f08f29d769c277577ccbb70bb6ff63bc/runtime/src/configs/mod.rs#L360 // 10 blocks = 1 minute, only testnet const PRECOMMIT_CHALLENGE_DELAY: u64 = 10; - let precommit_block = sector - .precommit_block - .expect("sector to be pre-committed on-chain before proving"); - let prove_commit_block = precommit_block + PRECOMMIT_CHALLENGE_DELAY; + let prove_commit_block = sector.precommit_block + PRECOMMIT_CHALLENGE_DELAY; tracing::info!("Wait for block {} to get randomness", prove_commit_block); state @@ -444,7 +457,7 @@ async fn prove_commit( let prover_id = derive_prover_id(state.xt_keypair.account_id()); tracing::debug!("Performing prove commit for, seal_randomness_height {}, pre_commit_block: {}, prove_commit_block: {}, entropy: {}, ticket: {}, seed: {}", - seal_randomness_height, precommit_block, prove_commit_block, hex::encode(entropy), hex::encode(ticket), hex::encode(seed)); + seal_randomness_height, sector.precommit_block, prove_commit_block, hex::encode(entropy), hex::encode(ticket), hex::encode(seed)); tracing::debug!( "Prover Id: {}, Sector Number: {}", hex::encode(prover_id), @@ -456,14 +469,8 @@ async fn prove_commit( let cache_dir = state.sealing_cache_dir.clone(); let sealed_path = sector.sealed_path.clone(); let piece_infos = sector.piece_infos.clone(); - let comm_r = sector - .comm_r - .expect("sector to be sealed and it's comm_r set before proving") - .raw(); - let comm_d = sector - .comm_d - .expect("sector to be sealed and it's comm_d set before proving") - .raw(); + let comm_r = sector.comm_r.raw(); + let comm_d = sector.comm_d.raw(); tokio::task::spawn_blocking(move || { sealer.prove_sector( porep_params.as_ref(), @@ -489,9 +496,6 @@ async fn prove_commit( let proof = codec::Encode::encode(&proof); tracing::info!("Proven sector: {}", sector_number); - sector.state = SectorState::Proven; - state.db.save_sector(sector.sector_number, §or)?; - let result = state .xt_client .prove_commit_sectors( @@ -513,7 +517,7 @@ async fn prove_commit( tracing::info!("Successfully proven sectors on-chain: {:?}", proven_sectors); - sector.state = SectorState::ProveCommitted; + let sector = ProvenSector::create(sector); state.db.save_sector(sector.sector_number, §or)?; Ok(()) diff --git a/cli/polka-storage-provider/server/src/pipeline/types.rs b/cli/polka-storage-provider/server/src/pipeline/types.rs index 8628a5fde..5451446ed 100644 --- a/cli/polka-storage-provider/server/src/pipeline/types.rs +++ b/cli/polka-storage-provider/server/src/pipeline/types.rs @@ -42,31 +42,51 @@ pub struct ProveCommitMessage { pub sector_number: SectorNumber, } -/// Sector State serialized and stored in the RocksDB database -/// It is used for tracking the sector lifetime, precommiting and proving. +/// Unsealed Sector which still accepts deals and pieces. +/// When sealed it's converted into [`PreCommittedSector`]. #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] -pub struct Sector { +pub struct UnsealedSector { /// [`SectorNumber`] which identifies a sector in the Storage Provider. /// /// It *should be centrally generated* by the Storage Provider, currently by [`crate::db::DealDB::next_sector_number`]. pub sector_number: SectorNumber, - /// Initially the sector is in [`SectorState::Unsealed`] state, should be changed after each of the sealing steps. - pub state: SectorState, + /// Tracks how much bytes have been written into [`Sector::unsealed_path`] /// by [`polka_storage_proofs::porep::sealer::Sealer::add_piece`] which adds padding. /// /// It is used before precomit to calculate padding /// with zero pieces by [`polka_storage_proofs::porep::sealer::Sealer::pad_sector`]. pub occupied_sector_space: u64, + /// Tracks all of the pieces that has been added to the sector. /// Indexes match with corresponding deals in [`Sector::deals`]. pub piece_infos: Vec, + /// Tracks all of the deals that have been added to the sector. pub deals: Vec<(DealId, DealProposal)>, + /// Path of an existing file where the pieces unsealed and padded data is stored. /// /// File at this path is created when the sector is created by [`Sector::create`]. pub unsealed_path: std::path::PathBuf, +} + +/// Sector which has been sealed and pre-committed on-chain. +/// When proven, it's converted into [`ProvenSector`]. +#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] +pub struct PreCommittedSector { + /// [`SectorNumber`] which identifies a sector in the Storage Provider. + /// + /// It *should be centrally generated* by the Storage Provider, currently by [`crate::db::DealDB::next_sector_number`]. + pub sector_number: SectorNumber, + + /// Tracks all of the pieces that has been added to the sector. + /// Indexes match with corresponding deals in [`Sector::deals`]. + pub piece_infos: Vec, + + /// Tracks all of the deals that have been added to the sector. + pub deals: Vec<(DealId, DealProposal)>, + /// Path of an existing file where the sealed sector data is stored. /// /// File at this path is initially created by [`Sector::create`], however it's empty. @@ -74,66 +94,110 @@ pub struct Sector { /// Only after pipeline [`PipelineMessage::PreCommit`], /// the file has contents which should not be touched and are used for later steps. pub sealed_path: std::path::PathBuf, + /// CID of the sealed sector. - /// - /// Available at [`SectorState::Sealed`]/[`PipelineMessage::PreCommit`] and later. - pub comm_r: Option, + pub comm_r: Commitment, + /// CID of the unsealed data of the sector. - /// - /// Available at [`SectorState::Sealed`]/[`PipelineMessage::PreCommit`] and later. - pub comm_d: Option, + pub comm_d: Commitment, + /// Block at which randomness has been fetched to perform [`PipelineMessage::PreCommit`]. /// /// It is used as a randomness seed to create a replica. /// Available at [`SectorState::Sealed`] and later. - pub seal_randomness_height: Option, + pub seal_randomness_height: u64, + /// Block at which the sector was precommitted (extrinsic submitted on-chain). /// /// It is used as a randomness seed to create a PoRep. /// Available at [`SectorState::Precommitted`] and later. - pub precommit_block: Option, + pub precommit_block: u64, } -impl Sector { - /// Creates a new sector and empty files at the provided paths. +impl UnsealedSector { + /// Creates a new sector and empty file at the provided path. /// /// Sector Number must be unique - generated by [`crate::db::DealDB::next_sector_number`] /// otherwise the data will be overwritten. - pub async fn create_unsealed( + pub async fn create( sector_number: SectorNumber, unsealed_path: std::path::PathBuf, - sealed_path: std::path::PathBuf, - ) -> Result { + ) -> Result { tokio::fs::File::create_new(&unsealed_path).await?; - tokio::fs::File::create_new(&sealed_path).await?; Ok(Self { sector_number, - state: SectorState::Unsealed, occupied_sector_space: 0, piece_infos: vec![], deals: vec![], unsealed_path, + }) + } +} + +impl PreCommittedSector { + /// Transforms [`UnsealedSector`] and removes it's underlying data. + /// + /// Expects that file at `sealed_path` contains sealed_data. + /// Should only be called after sealing and pre-commit process has ended. + pub async fn create( + unsealed: UnsealedSector, + sealed_path: std::path::PathBuf, + comm_r: Commitment, + comm_d: Commitment, + seal_randomness_height: u64, + precommit_block: u64, + ) -> Result { + tokio::fs::remove_file(unsealed.unsealed_path).await?; + + Ok(Self { + sector_number: unsealed.sector_number, + piece_infos: unsealed.piece_infos, + deals: unsealed.deals, sealed_path, - comm_r: None, - comm_d: None, - seal_randomness_height: None, - precommit_block: None, + comm_r, + comm_d, + seal_randomness_height, + precommit_block, }) } } -/// Represents a Sector's lifetime, sequentially. +// Sector which has been sealed, precommitted and proven on-chain. #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] -pub enum SectorState { - /// When sector still has remaining space to add pieces or has not been sealed yet. - Unsealed, - /// After sector has been filled with pieces, padded and a replica with CommR has been created out of it. - Sealed, - /// Sealed sector has been published on-chain, so now the PoRep must be generated for it. - Precommitted, - /// After a PoRep for a sector has been generated. - Proven, - /// Generated PoRep for a sector has been published and verified on-chain. - ProveCommitted, +pub struct ProvenSector { + /// [`SectorNumber`] which identifies a sector in the Storage Provider. + /// + /// It *should be centrally generated* by the Storage Provider, currently by [`crate::db::DealDB::next_sector_number`]. + pub sector_number: SectorNumber, + + /// Tracks all of the pieces that has been added to the sector. + /// Indexes match with corresponding deals in [`Sector::deals`]. + pub piece_infos: Vec, + + /// Tracks all of the deals that have been added to the sector. + pub deals: Vec<(DealId, DealProposal)>, + + /// Path of an existing file where the sealed sector data is stored. + pub sealed_path: std::path::PathBuf, + + /// CID of the sealed sector. + pub comm_r: Commitment, + + /// CID of the unsealed data of the sector. + pub comm_d: Commitment, +} + +impl ProvenSector { + /// Creates a [`ProvenSector`] from a [`PreCommittedSector`]. + pub fn create(sector: PreCommittedSector) -> Self { + Self { + sector_number: sector.sector_number, + piece_infos: sector.piece_infos, + deals: sector.deals, + sealed_path: sector.sealed_path, + comm_r: sector.comm_r, + comm_d: sector.comm_d, + } + } } From c2a5dbfd189707fc93effe587d6fbc1f4c6433ff Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 6 Nov 2024 15:50:30 +0100 Subject: [PATCH 17/19] fix: feedback pr --- cli/polka-storage-provider/server/src/pipeline/mod.rs | 10 ++++++---- .../server/src/pipeline/types.rs | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index ea5e2b02e..6ae0ce8b3 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -294,10 +294,12 @@ async fn precommit( current_block = MINIMUM_BLOCK_WITH_RANDOMNESS_AVAILABLE; } - let Some(digest) = state.xt_client.get_randomness(current_block).await? else { - tracing::error!("Precommit was scheduled too early, when the randomness on-chain was not yet available..."); - return Err(PipelineError::RandomnessNotAvailable); - }; + let digest = state + .xt_client + .get_randomness(current_block) + .await? + .expect("randomness to be available as we wait for it"); + let entropy = state.xt_keypair.account_id().encode(); // Must match pallet's logic or otherwise proof won't be verified: // https://github.com/eigerco/polka-storage/blob/af51a9b121c9b02e0bf6f02f5e835091ab46af76/pallets/storage-provider/src/lib.rs#L1539 diff --git a/cli/polka-storage-provider/server/src/pipeline/types.rs b/cli/polka-storage-provider/server/src/pipeline/types.rs index 5451446ed..34a5cc12d 100644 --- a/cli/polka-storage-provider/server/src/pipeline/types.rs +++ b/cli/polka-storage-provider/server/src/pipeline/types.rs @@ -163,7 +163,7 @@ impl PreCommittedSector { } } -// Sector which has been sealed, precommitted and proven on-chain. +/// Sector which has been sealed, precommitted and proven on-chain. #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] pub struct ProvenSector { /// [`SectorNumber`] which identifies a sector in the Storage Provider. From 84362ec257ef2b06fd49bb30227e6195474a9c52 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 6 Nov 2024 16:35:22 +0100 Subject: [PATCH 18/19] chore: taplo --- cli/polka-storage-provider/server/Cargo.toml | 2 +- pallets/storage-provider/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/polka-storage-provider/server/Cargo.toml b/cli/polka-storage-provider/server/Cargo.toml index 7fd782802..13c67e25b 100644 --- a/cli/polka-storage-provider/server/Cargo.toml +++ b/cli/polka-storage-provider/server/Cargo.toml @@ -22,11 +22,11 @@ cid = { workspace = true, features = ["serde", "std"] } clap = { workspace = true, features = ["derive"] } codec = { workspace = true } futures = { workspace = true } +hex = { workspace = true, features = ["std"] } jsonrpsee = { workspace = true, features = ["http-client", "macros", "server", "ws-client"] } rand = { workspace = true } rocksdb = { workspace = true } sc-cli = { workspace = true } -hex = { workspace = true, features = ["std"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } subxt = { workspace = true } diff --git a/pallets/storage-provider/Cargo.toml b/pallets/storage-provider/Cargo.toml index 4307b8181..c9e88f193 100644 --- a/pallets/storage-provider/Cargo.toml +++ b/pallets/storage-provider/Cargo.toml @@ -17,6 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] cid = { workspace = true, features = ["alloc"] } codec = { workspace = true, default-features = false, features = ["derive"] } +hex = { workspace = true, default-features = false, features = ["alloc"] } log = { workspace = true, features = ["kv"] } pallet-insecure-randomness-collective-flip = { workspace = true, default-features = false } pallet-proofs = { workspace = true, default-features = false } @@ -27,7 +28,6 @@ scale-info = { workspace = true, default-features = false, features = ["derive"] sp-arithmetic = { workspace = true, default-features = false } sp-core = { workspace = true, default-features = false } sp-runtime = { workspace = true, default-features = false } -hex = { workspace = true, default-features = false, features = ["alloc"] } # Frame deps From 08c1440b488bcdeff43cfae0e791fe3c2345e680 Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Thu, 7 Nov 2024 08:20:22 +0100 Subject: [PATCH 19/19] fix: last touches --- cli/polka-storage-provider/server/src/main.rs | 2 +- cli/polka-storage-provider/server/src/pipeline/mod.rs | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/cli/polka-storage-provider/server/src/main.rs b/cli/polka-storage-provider/server/src/main.rs index 151057aea..b7d4c44a9 100644 --- a/cli/polka-storage-provider/server/src/main.rs +++ b/cli/polka-storage-provider/server/src/main.rs @@ -240,7 +240,7 @@ pub struct ServerConfiguration { /// Proof of Spacetime proof type. post_proof: RegisteredPoStProof, - /// Proving Parameters for PoRep proof + /// Proving Parameters for PoRep proof. /// For 2KiB sectors they're ~1GiB of data. porep_parameters: PoRepParameters, } diff --git a/cli/polka-storage-provider/server/src/pipeline/mod.rs b/cli/polka-storage-provider/server/src/pipeline/mod.rs index 6ae0ce8b3..b900f4216 100644 --- a/cli/polka-storage-provider/server/src/pipeline/mod.rs +++ b/cli/polka-storage-provider/server/src/pipeline/mod.rs @@ -458,13 +458,8 @@ async fn prove_commit( ); let prover_id = derive_prover_id(state.xt_keypair.account_id()); - tracing::debug!("Performing prove commit for, seal_randomness_height {}, pre_commit_block: {}, prove_commit_block: {}, entropy: {}, ticket: {}, seed: {}", - seal_randomness_height, sector.precommit_block, prove_commit_block, hex::encode(entropy), hex::encode(ticket), hex::encode(seed)); - tracing::debug!( - "Prover Id: {}, Sector Number: {}", - hex::encode(prover_id), - sector_number - ); + tracing::debug!("Performing prove commit for, seal_randomness_height {}, pre_commit_block: {}, prove_commit_block: {}, entropy: {}, ticket: {}, seed: {}, prover id: {}, sector_number: {}", + seal_randomness_height, sector.precommit_block, prove_commit_block, hex::encode(entropy), hex::encode(ticket), hex::encode(seed), hex::encode(prover_id), sector_number); let sealing_handle: JoinHandle, _>> = { let porep_params = state.porep_parameters.clone();