feat(polka-storage-provider-server): prove commit (#502)

cli/polka-storage-provider/server/Cargo.toml

@@ -10,7 +10,7 @@ version = "0.1.0"
 [dependencies]
 # "Homegrown" crates
 mater = { workspace = true }
-polka-storage-proofs = { workspace = true, features = ["std"] }
+polka-storage-proofs = { workspace = true, features = ["std", "substrate"] }
 polka-storage-provider-common = { workspace = true }
 primitives-commitment = { workspace = true, features = ["serde", "std"] }
 primitives-proofs = { workspace = true, features = ["clap"] }
@@ -20,7 +20,9 @@ async-trait = { workspace = true }
 axum = { workspace = true, features = ["macros", "multipart"] }
 cid = { workspace = true, features = ["serde", "std"] }
 clap = { workspace = true, features = ["derive"] }
+codec = { workspace = true }
 futures = { workspace = true }
+hex = { workspace = true, features = ["std"] }
 jsonrpsee = { workspace = true, features = ["http-client", "macros", "server", "ws-client"] }
 rand = { workspace = true }
 rocksdb = { workspace = true }

cli/polka-storage-provider/server/src/db.rs

@@ -5,10 +5,9 @@ use std::{
 
 use primitives_proofs::SectorNumber;
 use rocksdb::{ColumnFamily, ColumnFamilyDescriptor, Options as DBOptions, DB as RocksDB};
+use serde::{de::DeserializeOwned, Serialize};
 use storagext::types::market::{ConversionError, DealProposal};
 
-use crate::pipeline::types::Sector;
-
 #[derive(Debug, thiserror::Error)]
 pub enum DBError {
     #[error(transparent)]
@@ -25,6 +24,9 @@ pub enum DBError {
 
     #[error(transparent)]
     Json(#[from] serde_json::Error),
+
+    #[error("unexpected data when trying to serialize given sector type: {0}")]
+    InvalidSectorData(serde_json::Error),
 }
 
 const ACCEPTED_DEAL_PROPOSALS_CF: &str = "accepted_deal_proposals";
@@ -118,25 +120,30 @@ impl DealDB {
         )?)
     }
 
-    pub fn get_sector(&self, sector_id: SectorNumber) -> Result<Option<Sector>, DBError> {
+    pub fn get_sector<SectorType: DeserializeOwned>(
+        &self,
+        sector_number: SectorNumber,
+    ) -> Result<Option<SectorType>, DBError> {
         let Some(sector_slice) = self
             .database
-            .get_pinned_cf(self.cf_handle(SECTORS_CF), sector_id.to_le_bytes())?
+            .get_pinned_cf(self.cf_handle(SECTORS_CF), sector_number.to_le_bytes())?
         else {
             return Ok(None);
         };
 
         let sector = serde_json::from_reader(sector_slice.as_ref())
-            // SAFETY: this should never fail since the API sets a sector
-            // if this happens, it means that someone wrote it from a side channel
-            .expect("invalid content was placed in the database from outside this API");
+            .map_err(|e| DBError::InvalidSectorData(e))?;
 
         Ok(Some(sector))
     }
 
-    pub fn save_sector(&self, sector: &Sector) -> Result<(), DBError> {
+    pub fn save_sector<SectorType: Serialize>(
+        &self,
+        sector_number: SectorNumber,
+        sector: &SectorType,
+    ) -> Result<(), DBError> {
         let cf_handle = self.cf_handle(SECTORS_CF);
-        let key = sector.sector_number.to_le_bytes();
+        let key = sector_number.to_le_bytes();
         let json = serde_json::to_vec(&sector)?;
 
         self.database.put_cf(cf_handle, key, json)?;

cli/polka-storage-provider/server/src/main.rs

@@ -11,6 +11,7 @@ use std::{env::temp_dir, net::SocketAddr, path::PathBuf, sync::Arc, time::Durati
 
 use clap::Parser;
 use pipeline::types::PipelineMessage;
+use polka_storage_proofs::porep::{self, PoRepParameters};
 use polka_storage_provider_common::rpc::ServerInfo;
 use primitives_proofs::{RegisteredPoStProof, RegisteredSealProof};
 use rand::Rng;
@@ -118,6 +119,9 @@ pub enum ServerError {
     #[error("proof sectors sizes do not match")]
     SectorSizeMismatch,
 
+    #[error("failed to load PoRep parameters from: {0}, because: {1}")]
+    InvalidPoRepParameters(std::path::PathBuf, porep::PoRepError),
+
     #[error("FromEnv error: {0}")]
     EnvFilter(#[from] tracing_subscriber::filter::FromEnvError),
 
@@ -194,13 +198,22 @@ pub struct ServerArguments {
     /// Proof of Spacetime proof type.
     #[arg(long)]
     post_proof: RegisteredPoStProof,
+
+    /// Proving Parameters for PoRep proof, corresponding to given `seal_proof` sector size.
+    /// They are shared across all of the nodes in the network, as the chain stores corresponding Verifying Key parameters.
+    /// Shared parameters available to get in the [root repo](http://github.com/eigerco/polka-storage/README.md#Parameters).
+    ///
+    /// Testing/temporary parameters can be also generated via `polka-storage-provider-client proofs porep-params` command.
+    /// Note that when you generate keys, for local testnet,
+    /// **they need to be set** via an extrinsic pallet-proofs::set_porep_verifyingkey.
+    #[arg(long)]
+    porep_parameters: PathBuf,
 }
 
 /// A valid server configuration. To be created using [`ServerConfiguration::try_from`].
 ///
 /// The main difference to [`Server`] is that this structure only contains validated and
 /// ready to use parameters.
-#[derive(Debug)]
 pub struct ServerConfiguration {
     /// Storage server listen address.
     upload_listen_address: SocketAddr,
@@ -226,6 +239,10 @@ pub struct ServerConfiguration {
 
     /// Proof of Spacetime proof type.
     post_proof: RegisteredPoStProof,
+
+    /// Proving Parameters for PoRep proof.
+    /// For 2KiB sectors they're ~1GiB of data.
+    porep_parameters: PoRepParameters,
 }
 
 impl TryFrom<ServerArguments> for ServerConfiguration {
@@ -264,6 +281,9 @@ impl TryFrom<ServerArguments> for ServerConfiguration {
         });
         std::fs::create_dir_all(&storage_directory)?;
 
+        let porep_parameters = porep::load_groth16_parameters(value.porep_parameters.clone())
+            .map_err(|e| ServerError::InvalidPoRepParameters(value.porep_parameters, e))?;
+
         Ok(Self {
             upload_listen_address: value.upload_listen_address,
             rpc_listen_address: value.rpc_listen_address,
@@ -273,6 +293,7 @@ impl TryFrom<ServerArguments> for ServerConfiguration {
             storage_directory,
             seal_proof: value.seal_proof,
             post_proof: value.post_proof,
+            porep_parameters,
         })
     }
 }
@@ -395,6 +416,7 @@ impl ServerConfiguration {
             unsealed_sectors_dir: unsealed_sector_storage_dir,
             sealed_sectors_dir: sealed_sector_storage_dir,
             sealing_cache_dir,
+            porep_parameters: Arc::new(self.porep_parameters),
             xt_client,
             xt_keypair: self.multi_pair_signer,
             pipeline_sender: pipeline_tx,