Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: integrate submit_windowed_post with pallet proofs #593

Merged
merged 25 commits into from
Nov 27, 2024
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
88971bb
feat: submit_windowed_post
th7nder Nov 19, 2024
dee813e
fix: set porep-verifying-key in rpc publish
th7nder Nov 20, 2024
7a400a6
refactor: clean-up unwraps and expects
th7nder Nov 20, 2024
69ca9e8
fix: set post verifying key in rpc publish
th7nder Nov 20, 2024
82c9bd1
feat: add set_post_verifying_key to storagext cli
th7nder Nov 20, 2024
bf92567
fix: set_post_verifying_key
th7nder Nov 20, 2024
2993254
fix: restore 64 bytes
th7nder Nov 20, 2024
d8ddfa7
fix(pallet-storage-provider): commr
th7nder Nov 20, 2024
9fb24f9
fix: make it work with fixed randomness
th7nder Nov 20, 2024
99d2dfc
fix: add expect instead of unwrap:
th7nder Nov 20, 2024
478f19d
try to implement runtime apis
th7nder Nov 21, 2024
c19c474
fix: remove randomness param
th7nder Nov 25, 2024
5fdd397
fix: impl example runtime api
th7nder Nov 25, 2024
4a7bece
feat: implement current_deadline runtime api
th7nder Nov 25, 2024
87f1e5f
fix: regenerate scale
th7nder Nov 25, 2024
88acdab
fix: change post randomness
th7nder Nov 25, 2024
42a5b00
fix: fixes after e2e test
th7nder Nov 25, 2024
9de3841
docs: improve em
th7nder Nov 26, 2024
0454cd4
fix: randomness test
th7nder Nov 26, 2024
bb5b984
Merge branch 'develop' into feat/449/verify-post
th7nder Nov 26, 2024
1650a40
fix: coerce error in submit windowed post
th7nder Nov 26, 2024
d32a25f
refactor: remove option qualifier
th7nder Nov 26, 2024
bf1cf69
Merge branch 'develop' into feat/449/verify-post
th7nder Nov 26, 2024
e649815
chore: remove fixed todo
th7nder Nov 27, 2024
e8cb91f
Merge branch 'develop' into feat/449/verify-post
th7nder Nov 27, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

34 changes: 24 additions & 10 deletions cli/polka-storage-provider/client/src/commands/proofs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,13 +115,20 @@ pub enum ProofsCommand {
/// It must be the same, or else it won't work.
#[arg(short, long)]
cache_directory: PathBuf,
#[arg(short, long)]
/// Directory where the PoSt proof will be stored. Defaults to the current directory.
output_path: Option<PathBuf>,
/// Sector Number used in the PoRep command.
#[arg(long)]
sector_number: u32,
/// Block Number at which the randomness should be fetched from.
/// It comes from the [`pallet_storage_provider::DeadlineInfo::challenge`] field.
#[arg(long)]
challenge_block: u64,
/// Replica file generated with `porep` command e.g. `77.sector.sealed`.
replica_path: PathBuf,
/// CID - CommR of a replica (output of `porep` command)
comm_r: String,
#[arg(short, long)]
/// Directory where the PoSt proof will be stored. Defaults to the current directory.
output_path: Option<PathBuf>,
},
}

Expand Down Expand Up @@ -405,16 +412,19 @@ impl ProofsCommand {
replica_path,
comm_r,
output_path,
sector_number,
challenge_block,
} => {
let Some(signer) = Option::<MultiPairSigner>::from(signer_key) else {
return Err(UtilsCommandError::NoSigner)?;
};
let prover_id = derive_prover_id(signer.account_id());

// Those are hardcoded for the showcase only.
// They should come from Storage Provider Node, precommits and other information.
let sector_id = 77.into();
let randomness = [1u8; 32];
let entropy = signer.account_id().encode();
let randomness = get_randomness(
DomainSeparationTag::WindowedPoStChallengeSeed,
challenge_block,
&entropy,
);

let output_path = if let Some(output_path) = output_path {
output_path
Expand All @@ -424,15 +434,18 @@ impl ProofsCommand {

let (proof_scale_filename, mut proof_scale_file) = file_with_extension(
&output_path,
format!("{}", sector_id).as_str(),
format!("{}", sector_number).as_str(),
POST_PROOF_EXT,
)?;

let comm_r =
cid::Cid::from_str(&comm_r).map_err(|_| UtilsCommandError::CommRError)?;

let sector_number = SectorNumber::try_from(sector_number)
.map_err(|_| UtilsCommandError::InvalidSectorId)?;

let replicas = vec![ReplicaInfo {
sector_id,
sector_id: sector_number,
comm_r: comm_r
.hash()
.digest()
Expand All @@ -445,6 +458,7 @@ impl ProofsCommand {
let proof_parameters = post::load_groth16_parameters(proof_parameters_path)
.map_err(|e| UtilsCommandError::GeneratePoStError(e))?;

let prover_id = derive_prover_id(signer.account_id());
let proofs = post::generate_window_post(
post_type,
&proof_parameters,
Expand Down
6 changes: 5 additions & 1 deletion cli/polka-storage-provider/server/src/pipeline/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,11 @@ async fn precommit(
})
};
let sealing_output = sealing_handle.await??;
tracing::info!("Created sector's replica: {:?}", sealing_output);
tracing::info!(
"Created sector's replica, CommD: {}, CommR: {}",
sealing_output.comm_d.cid(),
sealing_output.comm_r.cid()
);

let sealing_output_commr = Commitment::<CommR>::from(sealing_output.comm_r);
let sealing_output_commd = Commitment::<CommD>::from(sealing_output.comm_d);
Expand Down
8 changes: 3 additions & 5 deletions examples/rpc_publish.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,9 @@ target/release/storagext-cli --sr25519-key "$PROVIDER" market add-balance 250000
# register one of them as the storage provider
wait

# Register the SP
target/release/storagext-cli --sr25519-key "$PROVIDER" storage-provider register "peer_id"

# Set PoRep verifying key
target/release/storagext-cli --sr25519-key "$PROVIDER" proofs set-porep-verifying-key @2KiB.porep.vk.scale
target/release/storagext-cli --sr25519-key "//Charlie" storage-provider register "peer_id"
target/release/storagext-cli --sr25519-key "//Charlie" proofs set-porep-verifying-key @2KiB.porep.vk.scale
target/release/storagext-cli --sr25519-key "//Charlie" proofs set-post-verifying-key @2KiB.post.vk.scale

DEAL_JSON=$(
jq -n \
Expand Down
1 change: 1 addition & 0 deletions pallets/storage-provider/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ std = [
"frame-support/std",
"frame-system/std",
"pallet-balances/std",
"primitives-proofs/std",
"scale-info/std",
"sp-core/std",
"sp-io/std",
Expand Down
124 changes: 102 additions & 22 deletions pallets/storage-provider/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ pub mod pallet {
use primitives_proofs::{
derive_prover_id,
randomness::{draw_randomness, DomainSeparationTag},
Market, ProofVerification, Randomness, RegisteredPoStProof, SectorNumber,
StorageProviderValidation, MAX_SEAL_PROOF_BYTES, MAX_SECTORS_PER_CALL,
Market, ProofVerification, PublicReplicaInfo, Randomness, RegisteredPoStProof,
SectorNumber, StorageProviderValidation, MAX_SEAL_PROOF_BYTES, MAX_SECTORS_PER_CALL,
};
use scale_info::TypeInfo;
use sp_arithmetic::traits::Zero;
Expand Down Expand Up @@ -317,6 +317,8 @@ pub mod pallet {
StorageProviderNotFound,
/// Emitted when trying to access an invalid sector.
InvalidSector,
/// Emitted when trying to submit PoSt for an not-existing partition for a deadline.
InvalidPartition,
/// Emitted when submitting an invalid proof type.
InvalidProofType,
/// Emitted when the proof is invalid
Expand Down Expand Up @@ -358,6 +360,8 @@ pub mod pallet {
CouldNotTerminateDeals,
/// Tried to terminate sectors that are not mutable.
CannotTerminateImmutableDeadline,
/// Emitted when trying to submit PoSt with partitions containing too many sectors (>2349).
TooManyReplicas,
/// Inner pallet errors
GeneralPalletError(crate::error::GeneralPalletError),
}
Expand Down Expand Up @@ -658,8 +662,6 @@ pub mod pallet {
}

/// The SP uses this extrinsic to submit their Proof-of-Spacetime.
///
/// * Currently the proof is considered valid when `proof.len() > 0`.
pub fn submit_windowed_post(
origin: OriginFor<T>,
windowed_post: SubmitWindowedPoStParams,
Expand All @@ -686,10 +688,14 @@ pub mod pallet {
// Ensure a valid proof size
// TODO(@jmg-duarte,#91,19/8/24): correctly check the length
th7nder marked this conversation as resolved.
Show resolved Hide resolved
// https://github.com/filecoin-project/builtin-actors/blob/17ede2b256bc819dc309edf38e031e246a516486/actors/miner/src/lib.rs#L565-L573
ensure!(windowed_post.proof.proof_bytes.len() > 0, {
log::error!("submit_window_post: invalid proof size");
Error::<T>::PoStProofInvalid
});
ensure!(
windowed_post.proof.proof_bytes.len()
<= primitives_proofs::MAX_POST_PROOF_BYTES as usize,
{
log::error!("submit_window_post: invalid proof size");
Error::<T>::PoStProofInvalid
}
);

// If the proving period is in the future, we can't submit a proof yet
// Related issue: https://github.com/filecoin-project/specs-actors/issues/946
Expand All @@ -714,6 +720,52 @@ pub mod pallet {

Self::validate_deadline(&current_deadline, &windowed_post)?;

// record sector as proven
let all_sectors = sp.sectors.clone();
let deadlines = sp.get_deadlines_mut();
deadlines
.record_proven(
windowed_post.deadline as usize,
&all_sectors,
windowed_post.partitions.clone(),
)
.map_err(|e| Error::<T>::GeneralPalletError(e))?;

// TODO(@th7nder,#592, 19/11/2024): handle faulty and recovered sectors, we don't take them into account now
let deadlines = &sp.deadlines;
let mut replicas = BoundedBTreeMap::new();
// Take all the sectors that were assigned to all of the partitions
for partition in &windowed_post.partitions {
// Deadline is validated by `Self::validate_deadline`, so we're sure it can be used as an index.
let deadline = &deadlines.due[windowed_post.deadline as usize];
let sectors = &deadline
.partitions
.get(&partition)
.ok_or(Error::<T>::InvalidPartition)?
.sectors;
for sector_number in sectors {
// Sectors stored in the Storage Provider struct should be consistently stored, without breaking invariants.
let sector_info = &sp.sectors[sector_number];
let comm_r = Commitment::<CommR>::from_cid_bytes(&sector_info.sealed_cid)
.expect("CommR to be validated on pre-commit");
let _ = replicas
.try_insert(
*sector_number,
PublicReplicaInfo {
comm_r: comm_r.raw(),
},
)
.map_err(|_| Error::<T>::TooManyReplicas)?;
}
}

log::debug!(target: LOG_TARGET, "submit_windowed_post: index {:?} challenge {:?}, replicas: {:?}",
current_deadline.idx,
current_deadline.challenge,
replicas
);

let entropy = owner.encode();
// The `chain_commit_epoch` should be `current_deadline.challenge` as per:
//
// These issues that were filed against the original implementation:
Expand All @@ -733,25 +785,23 @@ pub mod pallet {
// * https://github.com/filecoin-project/lotus/blob/4f70204342ce83671a7a261147a18865f1618967/storage/wdpost/wdpost_run.go#L334-L338
// * https://github.com/filecoin-project/lotus/blob/4f70204342ce83671a7a261147a18865f1618967/curiosrc/window/compute_do.go#L68-L72
// * https://github.com/filecoin-project/curio/blob/45373f7fc0431e41f987ad348df7ae6e67beaff9/tasks/window/compute_do.go#L71-L75
let randomness = get_randomness::<T>(
DomainSeparationTag::WindowedPoStChallengeSeed,
current_deadline.challenge,
&entropy,
)?;

// TODO(@aidan46, #91, 2024-07-03): Validate the proof after research is done
T::ProofVerification::verify_post(
windowed_post.proof.post_proof,
randomness,
replicas,
windowed_post.proof.proof_bytes,
)?;

// record sector as proven
let all_sectors = sp.sectors.clone();
let deadlines = sp.get_deadlines_mut();
deadlines
.record_proven(
windowed_post.deadline as usize,
&all_sectors,
windowed_post.partitions,
)
.map_err(|e| Error::<T>::GeneralPalletError(e))?;
log::debug!(target: LOG_TARGET, "submit_windowed_post: proof recorded");

// Store new storage provider state
StorageProviders::<T>::set(owner.clone(), Some(sp));

log::debug!(target: LOG_TARGET, "submit_windowed_post: proof recorded");

Self::deposit_event(Event::ValidPoStSubmitted { owner });

Ok(())
Expand Down Expand Up @@ -1015,6 +1065,36 @@ pub mod pallet {
}

impl<T: Config> Pallet<T> {
/// Gets the current deadline of the storage provider.
///
/// If there is no Storage Provider of given AccountId returns [`Option::None`].
/// May exceptionally return [`Option::None`] when
/// conversion between BlockNumbers fails, but technically should not ever happen.
pub fn current_deadline(
storage_provider: &T::AccountId,
) -> Option<primitives_proofs::CurrentDeadline<BlockNumberFor<T>>> {
let sp = StorageProviders::<T>::try_get(storage_provider).ok()?;
let current_block = <frame_system::Pallet<T>>::block_number();

let deadline = sp
.deadline_info(
current_block,
T::WPoStPeriodDeadlines::get(),
T::WPoStProvingPeriod::get(),
T::WPoStChallengeWindow::get(),
T::WPoStChallengeLookBack::get(),
T::FaultDeclarationCutoff::get(),
)
.ok()?;

Some(primitives_proofs::CurrentDeadline {
deadline_index: deadline.idx,
open: deadline.is_open(),
challenge_block: deadline.challenge,
start: deadline.open_at,
})
}

fn validate_expiration(
curr_block: BlockNumberFor<T>,
activation: BlockNumberFor<T>,
Expand Down
6 changes: 3 additions & 3 deletions pallets/storage-provider/src/proofs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use frame_support::{
pallet_prelude::{ConstU32, RuntimeDebug},
sp_runtime::BoundedVec,
};
use primitives_proofs::RegisteredPoStProof;
use primitives_proofs::{RegisteredPoStProof, MAX_POST_PROOF_BYTES};
use scale_info::TypeInfo;
use sp_core::blake2_64;

Expand All @@ -14,8 +14,8 @@ use crate::partition::{PartitionNumber, MAX_PARTITIONS_PER_DEADLINE};
pub struct PoStProof {
/// The proof type, currently only one type is supported.
pub post_proof: RegisteredPoStProof,
/// The proof submission, to be checked in the storage provider pallet.
pub proof_bytes: BoundedVec<u8, ConstU32<256>>, // Arbitrary length
/// The proof submission, to be checked by [`ProofVerification::verify_post`], usually [`pallet_proofs`].
pub proof_bytes: BoundedVec<u8, ConstU32<MAX_POST_PROOF_BYTES>>,
}

/// Parameter type for `submit_windowed_post` extrinsic.
Expand Down
7 changes: 6 additions & 1 deletion pallets/storage-provider/src/tests/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ impl pallet_balances::Config for Test {
type AccountStore = System;
}

pub const INVALID_PROOF: [u8; 2] = [0xd, 0xe];

/// This is dummy proofs pallet implementation. All proofs are accepted as valid
pub struct DummyProofsVerification;
impl ProofVerification for DummyProofsVerification {
Expand All @@ -104,8 +106,11 @@ impl ProofVerification for DummyProofsVerification {
PublicReplicaInfo,
ConstU32<MAX_SECTORS_PER_PROOF>,
>,
_proof: BoundedVec<u8, ConstU32<MAX_POST_PROOF_BYTES>>,
proof: BoundedVec<u8, ConstU32<MAX_POST_PROOF_BYTES>>,
) -> sp_runtime::DispatchResult {
if *proof == INVALID_PROOF {
return Err(sp_runtime::DispatchError::Other("invalid proof"));
}
Ok(())
}
}
Expand Down
7 changes: 4 additions & 3 deletions pallets/storage-provider/src/tests/submit_windowed_post.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use crate::{
account, declare_faults::setup_sp_with_many_sectors_multiple_partitions, events,
new_test_ext, register_storage_provider, run_to_block, DealProposalBuilder, Market,
RuntimeEvent, RuntimeOrigin, SectorPreCommitInfoBuilder, StorageProvider,
SubmitWindowedPoStBuilder, System, Test, ALICE, BOB,
SubmitWindowedPoStBuilder, System, Test, ALICE, BOB, INVALID_PROOF,
},
Config,
};
Expand Down Expand Up @@ -349,7 +349,8 @@ fn fail_windowed_post_wrong_signature() {

// Build window post proof
let windowed_post = SubmitWindowedPoStBuilder::default()
.proof_bytes(vec![]) // Wrong proof
.partition(0)
.proof_bytes(INVALID_PROOF.into())
.build();

// Run extrinsic
Expand All @@ -358,7 +359,7 @@ fn fail_windowed_post_wrong_signature() {
RuntimeOrigin::signed(account(ALICE)),
windowed_post,
),
Error::<Test>::PoStProofInvalid
DispatchError::Other("invalid proof")
);
});
}
Expand Down
4 changes: 3 additions & 1 deletion primitives/proofs/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ codec = { workspace = true, default-features = false, features = ["derive"] }
scale-decode = { workspace = true, default-features = false, features = ["derive"] }
scale-encode = { workspace = true, default-features = false, features = ["derive"] }
scale-info = { workspace = true, default-features = false, features = ["derive"] }

sp-api = { workspace = true, default-features = false }
sp-core = { workspace = true, default-features = false }
sp-runtime = { workspace = true, default-features = false }
sp-std = { workspace = true, default-features = false }
Expand All @@ -30,4 +32,4 @@ workspace = true
[features]
clap = ["dep:clap", "std"]
default = ["std"]
std = ["cid/scale-codec", "codec/std", "scale-info/std", "sp-core/std", "sp-runtime/std", "sp-std/std"]
std = ["cid/scale-codec", "codec/std", "scale-info/std", "sp-api/std", "sp-core/std", "sp-runtime/std", "sp-std/std"]
Loading