Track historical vrfs

runtime/parachains/Cargo.toml

@@ -21,6 +21,8 @@ sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master
 sp-session = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-staking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
+sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-consensus-vrf = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 
 pallet-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }

runtime/parachains/src/disputes.rs

@@ -19,7 +19,7 @@
 use crate::{
 	configuration::{self, HostConfiguration},
 	initializer::SessionChangeNotification,
-	session_info,
+	session_info, shared,
 };
 use bitvec::{bitvec, order::Lsb0 as BitOrderLsb0};
 use frame_support::{ensure, traits::Get, weights::Weight};
@@ -31,7 +31,7 @@ use primitives::v1::{
 	ValidDisputeStatementKind, ValidatorId, ValidatorIndex, ValidatorSignature,
 };
 use sp_runtime::{
-	traits::{AppVerify, One, Saturating, Zero},
+	traits::{AppVerify, Saturating},
 	DispatchError, RuntimeDebug, SaturatedConversion,
 };
 use sp_std::{collections::btree_set::BTreeSet, prelude::*};
@@ -116,10 +116,10 @@ pub trait DisputesHandler<BlockNumber> {
 	) -> Result<Vec<(SessionIndex, CandidateHash)>, DispatchError>;
 
 	/// Note that the given candidate has been included.
-	fn note_included(
+	fn process_included(
 		session: SessionIndex,
 		candidate_hash: CandidateHash,
-		included_in: BlockNumber,
+		revert_to: BlockNumber,
 	);
 
 	/// Whether the given candidate could be invalid, i.e. there is an ongoing
@@ -151,10 +151,10 @@ impl<BlockNumber> DisputesHandler<BlockNumber> for () {
 		Ok(Vec::new())
 	}
 
-	fn note_included(
+	fn process_included(
 		_session: SessionIndex,
 		_candidate_hash: CandidateHash,
-		_included_in: BlockNumber,
+		_revert_to: BlockNumber,
 	) {
 	}
 
@@ -186,12 +186,12 @@ impl<T: Config> DisputesHandler<T::BlockNumber> for pallet::Pallet<T> {
 		pallet::Pallet::<T>::provide_multi_dispute_data(statement_sets)
 	}
 
-	fn note_included(
+	fn process_included(
 		session: SessionIndex,
 		candidate_hash: CandidateHash,
-		included_in: T::BlockNumber,
+		revert_to: T::BlockNumber,
 	) {
-		pallet::Pallet::<T>::note_included(session, candidate_hash, included_in)
+		pallet::Pallet::<T>::process_included(session, candidate_hash, revert_to);
 	}
 
 	fn could_be_invalid(session: SessionIndex, candidate_hash: CandidateHash) -> bool {
@@ -243,18 +243,6 @@ pub mod pallet {
 		DisputeState<T::BlockNumber>,
 	>;
 
-	/// All included blocks on the chain, as well as the block number in this chain that
-	/// should be reverted back to if the candidate is disputed and determined to be invalid.
-	#[pallet::storage]
-	pub(super) type Included<T: Config> = StorageDoubleMap<
-		_,
-		Twox64Concat,
-		SessionIndex,
-		Blake2_128Concat,
-		CandidateHash,
-		T::BlockNumber,
-	>;
-
 	/// Maps session indices to a vector indicating the number of potentially-spam disputes
 	/// each validator is participating in. Potentially-spam disputes are remote disputes which have
 	/// fewer than `byzantine_threshold + 1` validators.
@@ -565,7 +553,9 @@ impl<T: Config> Pallet<T> {
 				dispute.concluded_at = Some(now);
 				<Disputes<T>>::insert(session_index, candidate_hash, &dispute);
 
-				if <Included<T>>::contains_key(&session_index, &candidate_hash) {
+				let is_local =
+					<shared::Pallet<T>>::is_included_candidate(&session_index, &candidate_hash);
+				if is_local {
 					// Local disputes don't count towards spam.
 
 					weight += T::DbWeight::get().reads_writes(1, 1);
@@ -629,10 +619,12 @@ impl<T: Config> Pallet<T> {
 			for to_prune in to_prune {
 				// This should be small, as disputes are rare, so `None` is fine.
 				<Disputes<T>>::remove_prefix(to_prune, None);
-
-				// This is larger, and will be extracted to the `shared` module for more proper pruning.
-				// TODO: https://github.com/paritytech/polkadot/issues/3469
-				<Included<T>>::remove_prefix(to_prune, None);
+				// Mark the session as pruneable so that its candidates can be incrementally
+				// removed over the course of many block inclusions.
+				shared::Pallet::<T>::mark_session_pruneable(to_prune);
+				// TODO(ladi): remove this call, currently allows unit tests to pass. Need to
+				// figure out how to invoke paras_inherent::enter in run_to_block.
+				shared::Pallet::<T>::prune_ancient_sessions(shared::MAX_CANDIDATES_TO_PRUNE);
 				SpamSlots::<T>::remove(to_prune);
 			}
 
@@ -787,7 +779,8 @@ impl<T: Config> Pallet<T> {
 		};
 
 		// Apply spam slot changes. Bail early if too many occupied.
-		let is_local = <Included<T>>::contains_key(&set.session, &set.candidate_hash);
+		let is_local =
+			<shared::Pallet<T>>::is_included_candidate(&set.session, &set.candidate_hash);
 		if !is_local {
 			let mut spam_slots: Vec<u32> =
 				SpamSlots::<T>::get(&set.session).unwrap_or_else(|| vec![0; n_validators]);
@@ -919,7 +912,8 @@ impl<T: Config> Pallet<T> {
 		};
 
 		// Apply spam slot changes. Bail early if too many occupied.
-		let is_local = <Included<T>>::contains_key(&set.session, &set.candidate_hash);
+		let is_local =
+			<shared::Pallet<T>>::is_included_candidate(&set.session, &set.candidate_hash);
 		if !is_local {
 			let mut spam_slots: Vec<u32> =
 				SpamSlots::<T>::get(&set.session).unwrap_or_else(|| vec![0; n_validators]);
@@ -993,7 +987,9 @@ impl<T: Config> Pallet<T> {
 
 		// Freeze if just concluded against some local candidate
 		if summary.new_flags.contains(DisputeStateFlags::AGAINST_SUPERMAJORITY) {
-			if let Some(revert_to) = <Included<T>>::get(&set.session, &set.candidate_hash) {
+			if let Some((revert_to, _)) =
+				<shared::Pallet<T>>::get_included_candidate(&set.session, &set.candidate_hash)
+			{
 				Self::revert_and_freeze(revert_to);
 			}
 		}
@@ -1006,19 +1002,11 @@ impl<T: Config> Pallet<T> {
 		<Disputes<T>>::iter().collect()
 	}
 
-	pub(crate) fn note_included(
+	pub(crate) fn process_included(
 		session: SessionIndex,
 		candidate_hash: CandidateHash,
-		included_in: T::BlockNumber,
+		revert_to: T::BlockNumber,
 	) {
-		if included_in.is_zero() {
-			return
-		}
-
-		let revert_to = included_in - One::one();
-
-		<Included<T>>::insert(&session, &candidate_hash, revert_to);
-
 		// If we just included a block locally which has a live dispute, decrement spam slots
 		// for any involved validators, if the dispute is not already confirmed by f + 1.
 		if let Some(state) = <Disputes<T>>::get(&session, candidate_hash) {
@@ -1130,7 +1118,7 @@ mod tests {
 		traits::{OnFinalize, OnInitialize},
 	};
 	use frame_system::InitKind;
-	use primitives::v1::BlockNumber;
+	use primitives::v1::{BlockNumber, CoreIndex};
 	use sp_core::{crypto::CryptoType, Pair};
 
 	// All arguments for `initializer::on_new_session`
@@ -1171,6 +1159,21 @@ mod tests {
 		}
 	}
 
+	fn include_candidate(
+		session: SessionIndex,
+		candidate_hash: &CandidateHash,
+		block_number: BlockNumber,
+	) {
+		if let Some(revert_to) = shared::Pallet::<Test>::note_included_candidate(
+			session,
+			candidate_hash.clone(),
+			block_number,
+			CoreIndex(0),
+		) {
+			Pallet::<Test>::process_included(session, candidate_hash.clone(), revert_to);
+		}
+	}
+
 	#[test]
 	fn test_dispute_state_flag_from_state() {
 		assert_eq!(
@@ -1469,13 +1472,13 @@ mod tests {
 			let v0 = <ValidatorId as CryptoType>::Pair::generate().0;
 
 			let candidate_hash = CandidateHash(sp_core::H256::repeat_byte(1));
-			Pallet::<Test>::note_included(0, candidate_hash.clone(), 0);
-			Pallet::<Test>::note_included(1, candidate_hash.clone(), 1);
-			Pallet::<Test>::note_included(2, candidate_hash.clone(), 2);
-			Pallet::<Test>::note_included(3, candidate_hash.clone(), 3);
-			Pallet::<Test>::note_included(4, candidate_hash.clone(), 4);
-			Pallet::<Test>::note_included(5, candidate_hash.clone(), 5);
-			Pallet::<Test>::note_included(6, candidate_hash.clone(), 5);
+			include_candidate(0, &candidate_hash, 0);
+			include_candidate(1, &candidate_hash, 1);
+			include_candidate(2, &candidate_hash, 2);
+			include_candidate(3, &candidate_hash, 3);
+			include_candidate(4, &candidate_hash, 4);
+			include_candidate(5, &candidate_hash, 5);
+			include_candidate(6, &candidate_hash, 5);
 
 			run_to_block(7, |b| {
 				// a new session at each block
@@ -1485,13 +1488,13 @@ mod tests {
 			// current session is 7,
 			// we keep for dispute_period + 1 session and we remove in on_finalize
 			// thus we keep info for session 3, 4, 5, 6, 7.
-			assert_eq!(Included::<Test>::iter_prefix(0).count(), 0);
-			assert_eq!(Included::<Test>::iter_prefix(1).count(), 0);
-			assert_eq!(Included::<Test>::iter_prefix(2).count(), 0);
-			assert_eq!(Included::<Test>::iter_prefix(3).count(), 1);
-			assert_eq!(Included::<Test>::iter_prefix(4).count(), 1);
-			assert_eq!(Included::<Test>::iter_prefix(5).count(), 1);
-			assert_eq!(Included::<Test>::iter_prefix(6).count(), 1);
+			assert_eq!(shared::Pallet::<Test>::included_candidates_iter_prefix(0).count(), 0);
+			assert_eq!(shared::Pallet::<Test>::included_candidates_iter_prefix(1).count(), 0);
+			assert_eq!(shared::Pallet::<Test>::included_candidates_iter_prefix(2).count(), 0);
+			assert_eq!(shared::Pallet::<Test>::included_candidates_iter_prefix(3).count(), 1);
+			assert_eq!(shared::Pallet::<Test>::included_candidates_iter_prefix(4).count(), 1);
+			assert_eq!(shared::Pallet::<Test>::included_candidates_iter_prefix(5).count(), 1);
+			assert_eq!(shared::Pallet::<Test>::included_candidates_iter_prefix(6).count(), 1);
 		});
 	}
 
@@ -1593,7 +1596,7 @@ mod tests {
 	}
 
 	#[test]
-	fn test_freeze_on_note_included() {
+	fn test_freeze_on_process_included() {
 		new_test_ext(Default::default()).execute_with(|| {
 			let v0 = <ValidatorId as CryptoType>::Pair::generate().0;
 
@@ -1623,7 +1626,7 @@ mod tests {
 			}];
 			assert!(Pallet::<Test>::provide_multi_dispute_data(stmts).is_ok());
 
-			Pallet::<Test>::note_included(3, candidate_hash.clone(), 3);
+			include_candidate(3, &candidate_hash, 3);
 			assert_eq!(Frozen::<Test>::get(), Some(2));
 		});
 	}
@@ -1640,7 +1643,7 @@ mod tests {
 
 			let candidate_hash = CandidateHash(sp_core::H256::repeat_byte(1));
 
-			Pallet::<Test>::note_included(3, candidate_hash.clone(), 3);
+			include_candidate(3, &candidate_hash, 3);
 
 			// v0 votes for 3
 			let stmts = vec![DisputeStatementSet {
@@ -1669,7 +1672,7 @@ mod tests {
 	// * provide_multi_dispute: with success scenario
 	// * disputes: correctness of datas
 	// * could_be_invalid: correctness of datas
-	// * note_included: decrement spam correctly
+	// * process_included: decrement spam correctly
 	// * spam slots: correctly incremented and decremented
 	// * ensure rewards and punishment are correctly called.
 	#[test]
@@ -1945,7 +1948,7 @@ mod tests {
 
 			// Ensure inclusion removes spam slots
 			assert_eq!(SpamSlots::<Test>::get(4), Some(vec![0, 0, 0, 1]));
-			Pallet::<Test>::note_included(4, candidate_hash.clone(), 4);
+			include_candidate(4, &candidate_hash, 4);
 			assert_eq!(SpamSlots::<Test>::get(4), Some(vec![0, 0, 0, 0]));
 
 			// Ensure the reward_validator function was correctly called

runtime/parachains/src/paras_inherent.rs

@@ -211,8 +211,18 @@ pub mod pallet {
 
 			// Inform the disputes module of all included candidates.
 			let now = <frame_system::Pallet<T>>::block_number();
-			for (_, candidate_hash) in &freed_concluded {
-				T::DisputesHandler::note_included(current_session, *candidate_hash, now);
+			for (core_index, candidate_hash) in &freed_concluded {
+				let revert_to = match shared::Pallet::<T>::note_included_candidate(
+					current_session,
+					*candidate_hash,
+					now,
+					*core_index,
+				) {
+					Some(revert_to) => revert_to,
+					None => continue,
+				};
+
+				T::DisputesHandler::process_included(current_session, *candidate_hash, revert_to);
 			}
 
 			// Handle timeouts for any availability core work.
@@ -258,6 +268,8 @@ pub mod pallet {
 				<scheduler::Pallet<T>>::group_validators,
 			)?;
 
+			let vrfs = <shared::Pallet<T>>::extract_vrfs();
+			let _ = <shared::Pallet<T>>::note_vrfs(current_session, now, vrfs);
 			// Note which of the scheduled cores were actually occupied by a backed candidate.
 			<scheduler::Pallet<T>>::occupied(&occupied);
 
@@ -267,6 +279,9 @@ pub mod pallet {
 			// And track that we've finished processing the inherent for this block.
 			Included::<T>::set(Some(()));
 
+			// Prune candidates incrementally with each block inclusion.
+			shared::Pallet::<T>::prune_ancient_sessions(shared::MAX_CANDIDATES_TO_PRUNE);
+
 			Ok(Some(
 				MINIMAL_INCLUSION_INHERENT_WEIGHT +
 					(backed_candidates_len * BACKED_CANDIDATE_WEIGHT),
@@ -324,6 +339,8 @@ fn limit_backed_candidates<T: Config>(
 mod tests {
 	use super::*;
 
+	use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof};
+
 	use crate::mock::{new_test_ext, MockGenesisConfig, System, Test};
 
 	mod limit_backed_candidates {
@@ -492,5 +509,38 @@ mod tests {
 				assert_eq!(post_info.actual_weight.unwrap(), expected_weight);
 			});
 		}
+
+		// used for generating assignments where the validity of the VRF doesn't matter.
+		fn garbage_vrf() -> (VRFOutput, VRFProof) {
+			use sp_consensus_babe::Transcript;
+			let key = keyring::Sr25519Keyring::Alice.pair();
+			let key = key.as_ref();
+
+			use sp_runtime::ConsensusEngineId;
+			const BABE_ENGINE_ID: ConsensusEngineId = *b"BABE";
+			let (o, p, _) = key.vrf_sign(Transcript::new(&BABE_ENGINE_ID));
+			(VRFOutput(o.to_output()), VRFProof(p))
+		}
+
+		#[test]
+		fn test_vrf() {
+			use sp_runtime::{ConsensusEngineId, Digest, DigestItem};
+			const BABE_ENGINE_ID: ConsensusEngineId = *b"BABE";
+			new_test_ext(MockGenesisConfig::default()).execute_with(|| {
+				let mut digest = Digest::<<Test as frame_system::Config>::Hash>::default();
+				let (vrf_output, _) = garbage_vrf();
+				digest.push(DigestItem::PreRuntime(BABE_ENGINE_ID, vrf_output.encode()));
+				let (vrf_output, _) = garbage_vrf();
+				digest.push(DigestItem::PreRuntime(BABE_ENGINE_ID, vrf_output.encode()));
+				let (vrf_output, _) = garbage_vrf();
+				digest.push(DigestItem::PreRuntime(BABE_ENGINE_ID, vrf_output.encode()));
+				System::initialize(
+					&(1),
+					&Default::default(),
+					&digest,
+					frame_system::InitKind::Full,
+				);
+			});
+		}
 	}
 }