From e822aaa50f492d615b38112e76c80340eda99c09 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 14:38:20 +0200 Subject: [PATCH 01/39] stwo prover function skeleton --- backend/Cargo.toml | 4 + backend/src/lib.rs | 6 + backend/src/stwo/circuit_builder_paralle.rs | 230 ++++++++++++++++++++ backend/src/stwo/mod.rs | 63 ++++++ backend/src/stwo/prover.rs | 29 +++ cli/Cargo.toml | 1 + pipeline/Cargo.toml | 1 + rust-toolchain.toml | 2 +- 8 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 backend/src/stwo/circuit_builder_paralle.rs create mode 100644 backend/src/stwo/mod.rs create mode 100644 backend/src/stwo/prover.rs diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 1d1798b0d1..7b339556cd 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -12,6 +12,7 @@ default = [] halo2 = ["dep:halo2_proofs", "dep:halo2_curves", "dep:snark-verifier", "dep:halo2_solidity_verifier"] estark-polygon = ["dep:pil-stark-prover"] plonky3 = ["dep:powdr-plonky3"] +stwo = ["dep:stwo-prover"] [dependencies] powdr-ast.workspace = true @@ -35,6 +36,9 @@ halo2_curves = { version = "0.6.1", package = "halo2curves", optional = true} snark-verifier = { git = "https://github.com/powdr-labs/snark-verifier", branch = "powdr-halo2", optional = true } halo2_solidity_verifier = { git = "https://github.com/powdr-labs/halo2-solidity-verifier", branch = "main", features = ["evm"], optional = true } +#change this to main branch when updatetoolchain branch is merged,the main branch is using "nightly-2024-01-04", not compatiable with plonky3 +stwo-prover = { git= "https://github.com/starkware-libs/stwo.git",optional=true, branch = "andrew/dev/update-toolchain" } + strum = { version = "0.24.1", features = ["derive"] } log = "0.4.17" serde = "1.0" diff --git a/backend/src/lib.rs b/backend/src/lib.rs index 86ab2ab089..2604cadb43 100644 --- a/backend/src/lib.rs +++ b/backend/src/lib.rs @@ -8,6 +8,7 @@ mod plonky3; mod composite; mod field_filter; +mod stwo; use powdr_ast::analyzed::Analyzed; use powdr_executor::{constant_evaluator::VariablySizedColumn, witgen::WitgenCallback}; @@ -49,6 +50,9 @@ pub enum BackendType { #[cfg(feature = "plonky3")] #[strum(serialize = "plonky3-composite")] Plonky3Composite, + #[cfg(feature = "stwo")] + #[strum(serialize = "stwo")] + Stwo, } pub type BackendOptions = String; @@ -91,6 +95,8 @@ impl BackendType { BackendType::Plonky3Composite => { Box::new(composite::CompositeBackendFactory::new(plonky3::Factory)) } + #[cfg(feature = "stwo")] + BackendType::Stwo => Box::new(stwo::StwoProverFactory), } } } diff --git a/backend/src/stwo/circuit_builder_paralle.rs b/backend/src/stwo/circuit_builder_paralle.rs new file mode 100644 index 0000000000..aad122d5b0 --- /dev/null +++ b/backend/src/stwo/circuit_builder_paralle.rs @@ -0,0 +1,230 @@ +use itertools::Itertools; +use num_traits::{One, Zero}; +use powdr_ast::analyzed::{ + AlgebraicBinaryOperation, AlgebraicBinaryOperator, AlgebraicExpression, + AlgebraicUnaryOperation, AlgebraicUnaryOperator, Analyzed, IdentityKind, PolynomialType, +}; +use powdr_executor::witgen::WitgenCallback; +use powdr_number::{FieldElement, KnownField}; + +use stwo_prover::constraint_framework::{EvalAtRow, FrameworkComponent, FrameworkEval}; +use stwo_prover::core::backend::simd::m31::{PackedBaseField, LOG_N_LANES, N_LANES}; +use stwo_prover::core::backend::simd::SimdBackend; +use stwo_prover::core::backend::{Col, Column,CpuBackend}; +use stwo_prover::core::fields::m31::BaseField; +use stwo_prover::core::fields::FieldExpOps; +use stwo_prover::core::poly::circle::{CanonicCoset, CircleEvaluation}; +use stwo_prover::core::poly::BitReversedOrder; +use stwo_prover::core::ColumnVec; +use stwo_prover::core::pcs::{CommitmentSchemeProver, CommitmentSchemeVerifier, PcsConfig, TreeVec}; + +// Type alias for a wide Fibonacci component with a constant size. +pub type WideFibonacciComponent = FrameworkComponent>; + +// Input structure for Fibonacci computation. +#[derive(Debug)] +pub struct FibInput { + a: PackedBaseField, + b: PackedBaseField, +} + +// Evaluation structure for wide Fibonacci computation. +#[derive(Debug)] +pub struct WideFibonacciEval { + pub log_n_rows: u32, +} + +impl FrameworkEval for WideFibonacciEval { + fn log_size(&self) -> u32 { + self.log_n_rows + } + fn max_constraint_log_degree_bound(&self) -> u32 { + self.log_n_rows + 1 + } + fn evaluate(&self, mut eval: E) -> E { + let mut a = eval.next_trace_mask(); + let mut b = eval.next_trace_mask(); + for _ in 2..N { + let c = eval.next_trace_mask(); + eval.add_constraint(c - (a + b)); + a = b; + b = c; + } + eval + } +} + +// Circuit structure for POWDR. +pub(crate) struct PowdrCircuit<'a, T> { + analyzed: &'a Analyzed, + witness: Option<&'a [(String, Vec)]>, + _witgen_callback: Option>, +} + +impl<'a, T: FieldElement> PowdrCircuit<'a, T> { + // Constructs a new circuit instance. + pub(crate) fn new(analyzed: &'a Analyzed) -> Self { + Self { + analyzed, + witness: None, + _witgen_callback: None, + } + } + + // Returns the witness if it is set. + fn witness(&self) -> &'a [(String, Vec)] { + self.witness.as_ref().unwrap() + } + + // Associates a witness with the circuit. + pub(crate) fn with_witness(self, witness: &'a [(String, Vec)]) -> Self { + assert_eq!(witness.len(), self.analyzed.commitment_count()); + Self { + witness: Some(witness), + ..self + } + } + + // Associates a witgen callback with the circuit. + pub(crate) fn with_witgen_callback(self, witgen_callback: WitgenCallback) -> Self { + Self { + _witgen_callback: Some(witgen_callback), + ..self + } + } +} + +/// FrameworkEval is a trait that stwo uses to define constraints +impl<'a, T: FieldElement> FrameworkEval for PowdrCircuit<'a, T> { + fn log_size(&self) -> u32 { + // Assuming the log size is based on the analyzed data. + // Modify this logic as per the specific requirements. + self.analyzed.degree().ilog2() + } + + fn max_constraint_log_degree_bound(&self) -> u32 { + // Assuming the max constraint log degree is calculated based on the analyzed data. + // Modify this logic as per the specific requirements. + self.analyzed.degree().ilog2() + } + + fn evaluate(&self, mut eval: E) -> E { + // Assuming we are evaluating constraints based on the witness data. + // This is an example, modify according to the specific logic of the circuit. + + eval + } +} + + +/// Generate execution trace +pub fn generate_trace(length: usize, witness: &[(String, Vec)], log_n_instances: u32 +)-> ColumnVec> { + + let trace: Vec = witness + .iter() + .flat_map(|(_, vec)| { + vec.iter().flat_map(|mersenne| { + + let ptr = mersenne as *const T as *const u32; + + let value = unsafe { + *ptr // Dereference the pointer to get the u32 value + }; + + // Convert chunks to PackedBaseField + // Note: We use unsafe block because PackedBaseField::load is unsafe + + vec![ + BaseField::from_u32_unchecked(value), + ] + + }) + }) + .collect(); // Collect the flattened iterator into a Vec + + + + println!("the generated stwo trace is {:?}", trace); + + let domain = CanonicCoset::new((length as u32)); + // let evaluated_trace = CircleEvaluation::::new(domain, trace); + // vec![evaluated_trace] + + vec![CircleEvaluation::new_canonical_ordered(domain, trace)] +} + + + + + + +pub fn generate_parallel_stwo_trace_by_witness_repitition(length: usize, witness: &[(String, Vec)], log_n_instances: u32 +)-> ColumnVec> { + + let trace: Vec = witness + .iter() + .flat_map(|(_, vec)| { + vec.iter().flat_map(|mersenne| { + + let ptr = mersenne as *const T as *const u32; + + let value = unsafe { + *ptr // Dereference the pointer to get the u32 value + }; + + // Repeat the value 32 times + let repeated = vec![value; 32]; + + // Split the repeated vector into two chunks of 16 elements each + let chunk1: [u32; N_LANES] = repeated[0..16] + .try_into() + .expect("Chunk should be of size N_LANES"); + let chunk2: [u32; N_LANES] = repeated[16..32] + .try_into() + .expect("Chunk should be of size N_LANES"); + + // Convert chunks to PackedBaseField + // Note: We use unsafe block because PackedBaseField::load is unsafe + unsafe { + vec![ + PackedBaseField::load(chunk1.as_ptr()), + PackedBaseField::load(chunk2.as_ptr()), + ] + } + }) + }) + .collect(); // Collect the flattened iterator into a Vec + + + // println!("from generate stwo trace trace"); + // println!("{:?}", trace); + + let mut trace_stwo= (0..length)//fibonacci length + .map(|_| Col::::zeros(1 << log_n_instances)) + .collect_vec(); + + + // column x + trace_stwo[0].data[0]= trace[0]; //x + trace_stwo[0].data[1]= trace[1]; //y + + + for i in 1..length { + trace_stwo[i].data[0] = trace[2*length + 2 * (i - 1)]; + trace_stwo[i].data[1] = trace[2*length + 2 * (i - 1) + 1]; + } + + // println!("from generate stwo trace trace_stwo repititions"); + // println!("{:?}", trace_stwo); + + let domain = CanonicCoset::new(5).circle_domain(); + trace_stwo + .into_iter() + .map(|eval| CircleEvaluation::::new(domain, eval)) + .collect_vec() + + + + +} diff --git a/backend/src/stwo/mod.rs b/backend/src/stwo/mod.rs new file mode 100644 index 0000000000..d2f9f7578d --- /dev/null +++ b/backend/src/stwo/mod.rs @@ -0,0 +1,63 @@ +use std::io; +use std::path::PathBuf; +use std::sync::Arc; + +use crate::{Backend, BackendFactory, BackendOptions, Error, Proof}; +use powdr_ast::analyzed::Analyzed; +use powdr_executor::constant_evaluator::{get_uniquely_sized_cloned, VariablySizedColumn}; +use powdr_executor::witgen::WitgenCallback; +use powdr_number::FieldElement; +use prover::StwoProver; + +mod prover; + +pub(crate) struct StwoProverFactory; + +impl BackendFactory for StwoProverFactory { + fn create( + &self, + pil: Arc>, + fixed: Arc)>>, + _output_dir: Option, + setup: Option<&mut dyn io::Read>, + verification_key: Option<&mut dyn io::Read>, + verification_app_key: Option<&mut dyn io::Read>, + options: BackendOptions, + ) -> Result>, Error> { + if pil.degrees().len() > 1 { + return Err(Error::NoVariableDegreeAvailable); + } + let fixed = Arc::new( + get_uniquely_sized_cloned(&fixed).map_err(|_| Error::NoVariableDegreeAvailable)?, + ); + let stwo = Box::new(StwoProver::new(pil, fixed, setup)?); + println!("StwoProverFactory create not implemented yet"); + Ok(stwo) + } +} + +impl Backend for StwoProver { + fn verify(&self, proof: &[u8], instances: &[Vec]) -> Result<(), Error> { + assert!(instances.len() == 1); + Ok(()) + } + + fn prove( + &self, + witness: &[(String, Vec)], + prev_proof: Option, + witgen_callback: WitgenCallback, + ) -> Result { + if prev_proof.is_some() { + return Err(Error::NoAggregationAvailable); + } + + panic!("prove function is not implement yet"); + let mut proof = vec![0u8; 10]; + Ok(proof) + } + + fn export_verification_key(&self, output: &mut dyn io::Write) -> Result<(), Error> { + Ok(()) + } +} diff --git a/backend/src/stwo/prover.rs b/backend/src/stwo/prover.rs new file mode 100644 index 0000000000..5f6459835c --- /dev/null +++ b/backend/src/stwo/prover.rs @@ -0,0 +1,29 @@ +use powdr_ast::analyzed::Analyzed; +use std::sync::Arc; +use std::{io, option}; + +use powdr_number::FieldElement; + +pub struct StwoProver { + analyzed: Arc>, + fixed: Arc)>>, + /// Proving key placeholder + proving_key: Option<()>, + /// Verifying key placeholder + verifying_key: Option<()>, +} + +impl StwoProver { + pub fn new( + analyzed: Arc>, + fixed: Arc)>>, + setup: Option<&mut dyn io::Read>, + ) -> Result { + Ok(Self { + analyzed, + fixed, + proving_key: None, + verifying_key: None, + }) + } +} diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 6471d52fdb..7d84d1add6 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -16,6 +16,7 @@ estark-polygon = [ "powdr-backend/estark-polygon", "powdr-pipeline/estark-polygon", ] +stwo=["powdr-backend/stwo","powdr-pipeline/stwo"] [dependencies] powdr-backend.workspace = true diff --git a/pipeline/Cargo.toml b/pipeline/Cargo.toml index 9e29aa376b..68b15d9e51 100644 --- a/pipeline/Cargo.toml +++ b/pipeline/Cargo.toml @@ -12,6 +12,7 @@ default = [] # halo2 is disabled by default halo2 = ["powdr-backend/halo2"] plonky3 = ["powdr-backend/plonky3"] estark-polygon = ["powdr-backend/estark-polygon"] +stwo = ["powdr-backend/stwo"] [dependencies] powdr-airgen.workspace = true diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 4cef0b738f..b6930c7af4 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.81" +channel = "nightly-2024-09-21" From 245d95c2e8adb3b06536492b721ad5407e3b26e3 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 14:41:33 +0200 Subject: [PATCH 02/39] remove circuit build file, not ready yet --- backend/src/stwo/circuit_builder_paralle.rs | 230 -------------------- 1 file changed, 230 deletions(-) delete mode 100644 backend/src/stwo/circuit_builder_paralle.rs diff --git a/backend/src/stwo/circuit_builder_paralle.rs b/backend/src/stwo/circuit_builder_paralle.rs deleted file mode 100644 index aad122d5b0..0000000000 --- a/backend/src/stwo/circuit_builder_paralle.rs +++ /dev/null @@ -1,230 +0,0 @@ -use itertools::Itertools; -use num_traits::{One, Zero}; -use powdr_ast::analyzed::{ - AlgebraicBinaryOperation, AlgebraicBinaryOperator, AlgebraicExpression, - AlgebraicUnaryOperation, AlgebraicUnaryOperator, Analyzed, IdentityKind, PolynomialType, -}; -use powdr_executor::witgen::WitgenCallback; -use powdr_number::{FieldElement, KnownField}; - -use stwo_prover::constraint_framework::{EvalAtRow, FrameworkComponent, FrameworkEval}; -use stwo_prover::core::backend::simd::m31::{PackedBaseField, LOG_N_LANES, N_LANES}; -use stwo_prover::core::backend::simd::SimdBackend; -use stwo_prover::core::backend::{Col, Column,CpuBackend}; -use stwo_prover::core::fields::m31::BaseField; -use stwo_prover::core::fields::FieldExpOps; -use stwo_prover::core::poly::circle::{CanonicCoset, CircleEvaluation}; -use stwo_prover::core::poly::BitReversedOrder; -use stwo_prover::core::ColumnVec; -use stwo_prover::core::pcs::{CommitmentSchemeProver, CommitmentSchemeVerifier, PcsConfig, TreeVec}; - -// Type alias for a wide Fibonacci component with a constant size. -pub type WideFibonacciComponent = FrameworkComponent>; - -// Input structure for Fibonacci computation. -#[derive(Debug)] -pub struct FibInput { - a: PackedBaseField, - b: PackedBaseField, -} - -// Evaluation structure for wide Fibonacci computation. -#[derive(Debug)] -pub struct WideFibonacciEval { - pub log_n_rows: u32, -} - -impl FrameworkEval for WideFibonacciEval { - fn log_size(&self) -> u32 { - self.log_n_rows - } - fn max_constraint_log_degree_bound(&self) -> u32 { - self.log_n_rows + 1 - } - fn evaluate(&self, mut eval: E) -> E { - let mut a = eval.next_trace_mask(); - let mut b = eval.next_trace_mask(); - for _ in 2..N { - let c = eval.next_trace_mask(); - eval.add_constraint(c - (a + b)); - a = b; - b = c; - } - eval - } -} - -// Circuit structure for POWDR. -pub(crate) struct PowdrCircuit<'a, T> { - analyzed: &'a Analyzed, - witness: Option<&'a [(String, Vec)]>, - _witgen_callback: Option>, -} - -impl<'a, T: FieldElement> PowdrCircuit<'a, T> { - // Constructs a new circuit instance. - pub(crate) fn new(analyzed: &'a Analyzed) -> Self { - Self { - analyzed, - witness: None, - _witgen_callback: None, - } - } - - // Returns the witness if it is set. - fn witness(&self) -> &'a [(String, Vec)] { - self.witness.as_ref().unwrap() - } - - // Associates a witness with the circuit. - pub(crate) fn with_witness(self, witness: &'a [(String, Vec)]) -> Self { - assert_eq!(witness.len(), self.analyzed.commitment_count()); - Self { - witness: Some(witness), - ..self - } - } - - // Associates a witgen callback with the circuit. - pub(crate) fn with_witgen_callback(self, witgen_callback: WitgenCallback) -> Self { - Self { - _witgen_callback: Some(witgen_callback), - ..self - } - } -} - -/// FrameworkEval is a trait that stwo uses to define constraints -impl<'a, T: FieldElement> FrameworkEval for PowdrCircuit<'a, T> { - fn log_size(&self) -> u32 { - // Assuming the log size is based on the analyzed data. - // Modify this logic as per the specific requirements. - self.analyzed.degree().ilog2() - } - - fn max_constraint_log_degree_bound(&self) -> u32 { - // Assuming the max constraint log degree is calculated based on the analyzed data. - // Modify this logic as per the specific requirements. - self.analyzed.degree().ilog2() - } - - fn evaluate(&self, mut eval: E) -> E { - // Assuming we are evaluating constraints based on the witness data. - // This is an example, modify according to the specific logic of the circuit. - - eval - } -} - - -/// Generate execution trace -pub fn generate_trace(length: usize, witness: &[(String, Vec)], log_n_instances: u32 -)-> ColumnVec> { - - let trace: Vec = witness - .iter() - .flat_map(|(_, vec)| { - vec.iter().flat_map(|mersenne| { - - let ptr = mersenne as *const T as *const u32; - - let value = unsafe { - *ptr // Dereference the pointer to get the u32 value - }; - - // Convert chunks to PackedBaseField - // Note: We use unsafe block because PackedBaseField::load is unsafe - - vec![ - BaseField::from_u32_unchecked(value), - ] - - }) - }) - .collect(); // Collect the flattened iterator into a Vec - - - - println!("the generated stwo trace is {:?}", trace); - - let domain = CanonicCoset::new((length as u32)); - // let evaluated_trace = CircleEvaluation::::new(domain, trace); - // vec![evaluated_trace] - - vec![CircleEvaluation::new_canonical_ordered(domain, trace)] -} - - - - - - -pub fn generate_parallel_stwo_trace_by_witness_repitition(length: usize, witness: &[(String, Vec)], log_n_instances: u32 -)-> ColumnVec> { - - let trace: Vec = witness - .iter() - .flat_map(|(_, vec)| { - vec.iter().flat_map(|mersenne| { - - let ptr = mersenne as *const T as *const u32; - - let value = unsafe { - *ptr // Dereference the pointer to get the u32 value - }; - - // Repeat the value 32 times - let repeated = vec![value; 32]; - - // Split the repeated vector into two chunks of 16 elements each - let chunk1: [u32; N_LANES] = repeated[0..16] - .try_into() - .expect("Chunk should be of size N_LANES"); - let chunk2: [u32; N_LANES] = repeated[16..32] - .try_into() - .expect("Chunk should be of size N_LANES"); - - // Convert chunks to PackedBaseField - // Note: We use unsafe block because PackedBaseField::load is unsafe - unsafe { - vec![ - PackedBaseField::load(chunk1.as_ptr()), - PackedBaseField::load(chunk2.as_ptr()), - ] - } - }) - }) - .collect(); // Collect the flattened iterator into a Vec - - - // println!("from generate stwo trace trace"); - // println!("{:?}", trace); - - let mut trace_stwo= (0..length)//fibonacci length - .map(|_| Col::::zeros(1 << log_n_instances)) - .collect_vec(); - - - // column x - trace_stwo[0].data[0]= trace[0]; //x - trace_stwo[0].data[1]= trace[1]; //y - - - for i in 1..length { - trace_stwo[i].data[0] = trace[2*length + 2 * (i - 1)]; - trace_stwo[i].data[1] = trace[2*length + 2 * (i - 1) + 1]; - } - - // println!("from generate stwo trace trace_stwo repititions"); - // println!("{:?}", trace_stwo); - - let domain = CanonicCoset::new(5).circle_domain(); - trace_stwo - .into_iter() - .map(|eval| CircleEvaluation::::new(domain, eval)) - .collect_vec() - - - - -} From 3ef47d7fff4aca30f8c63ba262166374caf2b881 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 14:49:54 +0200 Subject: [PATCH 03/39] print info change --- backend/src/stwo/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/stwo/mod.rs b/backend/src/stwo/mod.rs index d2f9f7578d..0ed23ed617 100644 --- a/backend/src/stwo/mod.rs +++ b/backend/src/stwo/mod.rs @@ -31,7 +31,7 @@ impl BackendFactory for StwoProverFactory { get_uniquely_sized_cloned(&fixed).map_err(|_| Error::NoVariableDegreeAvailable)?, ); let stwo = Box::new(StwoProver::new(pil, fixed, setup)?); - println!("StwoProverFactory create not implemented yet"); + println!("StwoProverFactory is not complete yet"); Ok(stwo) } } From 83122aa1a195c52dbdbed2d3ee8dc2be784e1787 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 16:29:26 +0200 Subject: [PATCH 04/39] change back toolchain to be stable, but make github action to run stwo feature using the nightly toolchain only when required --- .github/workflows/pr-tests.yml | 19 +++++++++++++++++++ rust-toolchain.toml | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index bf3d59a5f1..523e0709f8 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -22,6 +22,10 @@ jobs: build: runs-on: warp-ubuntu-latest-x64-4x + strategy: + matrix: + toolchain: [stable, nightly] + steps: - uses: actions/checkout@v4 with: @@ -39,12 +43,27 @@ jobs: key: ${{ runner.os }}-cargo-pr-tests - name: Install Rust toolchain 1.81 (with clippy and rustfmt) run: rustup toolchain install 1.81-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.81-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu + - name: Install Rust toolchain + run: | + if [[ "${{ matrix.toolchain }}" == "nightly" ]]; then + rustup toolchain install nightly-2024-09-21-x86_64-unknown-linux-gnu + else + rustup toolchain install 1.81-x86_64-unknown-linux-gnu + rustup component add clippy rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu + fi - name: Install EStarkPolygon prover dependencies run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm - name: Lint no default features run: cargo clippy --all --all-targets --no-default-features --profile pr-tests -- -D warnings - name: Lint all features run: cargo clippy --all --all-targets --all-features --profile pr-tests -- -D warnings + - name: Build + run: | + if [[ "${{ matrix.toolchain }}" == "nightly" ]]; then + cargo +nightly build --all-targets --all --all-features --profile pr-tests --features stwo + else + cargo build --all-targets --all --all-features --profile pr-tests + fi - name: Format run: cargo fmt --all --check --verbose - name: Build diff --git a/rust-toolchain.toml b/rust-toolchain.toml index b6930c7af4..4cef0b738f 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2024-09-21" +channel = "1.81" From 11376e04a3faca69d494a8b9023b84cf0406741b Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 17:14:10 +0200 Subject: [PATCH 05/39] solve clippy pr test issues --- backend/src/stwo/mod.rs | 12 ++++++++---- backend/src/stwo/prover.rs | 25 ++++++++++++++----------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/backend/src/stwo/mod.rs b/backend/src/stwo/mod.rs index 0ed23ed617..11877217c4 100644 --- a/backend/src/stwo/mod.rs +++ b/backend/src/stwo/mod.rs @@ -14,6 +14,8 @@ mod prover; pub(crate) struct StwoProverFactory; impl BackendFactory for StwoProverFactory { + #[allow(unreachable_code)] + #[allow(unused_variables)] fn create( &self, pil: Arc>, @@ -37,11 +39,14 @@ impl BackendFactory for StwoProverFactory { } impl Backend for StwoProver { + + #[allow(unused_variables)] fn verify(&self, proof: &[u8], instances: &[Vec]) -> Result<(), Error> { assert!(instances.len() == 1); Ok(()) } - + #[allow(unreachable_code)] + #[allow(unused_variables)] fn prove( &self, witness: &[(String, Vec)], @@ -51,12 +56,11 @@ impl Backend for StwoProver { if prev_proof.is_some() { return Err(Error::NoAggregationAvailable); } - + let proof = vec![0u8; 10]; panic!("prove function is not implement yet"); - let mut proof = vec![0u8; 10]; Ok(proof) } - + #[allow(unused_variables)] fn export_verification_key(&self, output: &mut dyn io::Write) -> Result<(), Error> { Ok(()) } diff --git a/backend/src/stwo/prover.rs b/backend/src/stwo/prover.rs index 5f6459835c..32c288c9c2 100644 --- a/backend/src/stwo/prover.rs +++ b/backend/src/stwo/prover.rs @@ -1,29 +1,32 @@ use powdr_ast::analyzed::Analyzed; use std::sync::Arc; -use std::{io, option}; +use std::io; use powdr_number::FieldElement; +#[allow(unused_variables)] pub struct StwoProver { - analyzed: Arc>, - fixed: Arc)>>, + _analyzed: Arc>, + _fixed: Arc)>>, /// Proving key placeholder - proving_key: Option<()>, + _proving_key: Option<()>, /// Verifying key placeholder - verifying_key: Option<()>, + _verifying_key: Option<()>, } impl StwoProver { + + #[allow(unused_variables)] pub fn new( - analyzed: Arc>, - fixed: Arc)>>, + _analyzed: Arc>, + _fixed: Arc)>>, setup: Option<&mut dyn io::Read>, ) -> Result { Ok(Self { - analyzed, - fixed, - proving_key: None, - verifying_key: None, + _analyzed, + _fixed, + _proving_key: None, + _verifying_key: None, }) } } From 53db07fdd59fd50cadf4f7eb74ef4b2b8f3fe42d Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 19:00:57 +0200 Subject: [PATCH 06/39] fix pr fail --- backend/src/stwo/mod.rs | 4 +++- backend/src/stwo/prover.rs | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/src/stwo/mod.rs b/backend/src/stwo/mod.rs index 11877217c4..12f53b7a56 100644 --- a/backend/src/stwo/mod.rs +++ b/backend/src/stwo/mod.rs @@ -1,6 +1,7 @@ use std::io; use std::path::PathBuf; use std::sync::Arc; +use log::info; use crate::{Backend, BackendFactory, BackendOptions, Error, Proof}; use powdr_ast::analyzed::Analyzed; @@ -11,6 +12,7 @@ use prover::StwoProver; mod prover; +#[allow(dead_code)] pub(crate) struct StwoProverFactory; impl BackendFactory for StwoProverFactory { @@ -33,7 +35,7 @@ impl BackendFactory for StwoProverFactory { get_uniquely_sized_cloned(&fixed).map_err(|_| Error::NoVariableDegreeAvailable)?, ); let stwo = Box::new(StwoProver::new(pil, fixed, setup)?); - println!("StwoProverFactory is not complete yet"); + info!("StwoProverFactory is not complete yet"); Ok(stwo) } } diff --git a/backend/src/stwo/prover.rs b/backend/src/stwo/prover.rs index 32c288c9c2..335ffff902 100644 --- a/backend/src/stwo/prover.rs +++ b/backend/src/stwo/prover.rs @@ -1,3 +1,4 @@ + use powdr_ast::analyzed::Analyzed; use std::sync::Arc; use std::io; @@ -15,7 +16,7 @@ pub struct StwoProver { } impl StwoProver { - + #[allow(dead_code)] #[allow(unused_variables)] pub fn new( _analyzed: Arc>, From d77371734ece53854d2cb12dc9b108f19b987d98 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 19:29:13 +0200 Subject: [PATCH 07/39] fix nightly conditional build --- .github/workflows/pr-tests.yml | 24 +++++++++++------------- backend/src/stwo/mod.rs | 3 +-- backend/src/stwo/prover.rs | 3 +-- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 523e0709f8..0939a727d0 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -25,6 +25,7 @@ jobs: strategy: matrix: toolchain: [stable, nightly] + features: [default, stwo] steps: - uses: actions/checkout@v4 @@ -41,16 +42,15 @@ jobs: target/ Cargo.lock key: ${{ runner.os }}-cargo-pr-tests - - name: Install Rust toolchain 1.81 (with clippy and rustfmt) - run: rustup toolchain install 1.81-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.81-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu - - name: Install Rust toolchain + - name: Install Rust toolchain and components run: | - if [[ "${{ matrix.toolchain }}" == "nightly" ]]; then - rustup toolchain install nightly-2024-09-21-x86_64-unknown-linux-gnu - else - rustup toolchain install 1.81-x86_64-unknown-linux-gnu - rustup component add clippy rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu - fi + if [[ "${{ matrix.toolchain }}" == "nightly" ]]; then + rustup toolchain install nightly-2024-09-21-x86_64-unknown-linux-gnu + rustup component add clippy rustfmt --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu + else + rustup toolchain install 1.81-x86_64-unknown-linux-gnu + rustup component add clippy rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu + fi - name: Install EStarkPolygon prover dependencies run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm - name: Lint no default features @@ -60,14 +60,12 @@ jobs: - name: Build run: | if [[ "${{ matrix.toolchain }}" == "nightly" ]]; then - cargo +nightly build --all-targets --all --all-features --profile pr-tests --features stwo + cargo +nightly build --all-targets --all --all-features --profile pr-tests --features ${{ matrix.features }} else - cargo build --all-targets --all --all-features --profile pr-tests + cargo build --all-targets --all --all-features --profile pr-tests --features ${{ matrix.features }} fi - name: Format run: cargo fmt --all --check --verbose - - name: Build - run: cargo build --all-targets --all --all-features --profile pr-tests - uses: taiki-e/install-action@nextest - name: Archive EStarkPolygon prover built dependencies run: tar --zstd -cf pil-stark-prover-deps.tar.zst target/pr-tests/build/pil-stark-prover-*/out diff --git a/backend/src/stwo/mod.rs b/backend/src/stwo/mod.rs index 12f53b7a56..70a6152850 100644 --- a/backend/src/stwo/mod.rs +++ b/backend/src/stwo/mod.rs @@ -1,7 +1,7 @@ +use log::info; use std::io; use std::path::PathBuf; use std::sync::Arc; -use log::info; use crate::{Backend, BackendFactory, BackendOptions, Error, Proof}; use powdr_ast::analyzed::Analyzed; @@ -41,7 +41,6 @@ impl BackendFactory for StwoProverFactory { } impl Backend for StwoProver { - #[allow(unused_variables)] fn verify(&self, proof: &[u8], instances: &[Vec]) -> Result<(), Error> { assert!(instances.len() == 1); diff --git a/backend/src/stwo/prover.rs b/backend/src/stwo/prover.rs index 335ffff902..35f66e0ca9 100644 --- a/backend/src/stwo/prover.rs +++ b/backend/src/stwo/prover.rs @@ -1,7 +1,6 @@ - use powdr_ast::analyzed::Analyzed; -use std::sync::Arc; use std::io; +use std::sync::Arc; use powdr_number::FieldElement; From 60573800ffd9c30363d36615cdb9be70052c2282 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 19:43:56 +0200 Subject: [PATCH 08/39] fix stwo conditional build --- .github/workflows/pr-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 0939a727d0..57dfcc34d6 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -59,9 +59,9 @@ jobs: run: cargo clippy --all --all-targets --all-features --profile pr-tests -- -D warnings - name: Build run: | - if [[ "${{ matrix.toolchain }}" == "nightly" ]]; then + if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then cargo +nightly build --all-targets --all --all-features --profile pr-tests --features ${{ matrix.features }} - else + elif [[ "${{ matrix.toolchain }}" == "stable" && "${{ matrix.features }}" == "default" ]]; then cargo build --all-targets --all --all-features --profile pr-tests --features ${{ matrix.features }} fi - name: Format From 3dfd6ab82c22e6a71f4614d9145d4560650c6508 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 20:14:42 +0200 Subject: [PATCH 09/39] remove stwo from stable build --- .github/workflows/pr-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 57dfcc34d6..c846849c93 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -62,7 +62,7 @@ jobs: if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then cargo +nightly build --all-targets --all --all-features --profile pr-tests --features ${{ matrix.features }} elif [[ "${{ matrix.toolchain }}" == "stable" && "${{ matrix.features }}" == "default" ]]; then - cargo build --all-targets --all --all-features --profile pr-tests --features ${{ matrix.features }} + cargo build --all-targets --all --features "halo2,plonky3,estark-polygon" --profile pr-tests --features ${{ matrix.features }} fi - name: Format run: cargo fmt --all --check --verbose From 6a2a3f0d6992e92f41f10dab40b319e1be1dc9d7 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 20:33:10 +0200 Subject: [PATCH 10/39] remove stwo from stable build --- .github/workflows/pr-tests.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index c846849c93..2580a75360 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -25,7 +25,10 @@ jobs: strategy: matrix: toolchain: [stable, nightly] - features: [default, stwo] + features: [default, halo2, plonky3, estark-polygon, stwo] + exclude: + - toolchain: stable + features: stwo steps: - uses: actions/checkout@v4 @@ -60,9 +63,9 @@ jobs: - name: Build run: | if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly build --all-targets --all --all-features --profile pr-tests --features ${{ matrix.features }} - elif [[ "${{ matrix.toolchain }}" == "stable" && "${{ matrix.features }}" == "default" ]]; then - cargo build --all-targets --all --features "halo2,plonky3,estark-polygon" --profile pr-tests --features ${{ matrix.features }} + cargo +nightly build --all-targets --profile pr-tests --features ${{ matrix.features }} + elif [[ "${{ matrix.toolchain }}" == "stable" ]]; then + cargo build --all-targets --profile pr-tests --features ${{ matrix.features }} fi - name: Format run: cargo fmt --all --check --verbose From c43677314cf8dc9a79045fa747956909b57199b6 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 20:42:42 +0200 Subject: [PATCH 11/39] make lint conditional for stwo --- .github/workflows/pr-tests.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 2580a75360..4f39ce17fd 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -58,8 +58,13 @@ jobs: run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm - name: Lint no default features run: cargo clippy --all --all-targets --no-default-features --profile pr-tests -- -D warnings - - name: Lint all features - run: cargo clippy --all --all-targets --all-features --profile pr-tests -- -D warnings + - name: Lint + run: | + if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then + cargo +nightly clippy --all-targets --features stwo --profile pr-tests -- -D warnings + else + cargo clippy --all-targets --features ${{ matrix.features }} --profile pr-tests -- -D warnings + fi - name: Build run: | if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then From 7c3b790ad9cab82f68c06dc217ae22095c4f94d7 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 20:56:04 +0200 Subject: [PATCH 12/39] make sure nigtly toolchain is installed before lint step --- .github/workflows/pr-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 4f39ce17fd..cd63eaf21b 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -54,6 +54,8 @@ jobs: rustup toolchain install 1.81-x86_64-unknown-linux-gnu rustup component add clippy rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu fi + - name: Verify installed Rust toolchains + run: rustup show - name: Install EStarkPolygon prover dependencies run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm - name: Lint no default features From 00c4c6fd058563120dad40f1fa972164a0124307 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 21:00:25 +0200 Subject: [PATCH 13/39] specify nightly version in command --- .github/workflows/pr-tests.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index cd63eaf21b..2330ce4165 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -54,8 +54,7 @@ jobs: rustup toolchain install 1.81-x86_64-unknown-linux-gnu rustup component add clippy rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu fi - - name: Verify installed Rust toolchains - run: rustup show + - name: Install EStarkPolygon prover dependencies run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm - name: Lint no default features @@ -63,14 +62,14 @@ jobs: - name: Lint run: | if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly clippy --all-targets --features stwo --profile pr-tests -- -D warnings + cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu clippy --all-targets --features stwo --profile pr-tests -- -D warnings else cargo clippy --all-targets --features ${{ matrix.features }} --profile pr-tests -- -D warnings fi - name: Build run: | if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly build --all-targets --profile pr-tests --features ${{ matrix.features }} + cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu build --all-targets --profile pr-tests --features ${{ matrix.features }} elif [[ "${{ matrix.toolchain }}" == "stable" ]]; then cargo build --all-targets --profile pr-tests --features ${{ matrix.features }} fi From 84b2c89b9932ff5ce70c123e4ca15a8e607efd39 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 21:17:37 +0200 Subject: [PATCH 14/39] make nextest archive also conditional --- .github/workflows/pr-tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 2330ce4165..876273c0c8 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -79,7 +79,12 @@ jobs: - name: Archive EStarkPolygon prover built dependencies run: tar --zstd -cf pil-stark-prover-deps.tar.zst target/pr-tests/build/pil-stark-prover-*/out - name: Create tests archive - run: cargo nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --all-features + run: | + if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then + cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --features ${{ matrix.features }} + else + cargo nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --features ${{ matrix.features }} + fi - name: Upload build artifacts uses: actions/upload-artifact@v4 with: From 4f8401f05f8f31262938ad76126b8df4d9a66331 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 21:32:49 +0200 Subject: [PATCH 15/39] change to new nightly version --- .github/workflows/pr-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 876273c0c8..635eec1e0d 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -48,8 +48,8 @@ jobs: - name: Install Rust toolchain and components run: | if [[ "${{ matrix.toolchain }}" == "nightly" ]]; then - rustup toolchain install nightly-2024-09-21-x86_64-unknown-linux-gnu - rustup component add clippy rustfmt --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu + rustup toolchain install nightly + rustup component add clippy rustfmt --toolchain nightly else rustup toolchain install 1.81-x86_64-unknown-linux-gnu rustup component add clippy rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu @@ -62,14 +62,14 @@ jobs: - name: Lint run: | if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu clippy --all-targets --features stwo --profile pr-tests -- -D warnings + cargo +nightly clippy --all-targets --features stwo --profile pr-tests -- -D warnings else cargo clippy --all-targets --features ${{ matrix.features }} --profile pr-tests -- -D warnings fi - name: Build run: | if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu build --all-targets --profile pr-tests --features ${{ matrix.features }} + cargo +nightly build --all-targets --profile pr-tests --features ${{ matrix.features }} elif [[ "${{ matrix.toolchain }}" == "stable" ]]; then cargo build --all-targets --profile pr-tests --features ${{ matrix.features }} fi @@ -81,7 +81,7 @@ jobs: - name: Create tests archive run: | if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --features ${{ matrix.features }} + cargo +nightly nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --features ${{ matrix.features }} else cargo nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --features ${{ matrix.features }} fi From ea9e606ba373988032f4a920784e2987de5ad426 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 21:45:45 +0200 Subject: [PATCH 16/39] change back nightly version for stwo to be 9-21, exclude other features to be build by nightly --- .github/workflows/pr-tests.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 635eec1e0d..17e919401f 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -29,6 +29,8 @@ jobs: exclude: - toolchain: stable features: stwo + - toolchain: nightly + features: [default, halo2, plonky3, estark-polygon] steps: - uses: actions/checkout@v4 @@ -48,8 +50,8 @@ jobs: - name: Install Rust toolchain and components run: | if [[ "${{ matrix.toolchain }}" == "nightly" ]]; then - rustup toolchain install nightly - rustup component add clippy rustfmt --toolchain nightly + rustup toolchain install nightly-2024-09-21-x86_64-unknown-linux-gnu + rustup component add clippy rustfmt --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu else rustup toolchain install 1.81-x86_64-unknown-linux-gnu rustup component add clippy rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu @@ -62,14 +64,14 @@ jobs: - name: Lint run: | if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly clippy --all-targets --features stwo --profile pr-tests -- -D warnings + cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu clippy --all-targets --features stwo --profile pr-tests -- -D warnings else cargo clippy --all-targets --features ${{ matrix.features }} --profile pr-tests -- -D warnings fi - name: Build run: | if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly build --all-targets --profile pr-tests --features ${{ matrix.features }} + cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu build --all-targets --profile pr-tests --features ${{ matrix.features }} elif [[ "${{ matrix.toolchain }}" == "stable" ]]; then cargo build --all-targets --profile pr-tests --features ${{ matrix.features }} fi @@ -81,7 +83,7 @@ jobs: - name: Create tests archive run: | if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --features ${{ matrix.features }} + cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --all-features else cargo nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --features ${{ matrix.features }} fi From 06cfcf2788e0b4e290eaaa7e18b75e8b53f92862 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 21:51:27 +0200 Subject: [PATCH 17/39] change back nightly version for stwo to be 9-21, exclude other features to be build by nightly --- .github/workflows/pr-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 17e919401f..e4efd84a57 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -27,10 +27,10 @@ jobs: toolchain: [stable, nightly] features: [default, halo2, plonky3, estark-polygon, stwo] exclude: - - toolchain: stable - features: stwo - toolchain: nightly features: [default, halo2, plonky3, estark-polygon] + - toolchain: stable + features: stwo steps: - uses: actions/checkout@v4 From 8a50b27229214a7898a05f7ef77ab49d131c8030 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 21:56:59 +0200 Subject: [PATCH 18/39] change back nightly version for stwo to be 9-21, exclude other features to be build by nightly --- .github/workflows/pr-tests.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index e4efd84a57..d50eb85bda 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -28,7 +28,13 @@ jobs: features: [default, halo2, plonky3, estark-polygon, stwo] exclude: - toolchain: nightly - features: [default, halo2, plonky3, estark-polygon] + features: default + - toolchain: nightly + features: halo2 + - toolchain: nightly + feature: plonky3 + - toolchain: nightly + feature: estark-polygon - toolchain: stable features: stwo From 7a5d47d915e736dc4f0bd4bcd4d577c976590ebd Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 22:18:14 +0200 Subject: [PATCH 19/39] feature to features --- .github/workflows/pr-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index d50eb85bda..342caf6443 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -32,9 +32,9 @@ jobs: - toolchain: nightly features: halo2 - toolchain: nightly - feature: plonky3 + features: plonky3 - toolchain: nightly - feature: estark-polygon + features: estark-polygon - toolchain: stable features: stwo From 59a584cfd72cfd1fd28872eb6f58290873bbdaad Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 22:42:41 +0200 Subject: [PATCH 20/39] solve clippy issues --- analysis/src/vm/batcher.rs | 2 +- executor/src/constant_evaluator/mod.rs | 1 + riscv/src/continuations/bootloader.rs | 1 + riscv/src/runtime.rs | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/analysis/src/vm/batcher.rs b/analysis/src/vm/batcher.rs index 35a7f917b3..d781e9e1fb 100644 --- a/analysis/src/vm/batcher.rs +++ b/analysis/src/vm/batcher.rs @@ -18,7 +18,7 @@ struct Batch<'a> { } impl<'a> Batch<'a> { - fn from_statement(s: &'a FunctionStatement) -> Batch { + fn from_statement(s: &'a FunctionStatement) -> Batch<'a> { Batch { statements: vec![s], } diff --git a/executor/src/constant_evaluator/mod.rs b/executor/src/constant_evaluator/mod.rs index 3d74f0eb71..cda09e5f57 100644 --- a/executor/src/constant_evaluator/mod.rs +++ b/executor/src/constant_evaluator/mod.rs @@ -7,6 +7,7 @@ mod data_structures; mod interpreter; mod jit_compiler; +#[allow(clippy::too_long_first_doc_paragraph)] /// Generates the fixed column values for all fixed columns that are defined /// (and not just declared). /// @returns the names (in source order) and the values for the columns. diff --git a/riscv/src/continuations/bootloader.rs b/riscv/src/continuations/bootloader.rs index da553fa46e..d87b3528cc 100644 --- a/riscv/src/continuations/bootloader.rs +++ b/riscv/src/continuations/bootloader.rs @@ -130,6 +130,7 @@ pub const REGISTER_NAMES: [&str; 37] = [ /// Index of the PC in the bootloader input. pub const PC_INDEX: usize = REGISTER_NAMES.len() - 1; +#[allow(clippy::too_long_first_doc_paragraph)] /// The default PC that can be used in first chunk, will just continue with whatever comes after the bootloader. /// The value is 3, because we added a jump instruction at the beginning of the code. /// Specifically, the first instructions are: diff --git a/riscv/src/runtime.rs b/riscv/src/runtime.rs index 5048652a26..c7c78d3ac3 100644 --- a/riscv/src/runtime.rs +++ b/riscv/src/runtime.rs @@ -64,6 +64,7 @@ impl SubMachine { } } +#[allow(clippy::too_long_first_doc_paragraph)] /// Sequence of asm function statements. /// Any of the registers used as input/output to the syscall should be usable without issue. /// Other registers should be saved/restored from memory, as LLVM doesn't know about their usage here. From 0b551490295ba8330ed2fd97f72075f7ed9c3027 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 22:56:13 +0200 Subject: [PATCH 21/39] solve clippy issue by adding an empty line below the first line of the comment --- executor/src/constant_evaluator/mod.rs | 3 ++- riscv/src/continuations/bootloader.rs | 3 ++- riscv/src/runtime.rs | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/executor/src/constant_evaluator/mod.rs b/executor/src/constant_evaluator/mod.rs index cda09e5f57..b2dbc268bd 100644 --- a/executor/src/constant_evaluator/mod.rs +++ b/executor/src/constant_evaluator/mod.rs @@ -7,8 +7,9 @@ mod data_structures; mod interpreter; mod jit_compiler; -#[allow(clippy::too_long_first_doc_paragraph)] + /// Generates the fixed column values for all fixed columns that are defined +/// /// (and not just declared). /// @returns the names (in source order) and the values for the columns. /// Arrays of columns are flattened, the name of the `i`th array element diff --git a/riscv/src/continuations/bootloader.rs b/riscv/src/continuations/bootloader.rs index d87b3528cc..c5d519ee3f 100644 --- a/riscv/src/continuations/bootloader.rs +++ b/riscv/src/continuations/bootloader.rs @@ -130,8 +130,9 @@ pub const REGISTER_NAMES: [&str; 37] = [ /// Index of the PC in the bootloader input. pub const PC_INDEX: usize = REGISTER_NAMES.len() - 1; -#[allow(clippy::too_long_first_doc_paragraph)] + /// The default PC that can be used in first chunk, will just continue with whatever comes after the bootloader. +/// /// The value is 3, because we added a jump instruction at the beginning of the code. /// Specifically, the first instructions are: /// 0: reset diff --git a/riscv/src/runtime.rs b/riscv/src/runtime.rs index c7c78d3ac3..fea053d70b 100644 --- a/riscv/src/runtime.rs +++ b/riscv/src/runtime.rs @@ -64,8 +64,9 @@ impl SubMachine { } } -#[allow(clippy::too_long_first_doc_paragraph)] + /// Sequence of asm function statements. +/// /// Any of the registers used as input/output to the syscall should be usable without issue. /// Other registers should be saved/restored from memory, as LLVM doesn't know about their usage here. #[derive(Clone)] From 985d717cd9692bc98fab4144860f922a7638276d Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 23:27:38 +0200 Subject: [PATCH 22/39] fix format --- executor/src/constant_evaluator/mod.rs | 11 +++++------ riscv/src/continuations/bootloader.rs | 3 +-- riscv/src/runtime.rs | 3 +-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/executor/src/constant_evaluator/mod.rs b/executor/src/constant_evaluator/mod.rs index b2dbc268bd..26bb289db7 100644 --- a/executor/src/constant_evaluator/mod.rs +++ b/executor/src/constant_evaluator/mod.rs @@ -7,13 +7,12 @@ mod data_structures; mod interpreter; mod jit_compiler; - -/// Generates the fixed column values for all fixed columns that are defined -/// -/// (and not just declared). +/// Generates the fixed column values for all defined columns. +/// +/// It includes all columns that are defined, not just declared. +/// /// @returns the names (in source order) and the values for the columns. -/// Arrays of columns are flattened, the name of the `i`th array element -/// is `name[i]`. +/// Arrays of columns are flattened. The name of the `i`th array element is `name[i]`. pub fn generate(analyzed: &Analyzed) -> Vec<(String, VariablySizedColumn)> { let max_degree = analyzed .constant_polys_in_source_order() diff --git a/riscv/src/continuations/bootloader.rs b/riscv/src/continuations/bootloader.rs index c5d519ee3f..4c8330ee54 100644 --- a/riscv/src/continuations/bootloader.rs +++ b/riscv/src/continuations/bootloader.rs @@ -130,9 +130,8 @@ pub const REGISTER_NAMES: [&str; 37] = [ /// Index of the PC in the bootloader input. pub const PC_INDEX: usize = REGISTER_NAMES.len() - 1; - /// The default PC that can be used in first chunk, will just continue with whatever comes after the bootloader. -/// +/// /// The value is 3, because we added a jump instruction at the beginning of the code. /// Specifically, the first instructions are: /// 0: reset diff --git a/riscv/src/runtime.rs b/riscv/src/runtime.rs index fea053d70b..a07990e5f3 100644 --- a/riscv/src/runtime.rs +++ b/riscv/src/runtime.rs @@ -64,9 +64,8 @@ impl SubMachine { } } - /// Sequence of asm function statements. -/// +/// /// Any of the registers used as input/output to the syscall should be usable without issue. /// Other registers should be saved/restored from memory, as LLVM doesn't know about their usage here. #[derive(Clone)] From 2e02b913e1a0f930c7b5e4174be551e3c385fcb4 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Mon, 14 Oct 2024 23:51:41 +0200 Subject: [PATCH 23/39] fix upload artifact confilic --- .github/workflows/pr-tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 342caf6443..5f520d222f 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -172,9 +172,12 @@ jobs: with: submodules: recursive - name: Download build artifacts - uses: actions/download-artifact@v4 + uses: actions/upload-artifact@v4 with: - name: tests_archive + name: tests_archive-${{ matrix.toolchain }}-${{ matrix.features }} + path: | + tests.tar.zst + pil-stark-prover-deps.tar.zst - name: ⚡ Cache nodejs uses: actions/cache@v4 with: From 0b38415694b98126345451d2f28a0ae8cf14aa2a Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Tue, 15 Oct 2024 00:15:53 +0200 Subject: [PATCH 24/39] fix upload artifact confilic --- .github/workflows/pr-tests.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 5f520d222f..5ad39260a1 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -96,10 +96,10 @@ jobs: - name: Upload build artifacts uses: actions/upload-artifact@v4 with: - name: tests_archive - path: | - tests.tar.zst - pil-stark-prover-deps.tar.zst + name: tests_archive-${{ matrix.toolchain }}-${{ matrix.features }} + path: | + tests.tar.zst + pil-stark-prover-deps.tar.zst test_quick: needs: build @@ -172,12 +172,9 @@ jobs: with: submodules: recursive - name: Download build artifacts - uses: actions/upload-artifact@v4 + uses: actions/download-artifact@v4 with: - name: tests_archive-${{ matrix.toolchain }}-${{ matrix.features }} - path: | - tests.tar.zst - pil-stark-prover-deps.tar.zst + name: tests_archive - name: ⚡ Cache nodejs uses: actions/cache@v4 with: From 1602dc83ef2ae6c902ca08e046946bbead5563b4 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Tue, 15 Oct 2024 01:02:25 +0200 Subject: [PATCH 25/39] fix download artifact --- .github/workflows/pr-tests.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 5ad39260a1..2870819512 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -117,7 +117,12 @@ jobs: - name: Download build artifacts uses: actions/download-artifact@v4 with: - name: tests_archive + name: | + tests_archive-stable-default + tests_archive-stable-halo2 + tests_archive-stable-plonky3 + tests_archive-stable-estark-polygon + tests_archive-nightly-stwo - name: ⚡ Cache nodejs uses: actions/cache@v4 with: @@ -174,7 +179,12 @@ jobs: - name: Download build artifacts uses: actions/download-artifact@v4 with: - name: tests_archive + name: | + tests_archive-stable-default + tests_archive-stable-halo2 + tests_archive-stable-plonky3 + tests_archive-stable-estark-polygon + tests_archive-nightly-stwo - name: ⚡ Cache nodejs uses: actions/cache@v4 with: @@ -224,7 +234,12 @@ jobs: - name: Download build artifacts uses: actions/download-artifact@v4 with: - name: tests_archive + name: | + tests_archive-stable-default + tests_archive-stable-halo2 + tests_archive-stable-plonky3 + tests_archive-stable-estark-polygon + tests_archive-nightly-stwo - name: ⚡ Cache nodejs uses: actions/cache@v4 with: From e1d613c7c29fc73542834be7da8c94e6b56ec730 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Tue, 15 Oct 2024 01:39:36 +0200 Subject: [PATCH 26/39] fix download artifact --- .github/workflows/pr-tests.yml | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 2870819512..e916e4433a 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -117,12 +117,7 @@ jobs: - name: Download build artifacts uses: actions/download-artifact@v4 with: - name: | - tests_archive-stable-default - tests_archive-stable-halo2 - tests_archive-stable-plonky3 - tests_archive-stable-estark-polygon - tests_archive-nightly-stwo + name: tests_archive-${{ matrix.toolchain }}-${{ matrix.features }} - name: ⚡ Cache nodejs uses: actions/cache@v4 with: @@ -179,12 +174,7 @@ jobs: - name: Download build artifacts uses: actions/download-artifact@v4 with: - name: | - tests_archive-stable-default - tests_archive-stable-halo2 - tests_archive-stable-plonky3 - tests_archive-stable-estark-polygon - tests_archive-nightly-stwo + name: tests_archive-${{ matrix.toolchain }}-${{ matrix.features }} - name: ⚡ Cache nodejs uses: actions/cache@v4 with: @@ -234,12 +224,7 @@ jobs: - name: Download build artifacts uses: actions/download-artifact@v4 with: - name: | - tests_archive-stable-default - tests_archive-stable-halo2 - tests_archive-stable-plonky3 - tests_archive-stable-estark-polygon - tests_archive-nightly-stwo + name: tests_archive-${{ matrix.toolchain }}-${{ matrix.features }} - name: ⚡ Cache nodejs uses: actions/cache@v4 with: From df6de4a6f2329d2c319cf726dee02f96f4cbc388 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Tue, 15 Oct 2024 02:08:57 +0200 Subject: [PATCH 27/39] fix download artifact --- .github/workflows/pr-tests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index e916e4433a..cdfd539cd2 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -116,8 +116,6 @@ jobs: submodules: recursive - name: Download build artifacts uses: actions/download-artifact@v4 - with: - name: tests_archive-${{ matrix.toolchain }}-${{ matrix.features }} - name: ⚡ Cache nodejs uses: actions/cache@v4 with: @@ -173,8 +171,6 @@ jobs: submodules: recursive - name: Download build artifacts uses: actions/download-artifact@v4 - with: - name: tests_archive-${{ matrix.toolchain }}-${{ matrix.features }} - name: ⚡ Cache nodejs uses: actions/cache@v4 with: @@ -223,8 +219,6 @@ jobs: submodules: recursive - name: Download build artifacts uses: actions/download-artifact@v4 - with: - name: tests_archive-${{ matrix.toolchain }}-${{ matrix.features }} - name: ⚡ Cache nodejs uses: actions/cache@v4 with: From 8cead5cbb28d59c2385e5dfe693c3f9fa8a27b88 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Tue, 15 Oct 2024 09:09:35 +0200 Subject: [PATCH 28/39] fix download artifact --- .github/workflows/pr-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index cdfd539cd2..394d58114a 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -85,6 +85,7 @@ jobs: run: cargo fmt --all --check --verbose - uses: taiki-e/install-action@nextest - name: Archive EStarkPolygon prover built dependencies + if: ${{ matrix.toolchain == 'stable' && matrix.features == 'estark-polygon' }} run: tar --zstd -cf pil-stark-prover-deps.tar.zst target/pr-tests/build/pil-stark-prover-*/out - name: Create tests archive run: | From 59275b6e165c5b45eff615720475bf38a1775e62 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Tue, 15 Oct 2024 10:14:13 +0200 Subject: [PATCH 29/39] fix unpack issue --- .github/workflows/pr-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 394d58114a..3f2e9fb823 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -192,7 +192,7 @@ jobs: run: sudo apt-get install -y nlohmann-json3-dev - uses: taiki-e/install-action@nextest - name: Unpack EStarkPolygon built dependencies - run: tar --zstd -xf pil-stark-prover-deps.tar.zst + run: tar --zstd -xf tests_archive-stable-estark-polygon/pil-stark-prover-deps.tar.zst - name: Run EStark Polygon test run: cargo nextest run --archive-file tests.tar.zst --workspace-remap . --verbose --run-ignored=ignored-only --no-capture -E "test(=vec_median_estark_polygon)" env: From 372045c4e2306ede9fd6091f2c707e49db615555 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Tue, 15 Oct 2024 11:20:41 +0200 Subject: [PATCH 30/39] solve create test but cannot find file issue --- .github/workflows/pr-tests.yml | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 3f2e9fb823..b38c436da3 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -90,7 +90,7 @@ jobs: - name: Create tests archive run: | if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --all-features + cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --features ${{ matrix.features }} else cargo nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --features ${{ matrix.features }} fi @@ -107,6 +107,19 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + toolchain: [stable, nightly] + features: [default, halo2, plonky3, estark-polygon, stwo] + exclude: + - toolchain: nightly + features: default + - toolchain: nightly + features: halo2 + - toolchain: nightly + features: plonky3 + - toolchain: nightly + features: estark-polygon + - toolchain: stable + features: stwo test: - "1" - "2" @@ -135,7 +148,7 @@ jobs: run: git clone https://github.com/0xPolygonHermez/pilcom.git && cd pilcom && npm install - uses: taiki-e/install-action@nextest - name: Run default tests - run: cargo nextest run --archive-file tests.tar.zst --workspace-remap . --verbose --partition count:"${{ matrix.test }}"/2 + run: cargo nextest run --archive-file tests_archive-${{ matrix.toolchain }}-${{ matrix.features }}/tests.tar.zst --workspace-remap . --verbose --partition count:"${{ matrix.test }}"/2 env: PILCOM: ${{ github.workspace }}/pilcom/ POWDR_STD: ${{ github.workspace }}/std/ @@ -194,7 +207,7 @@ jobs: - name: Unpack EStarkPolygon built dependencies run: tar --zstd -xf tests_archive-stable-estark-polygon/pil-stark-prover-deps.tar.zst - name: Run EStark Polygon test - run: cargo nextest run --archive-file tests.tar.zst --workspace-remap . --verbose --run-ignored=ignored-only --no-capture -E "test(=vec_median_estark_polygon)" + run: cargo nextest run --archive-file tests_archive-stable-estark-polygon/tests.tar.zst --workspace-remap . --verbose --run-ignored=ignored-only --no-capture -E "test(=vec_median_estark_polygon)" env: PILCOM: ${{ github.workspace }}/pilcom/ POWDR_STD: ${{ github.workspace }}/std/ @@ -202,6 +215,19 @@ jobs: test_slow: strategy: matrix: + toolchain: [stable, nightly] + features: [default, halo2, plonky3, estark-polygon, stwo] + exclude: + - toolchain: nightly + features: default + - toolchain: nightly + features: halo2 + - toolchain: nightly + features: plonky3 + - toolchain: nightly + features: estark-polygon + - toolchain: stable + features: stwo test: - "1" - "2" @@ -243,7 +269,7 @@ jobs: # Number threads is set to 2 because the runner does not have enough memory for more. run: | NIGHTLY_TESTS=$(cat .github/workflows/nightly_tests_list.txt) - cargo nextest run --archive-file tests.tar.zst --workspace-remap . --verbose --run-ignored=ignored-only -E "!($NIGHTLY_TESTS)" --test-threads 2 --partition hash:"${{ matrix.test }}"/8 + cargo nextest run --archive-file tests_archive-${{ matrix.toolchain }}-${{ matrix.features }}/tests.tar.zst --workspace-remap . --verbose --run-ignored=ignored-only -E "!($NIGHTLY_TESTS)" --test-threads 2 --partition hash:"${{ matrix.test }}"/8 shell: bash env: PILCOM: ${{ github.workspace }}/pilcom/ From 9dac7dd2da0b694b1dc98d7ba7b5cb6e8ba02490 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Thu, 17 Oct 2024 08:47:42 +0200 Subject: [PATCH 31/39] update CI file --- .github/workflows/pr-tests.yml | 131 ++++++++++----------------------- backend/Cargo.toml | 2 +- 2 files changed, 40 insertions(+), 93 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index b38c436da3..b9a74538b5 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -22,22 +22,6 @@ jobs: build: runs-on: warp-ubuntu-latest-x64-4x - strategy: - matrix: - toolchain: [stable, nightly] - features: [default, halo2, plonky3, estark-polygon, stwo] - exclude: - - toolchain: nightly - features: default - - toolchain: nightly - features: halo2 - - toolchain: nightly - features: plonky3 - - toolchain: nightly - features: estark-polygon - - toolchain: stable - features: stwo - steps: - uses: actions/checkout@v4 with: @@ -53,73 +37,43 @@ jobs: target/ Cargo.lock key: ${{ runner.os }}-cargo-pr-tests - - name: Install Rust toolchain and components - run: | - if [[ "${{ matrix.toolchain }}" == "nightly" ]]; then - rustup toolchain install nightly-2024-09-21-x86_64-unknown-linux-gnu - rustup component add clippy rustfmt --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu - else - rustup toolchain install 1.81-x86_64-unknown-linux-gnu - rustup component add clippy rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu - fi - + - name: Date of the restored cache + run: cat target/cache-build-date.txt + continue-on-error: true + - name: Install Rust toolchain nightly-2024-09-21 (with clippy and rustfmt) + run: rustup toolchain install nightly-2024-09-21-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu - name: Install EStarkPolygon prover dependencies run: sudo apt-get install -y nlohmann-json3-dev libpqxx-dev nasm + - name: Install Rust toolchain 1.81 (stable) + run: rustup toolchain install 1.81.0-x86_64-unknown-linux-gnu + - name: Cargo check with Rust 1.81 (default features) + run: rustup default 1.81.0-x86_64-unknown-linux-gnu && cargo +1.81.0-x86_64-unknown-linux-gnu check --all-targets --no-default-features - name: Lint no default features run: cargo clippy --all --all-targets --no-default-features --profile pr-tests -- -D warnings - - name: Lint - run: | - if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu clippy --all-targets --features stwo --profile pr-tests -- -D warnings - else - cargo clippy --all-targets --features ${{ matrix.features }} --profile pr-tests -- -D warnings - fi - - name: Build - run: | - if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu build --all-targets --profile pr-tests --features ${{ matrix.features }} - elif [[ "${{ matrix.toolchain }}" == "stable" ]]; then - cargo build --all-targets --profile pr-tests --features ${{ matrix.features }} - fi + - name: Lint all features + run: cargo clippy --all --all-targets --all-features --profile pr-tests -- -D warnings - name: Format run: cargo fmt --all --check --verbose + - name: Build + run: cargo build --all-targets --all --all-features --profile pr-tests - uses: taiki-e/install-action@nextest - name: Archive EStarkPolygon prover built dependencies - if: ${{ matrix.toolchain == 'stable' && matrix.features == 'estark-polygon' }} run: tar --zstd -cf pil-stark-prover-deps.tar.zst target/pr-tests/build/pil-stark-prover-*/out - name: Create tests archive - run: | - if [[ "${{ matrix.toolchain }}" == "nightly" && "${{ matrix.features }}" == "stwo" ]]; then - cargo +nightly-2024-09-21-x86_64-unknown-linux-gnu nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --features ${{ matrix.features }} - else - cargo nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --features ${{ matrix.features }} - fi + run: cargo nextest archive --archive-file tests.tar.zst --cargo-profile pr-tests --workspace --all-features - name: Upload build artifacts uses: actions/upload-artifact@v4 with: - name: tests_archive-${{ matrix.toolchain }}-${{ matrix.features }} - path: | - tests.tar.zst - pil-stark-prover-deps.tar.zst + name: tests_archive + path: | + tests.tar.zst + pil-stark-prover-deps.tar.zst test_quick: needs: build runs-on: ubuntu-latest strategy: matrix: - toolchain: [stable, nightly] - features: [default, halo2, plonky3, estark-polygon, stwo] - exclude: - - toolchain: nightly - features: default - - toolchain: nightly - features: halo2 - - toolchain: nightly - features: plonky3 - - toolchain: nightly - features: estark-polygon - - toolchain: stable - features: stwo test: - "1" - "2" @@ -130,15 +84,17 @@ jobs: submodules: recursive - name: Download build artifacts uses: actions/download-artifact@v4 + with: + name: tests_archive - name: ⚡ Cache nodejs uses: actions/cache@v4 with: path: | ~/pilcom/node_modules key: ${{ runner.os }}-pilcom-node-modules - - name: Install Rust toolchain 1.81 (with clippy and rustfmt) - run: rustup toolchain install 1.81-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.81-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu - - name: Install nightly + - name: Install Rust toolchain nightly-2024-09-21 (with clippy and rustfmt) + run: rustup toolchain install nightly-2024-09-21-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu + - name: Install nightly-2024-08-01 run: rustup toolchain install nightly-2024-08-01-x86_64-unknown-linux-gnu - name: Install std source run: rustup component add rust-src --toolchain nightly-2024-08-01-x86_64-unknown-linux-gnu @@ -148,7 +104,7 @@ jobs: run: git clone https://github.com/0xPolygonHermez/pilcom.git && cd pilcom && npm install - uses: taiki-e/install-action@nextest - name: Run default tests - run: cargo nextest run --archive-file tests_archive-${{ matrix.toolchain }}-${{ matrix.features }}/tests.tar.zst --workspace-remap . --verbose --partition count:"${{ matrix.test }}"/2 + run: cargo nextest run --archive-file tests.tar.zst --workspace-remap . --verbose --partition count:"${{ matrix.test }}"/2 env: PILCOM: ${{ github.workspace }}/pilcom/ POWDR_STD: ${{ github.workspace }}/std/ @@ -171,8 +127,8 @@ jobs: target/ Cargo.lock key: ${{ runner.os }}-cargo-pr-tests - - name: Install Rust toolchain 1.81 (with clippy and rustfmt) - run: rustup toolchain install 1.81-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.81-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu + - name: Install Rust toolchain nightly-2024-09-21 (with clippy and rustfmt) + run: rustup toolchain install nightly-2024-09-21-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu - name: Run examples run: cargo run --example hello_world && cargo run --example sqrt_with_publics @@ -185,15 +141,17 @@ jobs: submodules: recursive - name: Download build artifacts uses: actions/download-artifact@v4 + with: + name: tests_archive - name: ⚡ Cache nodejs uses: actions/cache@v4 with: path: | ~/pilcom/node_modules key: ${{ runner.os }}-pilcom-node-modules - - name: Install Rust toolchain 1.81 (with clippy and rustfmt) - run: rustup toolchain install 1.81-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.81-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu - - name: Install nightly + - name: Install Rust toolchain nightly-2024-09-21(with clippy and rustfmt) + run: rustup toolchain install nightly-2024-09-21-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu + - name: Install nightly-2024-08-01 run: rustup toolchain install nightly-2024-08-01-x86_64-unknown-linux-gnu - name: Install std source run: rustup component add rust-src --toolchain nightly-2024-08-01-x86_64-unknown-linux-gnu @@ -205,9 +163,9 @@ jobs: run: sudo apt-get install -y nlohmann-json3-dev - uses: taiki-e/install-action@nextest - name: Unpack EStarkPolygon built dependencies - run: tar --zstd -xf tests_archive-stable-estark-polygon/pil-stark-prover-deps.tar.zst + run: tar --zstd -xf pil-stark-prover-deps.tar.zst - name: Run EStark Polygon test - run: cargo nextest run --archive-file tests_archive-stable-estark-polygon/tests.tar.zst --workspace-remap . --verbose --run-ignored=ignored-only --no-capture -E "test(=vec_median_estark_polygon)" + run: cargo nextest run --archive-file tests.tar.zst --workspace-remap . --verbose --run-ignored=ignored-only --no-capture -E "test(=vec_median_estark_polygon)" env: PILCOM: ${{ github.workspace }}/pilcom/ POWDR_STD: ${{ github.workspace }}/std/ @@ -215,19 +173,6 @@ jobs: test_slow: strategy: matrix: - toolchain: [stable, nightly] - features: [default, halo2, plonky3, estark-polygon, stwo] - exclude: - - toolchain: nightly - features: default - - toolchain: nightly - features: halo2 - - toolchain: nightly - features: plonky3 - - toolchain: nightly - features: estark-polygon - - toolchain: stable - features: stwo test: - "1" - "2" @@ -246,17 +191,19 @@ jobs: submodules: recursive - name: Download build artifacts uses: actions/download-artifact@v4 + with: + name: tests_archive - name: ⚡ Cache nodejs uses: actions/cache@v4 with: path: | ~/pilcom/node_modules key: ${{ runner.os }}-pilcom-node-modules - - name: Install Rust toolchain 1.81 (with clippy and rustfmt) - run: rustup toolchain install 1.81-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain 1.81-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain 1.81-x86_64-unknown-linux-gnu + - name: Install Rust toolchain nightly-2024-09-21 (with clippy and rustfmt) + run: rustup toolchain install nightly-2024-09-21-x86_64-unknown-linux-gnu && rustup component add clippy --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu && rustup component add rustfmt --toolchain nightly-2024-09-21-x86_64-unknown-linux-gnu - name: Install test dependencies run: sudo apt-get install -y binutils-riscv64-unknown-elf lld - - name: Install nightly + - name: Install nightly-2024-08-01 run: rustup toolchain install nightly-2024-08-01-x86_64-unknown-linux-gnu - name: Install std source run: rustup component add rust-src --toolchain nightly-2024-08-01-x86_64-unknown-linux-gnu @@ -269,7 +216,7 @@ jobs: # Number threads is set to 2 because the runner does not have enough memory for more. run: | NIGHTLY_TESTS=$(cat .github/workflows/nightly_tests_list.txt) - cargo nextest run --archive-file tests_archive-${{ matrix.toolchain }}-${{ matrix.features }}/tests.tar.zst --workspace-remap . --verbose --run-ignored=ignored-only -E "!($NIGHTLY_TESTS)" --test-threads 2 --partition hash:"${{ matrix.test }}"/8 + cargo nextest run --archive-file tests.tar.zst --workspace-remap . --verbose --run-ignored=ignored-only -E "!($NIGHTLY_TESTS)" --test-threads 2 --partition hash:"${{ matrix.test }}"/8 shell: bash env: PILCOM: ${{ github.workspace }}/pilcom/ diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7b339556cd..c786852a52 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -37,7 +37,7 @@ snark-verifier = { git = "https://github.com/powdr-labs/snark-verifier", branch halo2_solidity_verifier = { git = "https://github.com/powdr-labs/halo2-solidity-verifier", branch = "main", features = ["evm"], optional = true } #change this to main branch when updatetoolchain branch is merged,the main branch is using "nightly-2024-01-04", not compatiable with plonky3 -stwo-prover = { git= "https://github.com/starkware-libs/stwo.git",optional=true, branch = "andrew/dev/update-toolchain" } +stwo-prover = { git= "https://github.com/starkware-libs/stwo.git",optional=true, rev="52d050c18b5dbc74af40214b3b441a6f60a20d41" } strum = { version = "0.24.1", features = ["derive"] } log = "0.4.17" From f925601325a81d72ef531d27d572cc0a6de415b2 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Thu, 17 Oct 2024 08:56:39 +0200 Subject: [PATCH 32/39] fix comment --- executor/src/constant_evaluator/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/executor/src/constant_evaluator/mod.rs b/executor/src/constant_evaluator/mod.rs index 63b76d584c..2d15378765 100644 --- a/executor/src/constant_evaluator/mod.rs +++ b/executor/src/constant_evaluator/mod.rs @@ -10,7 +10,8 @@ mod jit_compiler; /// Generates the fixed column values for all fixed columns that are defined (and not just declared). /// /// @returns the names (in source order) and the values for the columns. -/// Arrays of columns are flattened. The name of the `i`th array element is `name[i]`. +/// Arrays of columns are flattened, the name of the `i`th array element +/// is `name[i]`. pub fn generate(analyzed: &Analyzed) -> Vec<(String, VariablySizedColumn)> { let max_degree = analyzed .constant_polys_in_source_order() From d1fc82bd3ece018dd976c94180b840a86cbe4a14 Mon Sep 17 00:00:00 2001 From: ShuangWu121 <47602565+ShuangWu121@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:40:31 +0200 Subject: [PATCH 33/39] Update backend/src/stwo/mod.rs Co-authored-by: Thibaut Schaeffer --- backend/src/stwo/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/src/stwo/mod.rs b/backend/src/stwo/mod.rs index 70a6152850..60dc114f68 100644 --- a/backend/src/stwo/mod.rs +++ b/backend/src/stwo/mod.rs @@ -57,9 +57,7 @@ impl Backend for StwoProver { if prev_proof.is_some() { return Err(Error::NoAggregationAvailable); } - let proof = vec![0u8; 10]; - panic!("prove function is not implement yet"); - Ok(proof) + unimplemented!() } #[allow(unused_variables)] fn export_verification_key(&self, output: &mut dyn io::Write) -> Result<(), Error> { From d508fed6a4cc077d4c6dc6f4d47bea92d357a474 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Thu, 17 Oct 2024 15:44:52 +0200 Subject: [PATCH 34/39] add unimplemented --- backend/src/stwo/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/stwo/mod.rs b/backend/src/stwo/mod.rs index 60dc114f68..86494df26c 100644 --- a/backend/src/stwo/mod.rs +++ b/backend/src/stwo/mod.rs @@ -61,6 +61,6 @@ impl Backend for StwoProver { } #[allow(unused_variables)] fn export_verification_key(&self, output: &mut dyn io::Write) -> Result<(), Error> { - Ok(()) + unimplemented!() } } From 4dceadbb4d9a831b63e96c0d0f512604b5f24407 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Thu, 17 Oct 2024 16:31:03 +0200 Subject: [PATCH 35/39] fix format --- backend/src/stwo/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/stwo/mod.rs b/backend/src/stwo/mod.rs index 86494df26c..6f037debb5 100644 --- a/backend/src/stwo/mod.rs +++ b/backend/src/stwo/mod.rs @@ -57,7 +57,7 @@ impl Backend for StwoProver { if prev_proof.is_some() { return Err(Error::NoAggregationAvailable); } - unimplemented!() + unimplemented!() } #[allow(unused_variables)] fn export_verification_key(&self, output: &mut dyn io::Write) -> Result<(), Error> { From 873865c237e7be6a220c323edd32a5a8136b96d5 Mon Sep 17 00:00:00 2001 From: ShuangWu121 <47602565+ShuangWu121@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:01:57 +0200 Subject: [PATCH 36/39] Update backend/Cargo.toml Co-authored-by: Georg Wiese --- backend/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 31c947e030..d8f3d1159c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -37,7 +37,7 @@ halo2_curves = { version = "0.6.1", package = "halo2curves", optional = true} snark-verifier = { git = "https://github.com/powdr-labs/snark-verifier", branch = "powdr-halo2", optional = true } halo2_solidity_verifier = { git = "https://github.com/powdr-labs/halo2-solidity-verifier", branch = "main", features = ["evm"], optional = true } -#change this to main branch when updatetoolchain branch is merged,the main branch is using "nightly-2024-01-04", not compatiable with plonky3 +# TODO: Change this to main branch when the `andrew/dev/update-toolchain` branch is merged,the main branch is using "nightly-2024-01-04", not compatiable with plonky3 stwo-prover = { git= "https://github.com/starkware-libs/stwo.git",optional=true, rev="52d050c18b5dbc74af40214b3b441a6f60a20d41" } strum = { version = "0.24.1", features = ["derive"] } From b34abb5276222f0cbc08239dda887daa7dc0b6fa Mon Sep 17 00:00:00 2001 From: ShuangWu121 <47602565+ShuangWu121@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:02:14 +0200 Subject: [PATCH 37/39] Update backend/src/stwo/mod.rs Co-authored-by: Georg Wiese --- backend/src/stwo/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/stwo/mod.rs b/backend/src/stwo/mod.rs index 6f037debb5..92f2438222 100644 --- a/backend/src/stwo/mod.rs +++ b/backend/src/stwo/mod.rs @@ -35,7 +35,6 @@ impl BackendFactory for StwoProverFactory { get_uniquely_sized_cloned(&fixed).map_err(|_| Error::NoVariableDegreeAvailable)?, ); let stwo = Box::new(StwoProver::new(pil, fixed, setup)?); - info!("StwoProverFactory is not complete yet"); Ok(stwo) } } From b6ea465b78d3b3bf7c476bc696da3338c2ed8121 Mon Sep 17 00:00:00 2001 From: ShuangWu121 <47602565+ShuangWu121@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:02:26 +0200 Subject: [PATCH 38/39] Update backend/src/stwo/mod.rs Co-authored-by: Georg Wiese --- backend/src/stwo/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/stwo/mod.rs b/backend/src/stwo/mod.rs index 92f2438222..52c15a9960 100644 --- a/backend/src/stwo/mod.rs +++ b/backend/src/stwo/mod.rs @@ -43,7 +43,7 @@ impl Backend for StwoProver { #[allow(unused_variables)] fn verify(&self, proof: &[u8], instances: &[Vec]) -> Result<(), Error> { assert!(instances.len() == 1); - Ok(()) + unimplemented!() } #[allow(unreachable_code)] #[allow(unused_variables)] From 29aa6193b0fe3621d6796bd52ad7a1e99bb0ff34 Mon Sep 17 00:00:00 2001 From: ShuangWu121 Date: Fri, 18 Oct 2024 11:33:33 +0200 Subject: [PATCH 39/39] remove not used import --- backend/src/stwo/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/stwo/mod.rs b/backend/src/stwo/mod.rs index 6f037debb5..03307a7355 100644 --- a/backend/src/stwo/mod.rs +++ b/backend/src/stwo/mod.rs @@ -1,4 +1,3 @@ -use log::info; use std::io; use std::path::PathBuf; use std::sync::Arc;