From 61704871971d0d55186dc2301a7e4877894c45da Mon Sep 17 00:00:00 2001 From: mattstam Date: Thu, 7 Nov 2024 18:43:52 -0800 Subject: [PATCH] retry both --- crates/sdk/src/network-v2/prover.rs | 196 ++++++++++++++++------------ 1 file changed, 109 insertions(+), 87 deletions(-) diff --git a/crates/sdk/src/network-v2/prover.rs b/crates/sdk/src/network-v2/prover.rs index 3f1ebdef26..ded3b0ae33 100644 --- a/crates/sdk/src/network-v2/prover.rs +++ b/crates/sdk/src/network-v2/prover.rs @@ -11,7 +11,7 @@ use crate::{ Prover, SP1Context, SP1ProofKind, SP1ProofWithPublicValues, SP1ProvingKey, SP1VerifyingKey, }; use anyhow::Result; -use backoff::{future::retry, ExponentialBackoff}; +use backoff::{future::retry, Error as BackoffError, ExponentialBackoff}; use serde::de::DeserializeOwned; use sp1_core_machine::io::SP1Stdin; use sp1_prover::{components::DefaultProverComponents, SP1Prover, SP1_CIRCUIT_VERSION}; @@ -80,20 +80,26 @@ impl NetworkProver { log::info!("Requesting proof with cycle limit: {}", cycle_limit); - // Request the proof. - let response = self - .client - .request_proof( - elf, - &stdin, - &vk, - mode, - SP1_CIRCUIT_VERSION, - FulfillmentStrategy::Hosted, - timeout_secs, - cycle_limit, - ) - .await?; + // Request the proof with retries. + let response = with_retry( + || async { + self.client + .request_proof( + elf, + &stdin, + &vk, + mode, + SP1_CIRCUIT_VERSION, + FulfillmentStrategy::Hosted, + timeout_secs, + cycle_limit, + ) + .await + }, + timeout, + "request proof", + ) + .await?; // Log the request ID and transaction hash. let tx_hash_hex = "0x".to_string() + &hex::encode(response.tx_hash); @@ -102,7 +108,7 @@ impl NetworkProver { log::info!("Created request {} in transaction {}", request_id_hex, tx_hash_hex); if NetworkClient::rpc_url() == DEFAULT_PROVER_NETWORK_RPC { - log::info!("View in explorer: https://explorer-v2.succinct.xyz/{}", request_id_hex); + log::info!("View in explorer: https://network.succinct.xyz/request/{}", request_id_hex); } Ok(request_id) @@ -118,89 +124,50 @@ impl NetworkProver { let mut is_assigned = false; let start_time = Instant::now(); - // Configure retries with exponential backoff. - let backoff = ExponentialBackoff { - initial_interval: Duration::from_secs(1), - max_interval: Duration::from_secs(30), - max_elapsed_time: timeout, - ..Default::default() - }; - loop { + // Calculate the remaining timeout. if let Some(timeout) = timeout { if start_time.elapsed() > timeout { return Err(anyhow::anyhow!("Proof request timed out.")); } } - - // Try to get proof status with retries. - let status_result = retry(backoff.clone(), || async { - match self.client.get_proof_request_status::

(request_id).await { - Ok(result) => Ok(result), - Err(e) => { - if let Some(status) = e.downcast_ref::() { - match status.code() { - Code::NotFound => { - log::error!("Proof request not found: {}", status.message()); - Err(backoff::Error::permanent(e)) - } - Code::Unavailable => { - log::warn!( - "Network temporarily unavailable, retrying: {}", - status.message() - ); - Err(backoff::Error::transient(e)) - } - Code::DeadlineExceeded => { - log::warn!( - "Request deadline exceeded, retrying: {}", - status.message() - ); - Err(backoff::Error::transient(e)) - } - _ => { - log::error!( - "Permanent error encountered: {} ({})", - status.message(), - status.code() - ); - Err(backoff::Error::permanent(e)) - } - } - } else { - log::error!("Unexpected error type: {}", e); - Err(backoff::Error::permanent(e)) - } - } + let remaining_timeout = timeout.map(|t| { + let elapsed = start_time.elapsed(); + if elapsed < t { + t - elapsed + } else { + Duration::from_secs(0) } - }) - .await; + }); - match status_result { - Ok((status, maybe_proof)) => { - if status.execution_status == ExecutionStatus::Unexecutable as i32 { - return Err(anyhow::anyhow!("Proof request is unexecutable")); - } + // Get status with retries. + let (status, maybe_proof) = with_retry( + || async { self.client.get_proof_request_status::

(request_id).await }, + remaining_timeout, + "get proof request status", + ) + .await?; - match FulfillmentStatus::try_from(status.fulfillment_status) { - Ok(FulfillmentStatus::Fulfilled) => { - return Ok(maybe_proof.unwrap()); - } - Ok(FulfillmentStatus::Assigned) => { - if !is_assigned { - log::info!("Proof request assigned, proving..."); - is_assigned = true; - } - } - Ok(FulfillmentStatus::Unfulfillable) => { - return Err(anyhow::anyhow!("Proof request is unfulfillable")); - } - _ => {} + // Check the execution status. + if status.execution_status == ExecutionStatus::Unexecutable as i32 { + return Err(anyhow::anyhow!("Proof request is unexecutable")); + } + + // Check the fulfillment status. + match FulfillmentStatus::try_from(status.fulfillment_status) { + Ok(FulfillmentStatus::Fulfilled) => { + return Ok(maybe_proof.unwrap()); + } + Ok(FulfillmentStatus::Assigned) => { + if !is_assigned { + log::info!("Proof request assigned, proving..."); + is_assigned = true; } } - Err(e) => { - return Err(e); + Ok(FulfillmentStatus::Unfulfillable) => { + return Err(anyhow::anyhow!("Proof request is unfulfillable")); } + _ => {} } sleep(Duration::from_secs(2)).await; @@ -282,3 +249,58 @@ impl From for ProofMode { } } } + +/// Execute an async operation with exponential backoff retries. +pub async fn with_retry( + operation: F, + timeout: Option, + operation_name: &str, +) -> Result +where + F: Fn() -> Fut, + Fut: std::future::Future>, +{ + let backoff = ExponentialBackoff { + initial_interval: Duration::from_secs(1), + max_interval: Duration::from_secs(120), + max_elapsed_time: timeout, + ..Default::default() + }; + + retry(backoff, || async { + match operation().await { + Ok(result) => Ok(result), + Err(e) => { + if let Some(status) = e.downcast_ref::() { + match status.code() { + Code::Unavailable => { + log::warn!( + "Network temporarily unavailable for {}, retrying: {}", + operation_name, + status.message() + ); + Err(BackoffError::transient(e)) + } + Code::NotFound => { + log::error!("{} not found: {}", operation_name, status.message()); + Err(BackoffError::permanent(e)) + } + _ => { + log::error!( + "Permanent error encountered in {}: {} ({})", + operation_name, + status.message(), + status.code() + ); + Err(BackoffError::permanent(e)) + } + } + } else { + log::error!("Unexpected error type in {}: {}", operation_name, e); + Err(BackoffError::permanent(e)) + } + } + } + }) + .await +}