Skip to content

Commit

Permalink
retry both
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattstam
mattstam committed Nov 8, 2024
1 parent f62cf52 commit 6170487
Showing 1 changed file with 109 additions and 87 deletions.
196 changes: 109 additions & 87 deletions crates/sdk/src/network-v2/prover.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use crate::{
Prover, SP1Context, SP1ProofKind, SP1ProofWithPublicValues, SP1ProvingKey, SP1VerifyingKey,
};
use anyhow::Result;
use backoff::{future::retry, ExponentialBackoff};
use backoff::{future::retry, Error as BackoffError, ExponentialBackoff};
use serde::de::DeserializeOwned;
use sp1_core_machine::io::SP1Stdin;
use sp1_prover::{components::DefaultProverComponents, SP1Prover, SP1_CIRCUIT_VERSION};
Expand Down Expand Up @@ -80,20 +80,26 @@ impl NetworkProver {

log::info!("Requesting proof with cycle limit: {}", cycle_limit);

// Request the proof.
let response = self
.client
.request_proof(
elf,
&stdin,
&vk,
mode,
SP1_CIRCUIT_VERSION,
FulfillmentStrategy::Hosted,
timeout_secs,
cycle_limit,
)
.await?;
// Request the proof with retries.
let response = with_retry(
|| async {
self.client
.request_proof(
elf,
&stdin,
&vk,
mode,
SP1_CIRCUIT_VERSION,
FulfillmentStrategy::Hosted,
timeout_secs,
cycle_limit,
)
.await
},
timeout,
"request proof",
)
.await?;

// Log the request ID and transaction hash.
let tx_hash_hex = "0x".to_string() + &hex::encode(response.tx_hash);
Expand All @@ -102,7 +108,7 @@ impl NetworkProver {
log::info!("Created request {} in transaction {}", request_id_hex, tx_hash_hex);

if NetworkClient::rpc_url() == DEFAULT_PROVER_NETWORK_RPC {
log::info!("View in explorer: https://explorer-v2.succinct.xyz/{}", request_id_hex);
log::info!("View in explorer: https://network.succinct.xyz/request/{}", request_id_hex);
}

Ok(request_id)
Expand All @@ -118,89 +124,50 @@ impl NetworkProver {
let mut is_assigned = false;
let start_time = Instant::now();

// Configure retries with exponential backoff.
let backoff = ExponentialBackoff {
initial_interval: Duration::from_secs(1),
max_interval: Duration::from_secs(30),
max_elapsed_time: timeout,
..Default::default()
};

loop {
// Calculate the remaining timeout.
if let Some(timeout) = timeout {
if start_time.elapsed() > timeout {
return Err(anyhow::anyhow!("Proof request timed out."));
}
}

// Try to get proof status with retries.
let status_result = retry(backoff.clone(), || async {
match self.client.get_proof_request_status::<P>(request_id).await {
Ok(result) => Ok(result),
Err(e) => {
if let Some(status) = e.downcast_ref::<tonic::Status>() {
match status.code() {
Code::NotFound => {
log::error!("Proof request not found: {}", status.message());
Err(backoff::Error::permanent(e))
}
Code::Unavailable => {
log::warn!(
"Network temporarily unavailable, retrying: {}",
status.message()
);
Err(backoff::Error::transient(e))
}
Code::DeadlineExceeded => {
log::warn!(
"Request deadline exceeded, retrying: {}",
status.message()
);
Err(backoff::Error::transient(e))
}
_ => {
log::error!(
"Permanent error encountered: {} ({})",
status.message(),
status.code()
);
Err(backoff::Error::permanent(e))
}
}
} else {
log::error!("Unexpected error type: {}", e);
Err(backoff::Error::permanent(e))
}
}
let remaining_timeout = timeout.map(|t| {
let elapsed = start_time.elapsed();
if elapsed < t {
t - elapsed
} else {
Duration::from_secs(0)
}
})
.await;
});

match status_result {
Ok((status, maybe_proof)) => {
if status.execution_status == ExecutionStatus::Unexecutable as i32 {
return Err(anyhow::anyhow!("Proof request is unexecutable"));
}
// Get status with retries.
let (status, maybe_proof) = with_retry(
|| async { self.client.get_proof_request_status::<P>(request_id).await },
remaining_timeout,
"get proof request status",
)
.await?;

match FulfillmentStatus::try_from(status.fulfillment_status) {
Ok(FulfillmentStatus::Fulfilled) => {
return Ok(maybe_proof.unwrap());
}
Ok(FulfillmentStatus::Assigned) => {
if !is_assigned {
log::info!("Proof request assigned, proving...");
is_assigned = true;
}
}
Ok(FulfillmentStatus::Unfulfillable) => {
return Err(anyhow::anyhow!("Proof request is unfulfillable"));
}
_ => {}
// Check the execution status.
if status.execution_status == ExecutionStatus::Unexecutable as i32 {
return Err(anyhow::anyhow!("Proof request is unexecutable"));
}

// Check the fulfillment status.
match FulfillmentStatus::try_from(status.fulfillment_status) {
Ok(FulfillmentStatus::Fulfilled) => {
return Ok(maybe_proof.unwrap());
}
Ok(FulfillmentStatus::Assigned) => {
if !is_assigned {
log::info!("Proof request assigned, proving...");
is_assigned = true;
}
}
Err(e) => {
return Err(e);
Ok(FulfillmentStatus::Unfulfillable) => {
return Err(anyhow::anyhow!("Proof request is unfulfillable"));
}
_ => {}
}

sleep(Duration::from_secs(2)).await;
Expand Down Expand Up @@ -282,3 +249,58 @@ impl From<SP1ProofKind> for ProofMode {
}
}
}

/// Execute an async operation with exponential backoff retries.
pub async fn with_retry<T, F, Fut>(
operation: F,
timeout: Option<Duration>,
operation_name: &str,
) -> Result<T>
where
F: Fn() -> Fut,
Fut: std::future::Future<Output = Result<T>>,
{
let backoff = ExponentialBackoff {
initial_interval: Duration::from_secs(1),
max_interval: Duration::from_secs(120),
max_elapsed_time: timeout,
..Default::default()
};

retry(backoff, || async {
match operation().await {
Ok(result) => Ok(result),
Err(e) => {
if let Some(status) = e.downcast_ref::<tonic::Status>() {
match status.code() {
Code::Unavailable => {
log::warn!(
"Network temporarily unavailable for {}, retrying: {}",
operation_name,
status.message()
);
Err(BackoffError::transient(e))
}
Code::NotFound => {
log::error!("{} not found: {}", operation_name, status.message());
Err(BackoffError::permanent(e))
}
_ => {
log::error!(
"Permanent error encountered in {}: {} ({})",
operation_name,
status.message(),
status.code()
);
Err(BackoffError::permanent(e))
}
}
} else {
log::error!("Unexpected error type in {}: {}", operation_name, e);
Err(BackoffError::permanent(e))
}
}
}
})
.await
}

0 comments on commit 6170487

Please sign in to comment.