Skip to content

Commit

Permalink
better retries
Browse files Browse the repository at this point in the history
  • Loading branch information
mattstam committed Nov 13, 2024
1 parent 5b98d4b commit 5e7a134
Showing 1 changed file with 22 additions and 3 deletions.
25 changes: 22 additions & 3 deletions crates/sdk/src/network-v2/prover.rs
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ where
match operation().await {
Ok(result) => Ok(result),
Err(e) => {
// Check for tonic status errors.
if let Some(status) = e.downcast_ref::<tonic::Status>() {
match status.code() {
Code::Unavailable => {
Expand Down Expand Up @@ -300,11 +301,29 @@ where
}
}
} else {
log::error!("Unexpected error type when {}: {}", operation_name, e);
Err(BackoffError::permanent(e))
// Check for common transport errors.
let error_msg = e.to_string().to_lowercase();
let is_transient = error_msg.contains("tls handshake") ||
error_msg.contains("dns error") ||
error_msg.contains("connection reset") ||
error_msg.contains("broken pipe") ||
error_msg.contains("transport error") ||
error_msg.contains("failed to lookup");

if is_transient {
log::warn!(
"Transient transport error when {}: {}, retrying...",
operation_name,
error_msg
);
Err(BackoffError::transient(e))
} else {
log::error!("Permanent error when {}: {}", operation_name, error_msg);
Err(BackoffError::permanent(e))
}
}
}
}
})
.await
}
}

0 comments on commit 5e7a134

Please sign in to comment.