Merge pull request #1321 from Phala-Network/ra-timeout

pruntime: Configurable RA timeout and retries

Cargo.toml

@@ -64,6 +64,7 @@ members = [
 	"crates/phala-allocator",
 	"crates/phala-sanitized-logger",
 	"crates/phala-wasm-checker",
+	"crates/phala-clap-parsers",
 	"crates/wasmer-tunables",
 	"crates/phala-rocket-middleware",
 	"crates/pink/runner",

crates/phactory/api/src/ecall_args.rs

@@ -1,8 +1,8 @@
 use alloc::string::String;
 use parity_scale_codec::{Decode, Encode};
-use serde::{Deserialize, Serialize};
+use core::time::Duration;
 
-#[derive(Serialize, Deserialize, Debug, Encode, Decode, Default, Clone)]
+#[derive(Debug, Encode, Decode, Default, Clone)]
 pub struct InitArgs {
     /// The GK master key sealing path.
     pub sealing_path: String,
@@ -42,6 +42,12 @@ pub struct InitArgs {
 
     /// Disable the RCU policy to update the Phactory state.
     pub no_rcu: bool,
+
+    /// The timeout of getting the attestation report.
+    pub ra_timeout: Duration,
+
+    /// The max retry times of getting the attestation report.
+    pub ra_max_retries: u32,
 }
 
 pub use phala_git_revision::git_revision;

crates/phactory/pal/src/lib.rs

@@ -2,6 +2,7 @@
 
 use std::fmt::Debug;
 use std::path::Path;
+use core::time::Duration;
 
 use phala_types::AttestationProvider;
 
@@ -24,6 +25,7 @@ pub trait RA {
         &self,
         provider: Option<AttestationProvider>,
         data: &[u8],
+        timeout: Duration,
     ) -> Result<Vec<u8>, Self::Error>;
     fn quote_test(&self, provider: Option<AttestationProvider>) -> Result<(), Self::Error>;
     fn measurement(&self) -> Option<Vec<u8>>;

crates/phactory/src/prpc_service.rs

@@ -4,6 +4,7 @@ use std::future::Future;
 use std::io::Read;
 use std::str::FromStr;
 use std::sync::{Arc, Mutex, MutexGuard};
+use std::time::Duration;
 
 use crate::benchmark::Flags;
 use crate::system::{System, MAX_SUPPORTED_CONSENSUS_VERSION};
@@ -558,25 +559,14 @@ impl<Platform: pal::Platform + Serialize + DeserializeOwned> Phactory<Platform>
             info!("Encoded runtime info");
             info!("{:?}", hex::encode(&cached_resp.encoded_runtime_info));
 
-            let encoded_report = match self
-                .platform
-                .create_attestation_report(self.attestation_provider, &runtime_info_hash)
-            {
-                Ok(r) => r,
-                Err(e) => {
-                    let message = format!("Failed to create attestation report: {e:?}");
-                    error!("{}", message);
-                    return Err(from_display(message));
-                }
-            };
-
-            cached_resp.attestation = Some(pb::Attestation {
-                version: 1,
-                provider: serde_json::to_string(&self.attestation_provider).unwrap(),
-                payload: None,
-                encoded_report,
-                timestamp: now(),
-            });
+            let report = create_attestation_report_on(
+                &self.platform,
+                self.attestation_provider,
+                &runtime_info_hash,
+                self.args.ra_timeout,
+                self.args.ra_max_retries,
+            )?;
+            cached_resp.attestation = Some(report);
         }
 
         Ok(cached_resp.clone())
@@ -1245,14 +1235,26 @@ fn create_attestation_report_on<Platform: pal::Platform>(
     platform: &Platform,
     attestation_provider: Option<AttestationProvider>,
     data: &[u8],
+    timeout: Duration,
+    max_retries: u32,
 ) -> RpcResult<pb::Attestation> {
-    let encoded_report = match platform.create_attestation_report(attestation_provider, data) {
-        Ok(r) => r,
-        Err(e) => {
-            let message = format!("Failed to create attestation report: {e:?}");
-            error!("{}", message);
-            return Err(from_display(message));
-        }
+    let mut tried = 0;
+    let encoded_report = loop {
+        break match platform.create_attestation_report(attestation_provider, data, timeout) {
+            Ok(r) => r,
+            Err(e) => {
+                let message = format!("Failed to create attestation report: {e:?}");
+                error!("{}", message);
+                if tried >= max_retries {
+                    return Err(from_display(message));
+                }
+                let sleep_secs = (1 << tried).min(8);
+                info!("Retrying after {} seconds...", sleep_secs);
+                std::thread::sleep(Duration::from_secs(sleep_secs));
+                tried += 1;
+                continue;
+            }
+        };
     };
     Ok(pb::Attestation {
         version: 1,
@@ -1633,6 +1635,8 @@ impl<Platform: pal::Platform + Serialize + DeserializeOwned> PhactoryApi for Rpc
                 &phactory.platform,
                 attestation_provider,
                 &worker_key_hash,
+                phactory.args.ra_timeout,
+                phactory.args.ra_max_retries,
             )?)
         } else {
             info!("Omit RA report in workerkey response in dev mode");
@@ -1689,6 +1693,8 @@ impl<Platform: pal::Platform + Serialize + DeserializeOwned> PhactoryApi for Rpc
                 &phactory.platform,
                 Some(AttestationProvider::Ias),
                 &handler_hash,
+                phactory.args.ra_timeout,
+                phactory.args.ra_max_retries,
             )?)
         } else {
             info!("Omit client RA report for dev mode challenge");

crates/phala-clap-parsers/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "phala-clap-parsers"
+version = "0.1.0"
+edition = "2021"
+description = "Some custom value parsers for clap"
+license = "MIT"
+repository = "https://github.com/Phala-Network/phala-blockchain"
+authors = ["Kevin Wang <[email protected]>"]
+
+[dependencies]

crates/phala-clap-parsers/src/duration.rs

@@ -0,0 +1,63 @@
+#[derive(Debug)]
+pub struct InvalidDuration;
+
+impl std::fmt::Display for InvalidDuration {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Invalid duration")
+    }
+}
+
+impl std::error::Error for InvalidDuration {}
+
+use core::time::Duration;
+
+pub fn parse_duration(s: &str) -> Result<Duration, InvalidDuration> {
+    let mut num_str = s;
+    let mut unit = "s";
+
+    if let Some(idx) = s.find(|c: char| !c.is_numeric()) {
+        num_str = &s[..idx];
+        unit = &s[idx..];
+    }
+
+    let num = num_str.parse::<u64>().or(Err(InvalidDuration))?;
+
+    let num = match unit {
+        "s" | "" => num * 1000,
+        "m" => num * 60 * 1000,
+        "h" => num * 60 * 60 * 1000,
+        "d" => num * 60 * 60 * 24 * 1000,
+        "ms" => num,
+        _ => return Err(InvalidDuration),
+    };
+
+    Ok(Duration::from_millis(num))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_duration() {
+        assert_eq!(parse_duration("10ms").unwrap(), Duration::from_millis(10));
+        assert_eq!(parse_duration("10m").unwrap(), Duration::from_secs(600));
+        assert_eq!(parse_duration("10s").unwrap(), Duration::from_secs(10));
+        assert_eq!(parse_duration("10m").unwrap(), Duration::from_secs(600));
+        assert_eq!(parse_duration("10h").unwrap(), Duration::from_secs(36000));
+        assert_eq!(parse_duration("10d").unwrap(), Duration::from_secs(864000));
+        assert_eq!(parse_duration("1").unwrap(), Duration::from_secs(1));
+        assert_eq!(parse_duration("100").unwrap(), Duration::from_secs(100));
+    }
+
+    #[test]
+    fn test_parse_duration_invalid() {
+        assert!(parse_duration("10x").is_err());
+        assert!(parse_duration("ms").is_err());
+    }
+
+    #[test]
+    fn test_parse_duration_empty() {
+        assert!(parse_duration("").is_err());
+    }
+}

crates/phala-clap-parsers/src/lib.rs

@@ -0,0 +1,3 @@
+pub use duration::{parse_duration, InvalidDuration};
+
+mod duration;

standalone/phat-poller/Cargo.toml

@@ -8,6 +8,7 @@ phaxt = { path = "../../crates/phaxt" }
 phactory-api = { path = "../../crates/phactory/api", features = ["pruntime-client"] }
 phala-crypto = { path = "../../crates/phala-crypto" }
 phala-types = { path = "../../crates/phala-types" }
+phala-clap-parsers = { path = "../../crates/phala-clap-parsers" }
 
 sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.42" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.42" }

standalone/phat-poller/src/args.rs

@@ -1,4 +1,5 @@
 use clap::{Parser, Subcommand};
+use phala_clap_parsers::parse_duration;
 use sp_core::{crypto::SecretStringError, Pair, H256};
 use std::time::Duration;
 
@@ -66,43 +67,10 @@ pub struct RunArgs {
     pub caller: sp_core::sr25519::Pair,
 }
 
-#[derive(Debug)]
-pub struct InvalidDuration;
-
-impl std::fmt::Display for InvalidDuration {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "Invalid duration")
-    }
-}
-
-impl std::error::Error for InvalidDuration {}
-
 fn parse_sr25519(s: &str) -> Result<sp_core::sr25519::Pair, SecretStringError> {
     <sp_core::sr25519::Pair as Pair>::from_string(s, None)
 }
 
-fn parse_duration(s: &str) -> Result<Duration, InvalidDuration> {
-    let mut num_str = s;
-    let mut unit = "s";
-
-    if let Some(idx) = s.find(|c: char| !c.is_numeric()) {
-        num_str = &s[..idx];
-        unit = &s[idx..];
-    }
-
-    let num = num_str.parse::<u64>().or(Err(InvalidDuration))?;
-
-    let num = match unit {
-        "s" | "" => num,
-        "m" => num * 60,
-        "h" => num * 60 * 60,
-        "d" => num * 60 * 60 * 24,
-        _ => return Err(InvalidDuration),
-    };
-
-    Ok(Duration::from_secs(num))
-}
-
 pub fn parse_hash(s: &str) -> Result<H256, hex::FromHexError> {
     let s = s.trim_start_matches("0x");
     if s.len() != 64 {
@@ -112,30 +80,3 @@ pub fn parse_hash(s: &str) -> Result<H256, hex::FromHexError> {
     hex::decode_to_slice(s, &mut buf)?;
     Ok(H256::from(buf))
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parse_duration() {
-        assert_eq!(parse_duration("10").unwrap(), Duration::from_secs(10));
-        assert_eq!(parse_duration("10s").unwrap(), Duration::from_secs(10));
-        assert_eq!(parse_duration("10m").unwrap(), Duration::from_secs(600));
-        assert_eq!(parse_duration("10h").unwrap(), Duration::from_secs(36000));
-        assert_eq!(parse_duration("10d").unwrap(), Duration::from_secs(864000));
-        assert_eq!(parse_duration("1").unwrap(), Duration::from_secs(1));
-        assert_eq!(parse_duration("100").unwrap(), Duration::from_secs(100));
-    }
-
-    #[test]
-    fn test_parse_duration_invalid() {
-        assert!(parse_duration("10x").is_err());
-        assert!(parse_duration("ms").is_err());
-    }
-
-    #[test]
-    fn test_parse_duration_empty() {
-        assert!(parse_duration("").is_err());
-    }
-}

standalone/pruntime/Cargo.toml

@@ -38,6 +38,7 @@ phala-allocator = { path = "../../crates/phala-allocator" }
 phala-rocket-middleware = { path = "../../crates/phala-rocket-middleware" }
 phala-types = { path = "../../crates/phala-types", features = ["enable_serde", "sgx"] }
 phala-git-revision = { path = "../../crates/phala-git-revision" }
+phala-clap-parsers = { path = "../../crates/phala-clap-parsers" }
 sgx-api-lite = { path = "../../crates/sgx-api-lite" }
 tracing = "0.1"
 hex_fmt = "0.3.0"

standalone/pruntime/src/ias.rs

@@ -7,17 +7,20 @@ use reqwest_env_proxy::EnvProxyBuilder as _;
 pub const IAS_HOST: &str = env!("IAS_HOST");
 pub const IAS_REPORT_ENDPOINT: &str = env!("IAS_REPORT_ENDPOINT");
 
-fn get_report_from_intel(quote: &[u8], ias_key: &str) -> Result<(String, String, String)> {
+fn get_report_from_intel(
+    quote: &[u8],
+    ias_key: &str,
+    timeout: Duration,
+) -> Result<(String, String, String)> {
     let encoded_quote = base64::encode(quote);
     let encoded_json = format!("{{\"isvEnclaveQuote\":\"{encoded_quote}\"}}\r\n");
 
     let mut res_body_buffer = Vec::new(); //container for body of a response
-    let timeout = Some(Duration::from_secs(8));
 
     let url: reqwest::Url = format!("https://{IAS_HOST}{IAS_REPORT_ENDPOINT}").parse()?;
     info!(from=%url, "Getting RA report");
     let mut res = reqwest::blocking::Client::builder()
-        .timeout(timeout)
+        .timeout(Some(timeout))
         .env_proxy(url.domain().unwrap_or_default())
         .build()
         .context("Failed to create http client, maybe invalid IAS URI")?
@@ -93,9 +96,13 @@ pub fn create_quote_vec(data: &[u8]) -> Result<Vec<u8>> {
     Ok(fs::read("/dev/attestation/quote")?)
 }
 
-pub fn create_attestation_report(data: &[u8], ias_key: &str) -> Result<(String, Vec<u8>, Vec<u8>)> {
+pub fn create_attestation_report(
+    data: &[u8],
+    ias_key: &str,
+    timeout: Duration,
+) -> Result<(String, Vec<u8>, Vec<u8>)> {
     let quote_vec = create_quote_vec(data)?;
-    let (attn_report, sig, cert) = get_report_from_intel(&quote_vec, ias_key)?;
+    let (attn_report, sig, cert) = get_report_from_intel(&quote_vec, ias_key, timeout)?;
 
     let sig = base64::decode(sig).expect("Sig should be a valid base64");
     let cert = base64::decode(cert).expect("SigCert should be a valid base64");