chore: WitnessGenExecutor

contracts/script/ZKDeployer.s.sol

@@ -11,6 +11,7 @@ contract ZKDeployer is Script, Utils {
         vm.startBroadcast();
 
         Config memory config = readJsonWithRPCFromEnv("zkconfig.json");
+        // TODO: This seems wrong. Why are we using the msg.sender as a proxy?
         config.l2OutputOracleProxy = address(new Proxy(msg.sender));
 
         address zkL2OutputOracleImpl = address(new ZKL2OutputOracle());

contracts/zkconfig.json

@@ -1,5 +1,5 @@
 {
-    "startingBlockNumber": 16621800,
+    "startingBlockNumber": 16795981,
     "l2RollupNode": "",
     "submissionInterval": 150,
     "l2BlockTime": 2,
@@ -11,4 +11,4 @@
     "vkey": "0x0010ea62be193a3c288183b203657e75184b8b92d5f872a25e2ff065d5e9c84d",
     "verifierGateway": "0x3B6041173B80E77f038f3F2C0f9744f04837185e",
     "l2OutputOracleProxy": "0x863508f057c09f7b94e582d74404859ecd36a306"
-}
+}

proposer/succinct/bin/cost_estimator.rs

@@ -2,25 +2,19 @@ use anyhow::Result;
 use clap::Parser;
 use kona_host::HostCli;
 use kona_primitives::RollupConfig;
-use log::{error, info};
+use log::info;
 use op_succinct_host_utils::{
     fetcher::{ChainMode, OPSuccinctDataFetcher},
     get_proof_stdin,
     stats::{get_execution_stats, ExecutionStats},
+    witnessgen::WitnessGenExecutor,
     ProgramType,
 };
-use op_succinct_proposer::run_native_host;
 use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
 use reqwest::Client;
 use serde::{Deserialize, Serialize};
 use sp1_sdk::{utils, ProverClient};
-use std::{
-    cmp::min,
-    env, fs,
-    future::Future,
-    path::PathBuf,
-    time::{Duration, Instant},
-};
+use std::{cmp::min, env, fs, future::Future, path::PathBuf, time::Instant};
 use tokio::task::block_in_place;
 
 pub const MULTI_BLOCK_ELF: &[u8] = include_bytes!("../../../elf/range-elf");
@@ -143,28 +137,30 @@ async fn run_native_data_generation(
     split_ranges: &[SpanBatchRange],
 ) -> Vec<BatchHostCli> {
     const CONCURRENT_NATIVE_HOST_RUNNERS: usize = 5;
-    const NATIVE_HOST_TIMEOUT: Duration = Duration::from_secs(300);
 
-    // TODO: Shut down all processes when the program exits OR a Ctrl+C is pressed.
-    let futures = split_ranges.chunks(CONCURRENT_NATIVE_HOST_RUNNERS).map(|chunk| {
-        futures::future::join_all(chunk.iter().map(|range| async {
+    let futures = split_ranges.chunks(CONCURRENT_NATIVE_HOST_RUNNERS).map(|chunk| async {
+        let mut witnessgen_executor = WitnessGenExecutor::default();
+
+        let mut batch_host_clis = Vec::new();
+        for range in chunk.iter() {
             let host_cli = data_fetcher
                 .get_host_cli_args(range.start, range.end, ProgramType::Multi)
                 .await
                 .unwrap();
+            batch_host_clis.push(BatchHostCli {
+                host_cli: host_cli.clone(),
+                start: range.start,
+                end: range.end,
+            });
+            witnessgen_executor
+                .spawn_witnessgen(&host_cli)
+                .await
+                .expect("Failed to spawn witness generation process.");
+        }
 
-            let data_dir = host_cli.data_dir.clone().expect("Data directory is not set.");
-
-            fs::create_dir_all(&data_dir).unwrap();
-
-            let res = run_native_host(&host_cli, NATIVE_HOST_TIMEOUT).await;
-            if res.is_err() {
-                error!("Failed to run native host: {:?}", res.err().unwrap());
-                std::process::exit(1);
-            }
+        witnessgen_executor.flush().await.expect("Failed to flush witness generation.");
 
-            BatchHostCli { host_cli, start: range.start, end: range.end }
-        }))
+        batch_host_clis
     });
 
     futures::future::join_all(futures).await.into_iter().flatten().collect()

proposer/succinct/bin/server.rs

@@ -10,9 +10,9 @@ use base64::{engine::general_purpose, Engine as _};
 use log::info;
 use op_succinct_client_utils::{RawBootInfo, BOOT_INFO_SIZE};
 use op_succinct_host_utils::{
-    fetcher::OPSuccinctDataFetcher, get_agg_proof_stdin, get_proof_stdin, ProgramType,
+    fetcher::OPSuccinctDataFetcher, get_agg_proof_stdin, get_proof_stdin,
+    witnessgen::WitnessGenExecutor, ProgramType,
 };
-use op_succinct_proposer::run_native_host;
 use serde::{Deserialize, Deserializer, Serialize};
 use sp1_sdk::{
     network::client::NetworkClient,
@@ -87,9 +87,9 @@ async fn request_span_proof(
     // Start the server and native client with a timeout.
     // Note: Ideally, the server should call out to a separate process that executes the native
     // host, and return an ID that the client can poll on to check if the proof was submitted.
-    // TODO: If this fails, we should definitely NOT request a proof! Otherwise, we get execution
-    // failures on the cluster.
-    run_native_host(&host_cli, Duration::from_secs(60)).await?;
+    let mut witnessgen_executor = WitnessGenExecutor::default();
+    witnessgen_executor.spawn_witnessgen(&host_cli).await?;
+    witnessgen_executor.flush().await?;
 
     let sp1_stdin = get_proof_stdin(&host_cli)?;
 

scripts/prove/bin/multi.rs

@@ -2,11 +2,11 @@ use std::{fs, time::Instant};
 
 use anyhow::Result;
 use clap::Parser;
-use kona_host::start_server_and_native_client;
 use op_succinct_host_utils::{
     fetcher::{ChainMode, OPSuccinctDataFetcher},
     get_proof_stdin,
     stats::get_execution_stats,
+    witnessgen::WitnessGenExecutor,
     ProgramType,
 };
 use sp1_sdk::{utils, ProverClient};
@@ -57,7 +57,9 @@ async fn main() -> Result<()> {
         fs::create_dir_all(&data_dir).unwrap();
 
         // Start the server and native client.
-        start_server_and_native_client(host_cli.clone()).await?;
+        let mut witnessgen_executor = WitnessGenExecutor::default();
+        witnessgen_executor.spawn_witnessgen(&host_cli).await?;
+        witnessgen_executor.flush().await?;
     }
     let execution_duration = start_time.elapsed();
     println!("Execution Duration: {:?}", execution_duration);

scripts/prove/bin/single.rs

@@ -2,9 +2,10 @@ use std::{env, fs};
 
 use anyhow::Result;
 use clap::Parser;
-use kona_host::start_server_and_native_client;
 use num_format::{Locale, ToFormattedString};
-use op_succinct_host_utils::{fetcher::OPSuccinctDataFetcher, get_proof_stdin, ProgramType};
+use op_succinct_host_utils::{
+    fetcher::OPSuccinctDataFetcher, get_proof_stdin, witnessgen::WitnessGenExecutor, ProgramType,
+};
 use sp1_sdk::{utils, ProverClient};
 
 pub const SINGLE_BLOCK_ELF: &[u8] = include_bytes!("../../../elf/fault-proof-elf");
@@ -46,7 +47,9 @@ async fn main() -> Result<()> {
         fs::create_dir_all(&data_dir).unwrap();
 
         // Start the server and native client.
-        start_server_and_native_client(host_cli.clone()).await.unwrap();
+        let mut witnessgen_executor = WitnessGenExecutor::default();
+        witnessgen_executor.spawn_witnessgen(&host_cli).await?;
+        witnessgen_executor.flush().await?;
     }
 
     // Get the stdin for the block.

scripts/witnessgen/bin/native_host_runner.rs

@@ -1,7 +1,7 @@
 use anyhow::Result;
 use clap::Parser;
 use kona_host::{init_tracing_subscriber, start_server, start_server_and_native_client, HostCli};
-use log::{error, info};
+use log::info;
 
 // Source: https://github.com/ethereum-optimism/kona/blob/main/bin/host/src/main.rs
 #[tokio::main(flavor = "multi_thread")]
@@ -12,11 +12,7 @@ async fn main() -> Result<()> {
     if cfg.server {
         start_server(cfg).await?;
     } else {
-        let res = start_server_and_native_client(cfg).await;
-        if res.is_err() {
-            error!("Failed to run server and native client: {:?}", res.err().unwrap());
-            std::process::exit(1);
-        }
+        start_server_and_native_client(cfg).await?;
     }
 
     info!("Exiting host program.");

utils/host/src/lib.rs

@@ -1,6 +1,7 @@
 pub mod fetcher;
 pub mod helpers;
 pub mod stats;
+pub mod witnessgen;
 
 use alloy_consensus::Header;
 use alloy_primitives::B256;