From f13b3e30aa70c04d1a4d2bacff297f9587567510 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Wed, 30 Aug 2023 12:02:59 -0700 Subject: [PATCH] fix: More improvements to file locking. (#179) --- .cargo/nextest.toml | 4 +- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 2 + Cargo.lock | 126 ++++++++++++++------------- Cargo.toml | 18 ++-- crates/cli/Cargo.toml | 2 +- crates/core/Cargo.toml | 2 +- crates/core/src/helpers.rs | 17 +--- crates/core/src/tool_manifest.rs | 5 -- crates/warpgate/Cargo.toml | 2 +- crates/warpgate/src/helpers.rs | 19 ++-- crates/wasm-plugin/src/host_funcs.rs | 10 ++- 12 files changed, 101 insertions(+), 108 deletions(-) diff --git a/.cargo/nextest.toml b/.cargo/nextest.toml index cb96acda0..7ec7147e3 100644 --- a/.cargo/nextest.toml +++ b/.cargo/nextest.toml @@ -1,5 +1,5 @@ [profile.ci] -retries = 1 -test-threads = 1 fail-fast = false +retries = 1 slow-timeout = { period = "120s", terminate-after = 4 } +test-threads = 2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e91ad7d8c..2389498eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: cargo wasi build -p proto_wasm_test; node ../.github/actions/delete-target-files.js; if: ${{ runner.os == 'Windows' }} - - run: cargo nextest run --workspace --exclude proto_pdk + - run: cargo nextest run --workspace --exclude proto_pdk --profile ci --config-file ./.cargo/nextest.toml env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9773cbfde..8b5dea47c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ #### ⚙️ Internal +- Improved file locking logic and scenarios. +- Improved logging to pinpoint slow operations. - Updated Rust to v1.72. ## 0.15.0 diff --git a/Cargo.lock b/Cargo.lock index 974f178f5..c538ffd81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,16 +70,15 @@ dependencies = [ [[package]] name = "anstream" -version = "0.3.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] @@ -109,9 +108,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -474,9 +473,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.21" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c27cdf28c0f604ba3f512b0c9a409f8de8513e4816705deb0498b627e7c3a3fd" +checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27" dependencies = [ "clap_builder", "clap_derive", @@ -485,9 +484,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.21" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a9f1ab5e9f01a9b81f202e8562eb9a10de70abf9eaeac1be465c28b75aa4aa" +checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d" dependencies = [ "anstream", "anstyle", @@ -497,18 +496,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.3.2" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc443334c81a804575546c5a8a79b4913b50e28d69232903604cada1de817ce" +checksum = "586a385f7ef2f8b4d86bddaa0c094794e7ccbfe5ffef1f434fe928143fc783a5" dependencies = [ - "clap 4.3.21", + "clap 4.4.1", ] [[package]] name = "clap_derive" -version = "4.3.12" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" dependencies = [ "heck", "proc-macro2", @@ -1581,7 +1580,7 @@ checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" dependencies = [ "http", "hyper", - "rustls 0.21.2", + "rustls 0.21.7", "tokio", "tokio-rustls", ] @@ -2130,9 +2129,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "once_map" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca76b681ece000dd746a1a75152d96001584cc5f0f0adc44e5a04c32fdb59ac" +checksum = "169e72192bc84493725bfca5d47da4ee2abc8acf155586d8127eb17f022497f4" dependencies = [ "ahash", "hashbrown 0.14.0", @@ -2301,7 +2300,7 @@ dependencies = [ name = "proto_cli" version = "0.15.0" dependencies = [ - "clap 4.3.21", + "clap 4.4.1", "clap_complete", "convert_case", "dialoguer", @@ -2325,7 +2324,7 @@ dependencies = [ "strum", "tokio", "tracing", - "winreg 0.50.0", + "winreg 0.51.0", ] [[package]] @@ -2523,14 +2522,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" dependencies = [ "aho-corasick 1.0.1", "memchr", - "regex-automata 0.3.6", - "regex-syntax 0.7.4", + "regex-automata 0.3.7", + "regex-syntax 0.7.5", ] [[package]] @@ -2544,13 +2543,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" dependencies = [ "aho-corasick 1.0.1", "memchr", - "regex-syntax 0.7.4", + "regex-syntax 0.7.5", ] [[package]] @@ -2561,9 +2560,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "relative-path" @@ -2573,9 +2572,9 @@ checksum = "4bf2521270932c3c7bed1a59151222bd7643c79310f2916f01925e1e16255698" [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ "base64 0.21.0", "bytes", @@ -2594,7 +2593,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.2", + "rustls 0.21.7", "rustls-pemfile", "serde", "serde_json", @@ -2608,8 +2607,8 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", - "winreg 0.10.1", + "webpki-roots 0.25.2", + "winreg 0.50.0", ] [[package]] @@ -2718,9 +2717,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.2" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring", @@ -2739,9 +2738,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.100.1" +version = "0.101.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" dependencies = [ "ring", "untrusted", @@ -2801,9 +2800,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] @@ -2820,9 +2819,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", @@ -2831,9 +2830,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" +checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" dependencies = [ "itoa", "ryu", @@ -3264,18 +3263,18 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.44" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.44" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ "proc-macro2", "quote", @@ -3369,9 +3368,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.30.0" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3ce25f50619af8b0aec2eb23deebe84249e19e2ddd393a6e16e3300a6dadfd" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", "bytes", @@ -3404,7 +3403,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.2", + "rustls 0.21.7", "tokio", ] @@ -3645,14 +3644,14 @@ dependencies = [ "rustls 0.20.8", "url", "webpki", - "webpki-roots", + "webpki-roots 0.22.6", ] [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -3879,9 +3878,9 @@ dependencies = [ [[package]] name = "wasm-streams" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" dependencies = [ "futures-util", "js-sys", @@ -4237,6 +4236,12 @@ dependencies = [ "webpki", ] +[[package]] +name = "webpki-roots" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" + [[package]] name = "wiggle" version = "8.0.1" @@ -4477,18 +4482,19 @@ dependencies = [ [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if", + "windows-sys 0.48.0", ] [[package]] name = "winreg" -version = "0.50.0" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "937f3df7948156640f46aacef17a70db0de5917bda9c92b0f751f3a955b588fc" dependencies = [ "cfg-if", "windows-sys 0.48.0", diff --git a/Cargo.toml b/Cargo.toml index 9bb06a9d6..d9779720e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,22 +5,22 @@ default-members = ["crates/cli"] [workspace.dependencies] cached = "0.44.0" -clap = "4.3.21" -clap_complete = "4.3.2" +clap = "4.4.1" +clap_complete = "4.4.0" convert_case = "0.6.0" extism = "0.4.0" extism-pdk = "0.3.3" human-sort = "0.2.2" miette = "5.10.0" once_cell = "1.18.0" -once_map = "0.4.7" -regex = "1.9.3" -reqwest = { version = "0.11.18", default-features = false, features = [ +once_map = "0.4.8" +regex = "1.9.4" +reqwest = { version = "0.11.20", default-features = false, features = [ "rustls-tls", ] } semver = "1.0.18" -serde = { version = "1.0.183", features = ["derive"] } -serde_json = "1.0.104" +serde = { version = "1.0.188", features = ["derive"] } +serde_json = "1.0.105" sha2 = "0.10.7" starbase_archive = { version = "0.2.0", features = [ "tar-gz", @@ -34,8 +34,8 @@ starbase_utils = { version = "0.2.20", default-features = false, features = [ "json", "toml", ] } -thiserror = "1.0.44" -tokio = { version = "1.30.0", features = ["full", "tracing"] } +thiserror = "1.0.47" +tokio = { version = "1.32.0", features = ["full", "tracing"] } tracing = "0.1.37" # Config for 'cargo dist' diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 4ec12da25..c3679ae32 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -54,7 +54,7 @@ tokio = { workspace = true } tracing = { workspace = true } [target.'cfg(windows)'.dependencies] -winreg = "0.50.0" +winreg = "0.51.0" [dev-dependencies] starbase_sandbox = { workspace = true } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 70ecca823..dd8636073 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -28,7 +28,7 @@ starbase_utils = { workspace = true, features = ["fs-lock"] } thiserror = { workspace = true } tinytemplate = "1.2.1" tracing = { workspace = true } -url = "2.4.0" +url = "2.4.1" [dev-dependencies] starbase_sandbox = { workspace = true } diff --git a/crates/core/src/helpers.rs b/crates/core/src/helpers.rs index e3c6dcbe9..78228addd 100644 --- a/crates/core/src/helpers.rs +++ b/crates/core/src/helpers.rs @@ -160,10 +160,6 @@ where let url = url.as_ref(); let dest_file = dest_file.as_ref(); - let handle_io_error = |error: io::Error| FsError::Create { - path: dest_file.to_path_buf(), - error, - }; let handle_http_error = |error: reqwest::Error| ProtoError::Http { url: url.to_owned(), error, @@ -175,11 +171,6 @@ where "Downloading file from URL", ); - // Ensure parent directories exist - if let Some(parent) = dest_file.parent() { - fs::create_dir_all(parent)?; - } - // Fetch the file from the HTTP source let response = reqwest::get(url).await.map_err(handle_http_error)?; let status = response.status(); @@ -200,10 +191,10 @@ where } // Write the bytes to our local file - let mut contents = io::Cursor::new(response.bytes().await.map_err(handle_http_error)?); - let mut file = fs::create_file(dest_file)?; - - io::copy(&mut contents, &mut file).map_err(handle_io_error)?; + fs::write_file_with_lock( + dest_file, + response.bytes().await.map_err(handle_http_error)?, + )?; Ok(()) } diff --git a/crates/core/src/tool_manifest.rs b/crates/core/src/tool_manifest.rs index 34ed24ad7..887c55eeb 100644 --- a/crates/core/src/tool_manifest.rs +++ b/crates/core/src/tool_manifest.rs @@ -4,7 +4,6 @@ use crate::{ }; use semver::Version; use serde::{Deserialize, Serialize}; -use starbase_utils::fs; use std::{ collections::{BTreeMap, HashSet}, env, @@ -79,10 +78,6 @@ impl ToolManifest { pub fn save(&self) -> miette::Result<()> { debug!(file = ?self.path, "Saving manifest"); - if let Some(parent) = self.path.parent() { - fs::create_dir_all(parent)?; - } - write_json_file_with_lock(&self.path, self)?; Ok(()) diff --git a/crates/warpgate/Cargo.toml b/crates/warpgate/Cargo.toml index a588ccb26..63e7cfdd6 100644 --- a/crates/warpgate/Cargo.toml +++ b/crates/warpgate/Cargo.toml @@ -18,7 +18,7 @@ serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } starbase_archive = { workspace = true } -starbase_utils = { workspace = true, features = ["glob"] } +starbase_utils = { workspace = true, features = ["fs-lock", "glob"] } starbase_styles = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } diff --git a/crates/warpgate/src/helpers.rs b/crates/warpgate/src/helpers.rs index 88f14d466..a2f950da8 100644 --- a/crates/warpgate/src/helpers.rs +++ b/crates/warpgate/src/helpers.rs @@ -3,9 +3,7 @@ use extism::Manifest; use miette::IntoDiagnostic; use reqwest::Url; use starbase_archive::Archiver; -use starbase_utils::fs::{self, FsError}; -use starbase_utils::glob; -use std::io; +use starbase_utils::{fs, glob}; use std::path::{Path, PathBuf}; use warpgate_api::VirtualPath; @@ -65,20 +63,15 @@ pub async fn download_url_to_temp(raw_url: &str, temp_dir: &Path) -> miette::Res } // Write the bytes to our temporary file - let mut contents = io::Cursor::new( + let temp_file = temp_dir.join(filename); + + fs::write_file_with_lock( + &temp_file, response .bytes() .await .map_err(|error| WarpgateError::Http { error })?, - ); - - let temp_file = temp_dir.join(filename); - let mut file = fs::create_file(&temp_file)?; - - io::copy(&mut contents, &mut file).map_err(|error| FsError::Create { - path: temp_file.to_path_buf(), - error, - })?; + )?; Ok(temp_file) } diff --git a/crates/wasm-plugin/src/host_funcs.rs b/crates/wasm-plugin/src/host_funcs.rs index ef63f1d54..830d34683 100644 --- a/crates/wasm-plugin/src/host_funcs.rs +++ b/crates/wasm-plugin/src/host_funcs.rs @@ -65,6 +65,14 @@ fn exec_command( let input_str = unsafe { (*plugin.memory).get_str(inputs[0].unwrap_i64() as usize)? }; let input: ExecCommandInput = serde_json::from_str(input_str)?; + trace!( + target: "proto_wasm::exec_command", + command = &input.command, + args = ?input.args, + env_vars = ?input.env_vars, + "Executing command from plugin" + ); + // let data = user_data.any().unwrap(); // let data = data.downcast_ref::().unwrap(); @@ -94,8 +102,6 @@ fn exec_command( trace!( target: "proto_wasm::exec_command", command = &input.command, - args = ?input.args, - env_vars = ?input.env_vars, exit_code = output.exit_code, stderr_len = output.stderr.len(), stdout_len = output.stdout.len(),