From 215033b4a192df66317b7f1d73c7983df84353c9 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Sat, 25 Jun 2022 00:08:24 -0700 Subject: [PATCH] new: Add caching to VCS layer. (#161) * Add caching. * Use same instance. * Update changelog. * Trigger runs. * Remove comment. --- crates/cli/src/commands/ci.rs | 2 +- crates/cli/src/commands/run.rs | 2 +- .../workspace/src/actions/hashing/target.rs | 2 +- crates/workspace/src/vcs/git.rs | 32 ++++++++++++++++--- crates/workspace/src/vcs/svn.rs | 31 +++++++++++++++--- crates/workspace/src/workspace.rs | 10 +++--- packages/cli/CHANGELOG.md | 7 ++++ 7 files changed, 68 insertions(+), 18 deletions(-) diff --git a/crates/cli/src/commands/ci.rs b/crates/cli/src/commands/ci.rs index a57ae3ceea1..1cfdc116887 100644 --- a/crates/cli/src/commands/ci.rs +++ b/crates/cli/src/commands/ci.rs @@ -40,7 +40,7 @@ async fn gather_touched_files( ) -> Result { print_header("Gathering touched files"); - let vcs = workspace.detect_vcs()?; + let vcs = &workspace.vcs; let default_branch = vcs.get_default_branch(); let current_branch = vcs.get_local_branch().await?; diff --git a/crates/cli/src/commands/run.rs b/crates/cli/src/commands/run.rs index 4f0a2131c18..a69adce94e5 100644 --- a/crates/cli/src/commands/run.rs +++ b/crates/cli/src/commands/run.rs @@ -40,7 +40,7 @@ async fn get_touched_files( status: &RunStatus, upstream: bool, ) -> Result { - let vcs = workspace.detect_vcs()?; + let vcs = &workspace.vcs; let touched_files = if upstream { vcs.get_touched_files_between_revisions(vcs.get_default_branch(), "HEAD") diff --git a/crates/workspace/src/actions/hashing/target.rs b/crates/workspace/src/actions/hashing/target.rs index 2cbdb831785..d3e2038b2cd 100644 --- a/crates/workspace/src/actions/hashing/target.rs +++ b/crates/workspace/src/actions/hashing/target.rs @@ -34,7 +34,7 @@ pub async fn create_target_hasher( task: &Task, passthrough_args: &[String], ) -> Result { - let vcs = workspace.detect_vcs()?; + let vcs = &workspace.vcs; let globset = task.create_globset()?; let mut hasher = TargetHasher::new(workspace.config.node.version.clone()); diff --git a/crates/workspace/src/vcs/git.rs b/crates/workspace/src/vcs/git.rs index ff900936da0..0e361b7f74f 100644 --- a/crates/workspace/src/vcs/git.rs +++ b/crates/workspace/src/vcs/git.rs @@ -5,10 +5,13 @@ use ignore::gitignore::{Gitignore, GitignoreBuilder}; use moon_utils::fs; use moon_utils::process::{output_to_string, output_to_trimmed_string, Command}; use regex::Regex; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio::sync::RwLock; pub struct Git { + cache: Arc>>, default_branch: String, ignore: Option, working_dir: PathBuf, @@ -30,6 +33,7 @@ impl Git { } Ok(Git { + cache: Arc::new(RwLock::new(HashMap::new())), default_branch: String::from(default_branch), ignore, working_dir: working_dir.to_path_buf(), @@ -71,13 +75,30 @@ impl Git { } async fn run_command(&self, command: &mut Command, trim: bool) -> VcsResult { - let output = command.exec_capture_output().await?; + let (cache_key, _) = command.get_command_line(); + + // Read first before locking with a write + { + let cache = self.cache.read().await; - if trim { - return Ok(output_to_trimmed_string(&output.stdout)); + if cache.contains_key(&cache_key) { + return Ok(cache.get(&cache_key).unwrap().clone()); + } } - Ok(output_to_string(&output.stdout)) + // Otherwise lock and calculate a new value to write + let mut cache = self.cache.write().await; + let output = command.exec_capture_output().await?; + + let value = if trim { + output_to_trimmed_string(&output.stdout) + } else { + output_to_string(&output.stdout) + }; + + cache.insert(cache_key.to_owned(), value.clone()); + + Ok(value) } } @@ -147,6 +168,7 @@ impl Vcs for Git { true, ) .await?; + let mut map = BTreeMap::new(); if output.is_empty() { diff --git a/crates/workspace/src/vcs/svn.rs b/crates/workspace/src/vcs/svn.rs index e3bd23f0bb5..7da3e867210 100644 --- a/crates/workspace/src/vcs/svn.rs +++ b/crates/workspace/src/vcs/svn.rs @@ -3,14 +3,17 @@ use async_trait::async_trait; use moon_utils::fs; use moon_utils::process::{output_to_string, output_to_trimmed_string, Command}; use regex::Regex; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fs::metadata; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::SystemTime; +use tokio::sync::RwLock; // TODO: This code hasn't been tested yet and may not be accurate! pub struct Svn { + cache: Arc>>, default_branch: String, working_dir: PathBuf, } @@ -18,6 +21,7 @@ pub struct Svn { impl Svn { pub fn new(default_branch: &str, working_dir: &Path) -> Self { Svn { + cache: Arc::new(RwLock::new(HashMap::new())), default_branch: String::from(default_branch), working_dir: working_dir.to_path_buf(), } @@ -103,13 +107,30 @@ impl Svn { } async fn run_command(&self, command: &mut Command, trim: bool) -> VcsResult { - let output = command.exec_capture_output().await?; + let (cache_key, _) = command.get_command_line(); + + // Read first before locking with a write + { + let cache = self.cache.read().await; - if trim { - return Ok(output_to_trimmed_string(&output.stdout)); + if cache.contains_key(&cache_key) { + return Ok(cache.get(&cache_key).unwrap().clone()); + } } - Ok(output_to_string(&output.stdout)) + // Otherwise lock and calculate a new value to write + let mut cache = self.cache.write().await; + let output = command.exec_capture_output().await?; + + let value = if trim { + output_to_trimmed_string(&output.stdout) + } else { + output_to_string(&output.stdout) + }; + + cache.insert(cache_key.to_owned(), value.clone()); + + Ok(value) } } diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index c49430a641d..50c33571778 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -145,6 +145,9 @@ pub struct Workspace { /// The root `tsconfig.json`. pub tsconfig_json: Option, + /// Configured version control system. + pub vcs: Box, + /// The current working directory. pub working_dir: PathBuf, } @@ -178,6 +181,7 @@ impl Workspace { let toolchain = Toolchain::create(&root_dir, &config).await?; let projects = ProjectGraph::create(&root_dir, project_config, &config.projects, &cache).await?; + let vcs = VcsManager::load(&config, &root_dir)?; Ok(Workspace { cache, @@ -187,12 +191,8 @@ impl Workspace { root: root_dir, toolchain, tsconfig_json, + vcs, working_dir, }) } - - /// Detect the version control system currently being used. - pub fn detect_vcs(&self) -> Result, WorkspaceError> { - VcsManager::load(&self.config, &self.working_dir) - } } diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index d4a3ff2d3f4..1d471de6b2a 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Unreleased + +#### 🚀 Updates + +- Added caching to our VCS layer which should greatly reduce the amount of `git` commands being + executed. + ### 0.4.1 #### 🐞 Fixes