From 9423a762d92cb5dfef23f0e9a7a3cd9b0490146b Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Mon, 2 Oct 2023 15:26:36 -0700 Subject: [PATCH 01/22] Start on new graph. --- Cargo.lock | 17 +++ nextgen/action-graph/Cargo.toml | 22 ++++ .../action-graph/src/action_graph_builder.rs | 106 ++++++++++++++++++ nextgen/action-graph/src/action_node.rs | 35 ++++++ nextgen/action-graph/src/lib.rs | 5 + 5 files changed, 185 insertions(+) create mode 100644 nextgen/action-graph/Cargo.toml create mode 100644 nextgen/action-graph/src/action_graph_builder.rs create mode 100644 nextgen/action-graph/src/action_node.rs create mode 100644 nextgen/action-graph/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index e1e0b9db207..9e47f77e53a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3026,6 +3026,23 @@ dependencies = [ "serde", ] +[[package]] +name = "moon_action_graph" +version = "0.1.0" +dependencies = [ + "miette", + "moon_common", + "moon_platform", + "moon_platform_runtime", + "moon_project", + "moon_project_graph", + "moon_task", + "petgraph", + "rustc-hash", + "thiserror", + "tracing", +] + [[package]] name = "moon_action_pipeline" version = "0.1.0" diff --git a/nextgen/action-graph/Cargo.toml b/nextgen/action-graph/Cargo.toml new file mode 100644 index 00000000000..b062c1f3aaf --- /dev/null +++ b/nextgen/action-graph/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "moon_action_graph" +version = "0.1.0" +edition = "2021" +license = "MIT" +description = "Dependency graph for actions (tasks)." +homepage = "https://moonrepo.dev/moon" +repository = "https://github.com/moonrepo/moon" + +[dependencies] +moon_common = { path = "../common" } +# TODO remove +moon_platform = { path = "../../crates/core/platform" } +moon_platform_runtime = { path = "../platform-runtime" } +moon_project = { path = "../project" } +moon_project_graph = { path = "../project-graph" } +moon_task = { path = "../task" } +miette = { workspace = true } +petgraph = { workspace = true, features = ["stable_graph"] } +rustc-hash = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs new file mode 100644 index 00000000000..b09fc4fb625 --- /dev/null +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -0,0 +1,106 @@ +use crate::action_node::ActionNode; +use moon_common::color; +use moon_platform::{PlatformManager, Runtime}; +use moon_project::Project; +use moon_project_graph::ProjectGraph; +use moon_task::Task; +use petgraph::prelude::*; +use rustc_hash::FxHashMap; +use tracing::debug; + +pub struct ActionGraphBuilder<'app> { + graph: StableGraph, + indices: FxHashMap, + project_graph: &'app ProjectGraph, +} + +impl<'app> ActionGraphBuilder<'app> { + pub fn get_index_from_node(&self, node: &ActionNode) -> Option<&NodeIndex> { + self.indices.get(node) + } + + pub fn get_target_runtime(&mut self, project: &Project, task: Option<&Task>) -> Runtime { + if let Some(platform) = PlatformManager::read().find(|p| match task { + Some(task) => p.matches(&task.platform, None), + None => p.matches(&project.language.clone().into(), None), + }) { + return platform.get_runtime_from_config(Some(&project.config)); + } + + Runtime::system() + } + + // ACTIONS + + pub fn setup_tool(&mut self, runtime: &Runtime) -> NodeIndex { + let node = ActionNode::SetupTool { + runtime: runtime.to_owned(), + }; + + if let Some(index) = self.get_index_from_node(&node) { + return *index; + } + + let sync_workspace_index = self.sync_workspace(); + let index = self.insert_node(node); + + self.graph.add_edge(index, sync_workspace_index, ()); + + index + } + + pub fn sync_project(&mut self, project: &Project) -> miette::Result { + let runtime = self.get_target_runtime(project, None); + + let node = ActionNode::SyncProject { + project: project.id.clone(), + runtime: runtime.clone(), + }; + + if let Some(index) = self.get_index_from_node(&node) { + return Ok(*index); + } + + // Syncing depends on the language's tool to be installed + let sync_workspace_index = self.sync_workspace(); + let setup_tool_index = self.setup_tool(&runtime); + let index = self.insert_node(node); + + self.graph.add_edge(index, sync_workspace_index, ()); + self.graph.add_edge(index, setup_tool_index, ()); + + // And we should also depend on other projects + for dep_project_id in self.project_graph.dependencies_of(project)? { + let dep_project = self.project_graph.get(dep_project_id)?; + let dep_project_index = self.sync_project(&dep_project)?; + + if index != dep_project_index { + self.graph.add_edge(index, dep_project_index, ()); + } + } + + Ok(index) + } + + pub fn sync_workspace(&mut self) -> NodeIndex { + let node = ActionNode::SyncWorkspace; + + if let Some(index) = self.get_index_from_node(&node) { + return *index; + } + + self.insert_node(node) + } + + // PRIVATE + + fn insert_node(&mut self, node: ActionNode) -> NodeIndex { + debug!("Adding {} to graph", color::muted_light(node.label())); + + let index = self.graph.add_node(node.clone()); + + self.indices.insert(node, index); + + index + } +} diff --git a/nextgen/action-graph/src/action_node.rs b/nextgen/action-graph/src/action_node.rs new file mode 100644 index 00000000000..70bea6bd6f2 --- /dev/null +++ b/nextgen/action-graph/src/action_node.rs @@ -0,0 +1,35 @@ +use std::hash::{Hash, Hasher}; + +use moon_common::Id; +use moon_platform_runtime::Runtime; + +#[derive(Clone, Debug, Eq)] +pub enum ActionNode { + /// Sync the entire moon workspace. + /// Install system dependencies. + SyncWorkspace, + + /// Setup a tool + version for the provided platform. + SetupTool { runtime: Runtime }, + + /// Sync a project with language specific semantics. + SyncProject { project: Id, runtime: Runtime }, +} + +impl ActionNode { + pub fn label(&self) -> String { + String::new() + } +} + +impl PartialEq for ActionNode { + fn eq(&self, other: &Self) -> bool { + self.label() == other.label() + } +} + +impl Hash for ActionNode { + fn hash(&self, state: &mut H) { + self.label().hash(state); + } +} diff --git a/nextgen/action-graph/src/lib.rs b/nextgen/action-graph/src/lib.rs new file mode 100644 index 00000000000..cb43a04dedb --- /dev/null +++ b/nextgen/action-graph/src/lib.rs @@ -0,0 +1,5 @@ +mod action_graph_builder; +mod action_node; + +pub use action_graph_builder::*; +pub use action_node::*; From 9b3897ba5e4035b19b1f072ef728be0863254078 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Mon, 2 Oct 2023 16:23:36 -0700 Subject: [PATCH 02/22] More graph work. --- Cargo.lock | 1 + nextgen/action-graph/Cargo.toml | 1 + .../action-graph/src/action_graph_builder.rs | 263 +++++++++++++++++- nextgen/action-graph/src/action_node.rs | 32 ++- nextgen/task/src/lib.rs | 2 +- 5 files changed, 285 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e47f77e53a..2b48cd78465 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3036,6 +3036,7 @@ dependencies = [ "moon_platform_runtime", "moon_project", "moon_project_graph", + "moon_query", "moon_task", "petgraph", "rustc-hash", diff --git a/nextgen/action-graph/Cargo.toml b/nextgen/action-graph/Cargo.toml index b062c1f3aaf..75c79bccc97 100644 --- a/nextgen/action-graph/Cargo.toml +++ b/nextgen/action-graph/Cargo.toml @@ -15,6 +15,7 @@ moon_platform_runtime = { path = "../platform-runtime" } moon_project = { path = "../project" } moon_project_graph = { path = "../project-graph" } moon_task = { path = "../task" } +moon_query = { path = "../query" } miette = { workspace = true } petgraph = { workspace = true, features = ["stable_graph"] } rustc-hash = { workspace = true } diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs index b09fc4fb625..849d88c2bd5 100644 --- a/nextgen/action-graph/src/action_graph_builder.rs +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -1,37 +1,282 @@ use crate::action_node::ActionNode; -use moon_common::color; +use moon_common::{color, path::WorkspaceRelativePathBuf}; use moon_platform::{PlatformManager, Runtime}; use moon_project::Project; use moon_project_graph::ProjectGraph; -use moon_task::Task; +use moon_query::{build_query, Criteria}; +use moon_task::{Target, TargetError, TargetLocator, TargetScope, Task}; use petgraph::prelude::*; -use rustc_hash::FxHashMap; -use tracing::debug; +use rustc_hash::{FxHashMap, FxHashSet}; +use tracing::{debug, trace}; + +type TouchedFilePaths = FxHashSet; + +// TODO: run task dependents pub struct ActionGraphBuilder<'app> { graph: StableGraph, indices: FxHashMap, project_graph: &'app ProjectGraph, + query: Option, } impl<'app> ActionGraphBuilder<'app> { + pub fn new(project_graph: &'app ProjectGraph) -> miette::Result { + Ok(ActionGraphBuilder { + graph: StableGraph::new(), + indices: FxHashMap::default(), + project_graph, + query: None, + }) + } + pub fn get_index_from_node(&self, node: &ActionNode) -> Option<&NodeIndex> { self.indices.get(node) } - pub fn get_target_runtime(&mut self, project: &Project, task: Option<&Task>) -> Runtime { + pub fn get_runtime( + &mut self, + project: &Project, + task: Option<&Task>, + allow_override: bool, + ) -> Runtime { if let Some(platform) = PlatformManager::read().find(|p| match task { Some(task) => p.matches(&task.platform, None), None => p.matches(&project.language.clone().into(), None), }) { - return platform.get_runtime_from_config(Some(&project.config)); + return platform.get_runtime_from_config(if allow_override { + Some(&project.config) + } else { + None + }); } Runtime::system() } + pub fn set_query(&mut self, input: &str) -> miette::Result<()> { + self.query = Some(build_query(input)?); + + Ok(()) + } + // ACTIONS + pub fn install_deps( + &mut self, + runtime: &Runtime, + project: &Project, + ) -> miette::Result { + let mut in_project = false; + + // If project is NOT in the package manager workspace, then we should + // install dependencies in the project, not the workspace root. + if let Ok(platform) = PlatformManager::read().get(project.language.clone()) { + if !platform.is_project_in_dependency_workspace(project.source.as_str())? { + in_project = true; + + debug!( + "Project {} not within dependency manager workspace, dependencies will be installed within the project instead of the root", + color::id(&project.id), + ); + } + } + + let node = if in_project { + ActionNode::InstallProjectDeps { + project: project.id.to_owned(), + runtime: runtime.to_owned(), + } + } else { + ActionNode::InstallDeps { + runtime: self.get_runtime(project, None, false), + } + }; + + if let Some(index) = self.get_index_from_node(&node) { + return Ok(*index); + } + + // Before we install deps, we must ensure the language has been installed + let setup_tool_index = self.setup_tool(node.get_runtime()); + let index = self.insert_node(node); + + self.graph.add_edge(index, setup_tool_index, ()); + + Ok(index) + } + + pub fn run_task( + &mut self, + project: &Project, + task: &Task, + touched_files: Option<&TouchedFilePaths>, + ) -> miette::Result> { + let node = ActionNode::RunTask { + interactive: task.is_interactive(), + persistent: task.is_persistent(), + runtime: self.get_runtime(project, Some(task), true), + target: task.target.to_owned(), + }; + + if let Some(index) = self.get_index_from_node(&node) { + return Ok(Some(*index)); + } + + // Compare against touched files if provided + if let Some(touched) = touched_files { + if !task.is_affected(touched)? { + trace!( + "Task {} not affected based on touched files, skipping", + color::label(&task.target), + ); + + return Ok(None); + } + } + + // We should install deps & sync projects *before* running targets + let install_deps_index = self.install_deps(node.get_runtime(), project)?; + let sync_project_index = self.sync_project(project)?; + let index = self.insert_node(node); + + self.graph.add_edge(index, install_deps_index, ()); + self.graph.add_edge(index, sync_project_index, ()); + + // And we also need to create edges for task dependencies + if !task.deps.is_empty() { + trace!( + deps = ?task.deps.iter().map(|d| d.as_str()).collect::>(), + "Adding dependencies for task {}", + color::label(&task.target), + ); + + // We don't pass touched files to dependencies, because if the parent + // task is affected/going to run, then so should all of these! + for dep_index in self.run_task_dependencies(task, None)? { + self.graph.add_edge(index, dep_index, ()); + } + } + + Ok(Some(index)) + } + + pub fn run_task_dependencies( + &mut self, + task: &Task, + touched_files: Option<&TouchedFilePaths>, + ) -> miette::Result> { + let parallel = task.options.run_deps_in_parallel; + let mut indices = vec![]; + let mut previous_target_index = None; + + for dep_target in &task.deps { + let (_, dep_indices) = self.run_task_by_target(dep_target, touched_files)?; + + for dep_index in dep_indices { + // When parallel, parent depends on child + if parallel { + indices.push(dep_index); + + // When serial, next child depends on previous child + } else if let Some(prev) = previous_target_index { + self.graph.add_edge(dep_index, prev, ()); + } + + previous_target_index = Some(dep_index); + } + } + + if !parallel { + indices.push(previous_target_index.unwrap()); + } + + Ok(indices) + } + + pub fn run_task_by_target>( + &mut self, + target: T, + touched_files: Option<&TouchedFilePaths>, + ) -> miette::Result<(FxHashSet, FxHashSet)> { + let target = target.as_ref(); + let mut inserted_targets = FxHashSet::default(); + let mut inserted_indices = FxHashSet::default(); + + match &target.scope { + // :task + TargetScope::All => { + let mut projects = vec![]; + + if let Some(all_query) = &self.query { + projects.extend(self.project_graph.query(all_query)?); + } else { + projects.extend(self.project_graph.get_all()?); + }; + + for project in projects { + // Don't error if the task does not exist + if let Ok(task) = project.get_task(&target.task_id) { + if let Some(index) = self.run_task(&project, task, touched_files)? { + inserted_targets.insert(task.target.clone()); + inserted_indices.insert(index); + } + } + } + } + // ^:task + TargetScope::Deps => { + return Err(TargetError::NoDepsInRunContext.into()); + } + // project:task + TargetScope::Project(project_locator) => { + let project = self.project_graph.get(project_locator)?; + let task = project.get_task(&target.task_id)?; + + if let Some(index) = self.run_task(&project, task, touched_files)? { + inserted_targets.insert(task.target.to_owned()); + inserted_indices.insert(index); + } + } + // #tag:task + TargetScope::Tag(tag) => { + let projects = self + .project_graph + .query(build_query(format!("tag={}", tag))?)?; + + for project in projects { + // Don't error if the task does not exist + if let Ok(task) = project.get_task(&target.task_id) { + if let Some(index) = self.run_task(&project, task, touched_files)? { + inserted_targets.insert(task.target.clone()); + inserted_indices.insert(index); + } + } + } + } + // ~:task + TargetScope::OwnSelf => { + return Err(TargetError::NoSelfInRunContext.into()); + } + }; + + Ok((inserted_targets, inserted_indices)) + } + + pub fn run_task_by_target_locator>( + &mut self, + target_locator: T, + touched_files: Option<&TouchedFilePaths>, + ) -> miette::Result<(FxHashSet, FxHashSet)> { + match target_locator.as_ref() { + TargetLocator::Qualified(target) => self.run_task_by_target(target, touched_files), + TargetLocator::TaskFromWorkingDir(task_id) => self.run_task_by_target( + Target::new(&self.project_graph.get_from_path(None)?.id, task_id)?, + touched_files, + ), + } + } + pub fn setup_tool(&mut self, runtime: &Runtime) -> NodeIndex { let node = ActionNode::SetupTool { runtime: runtime.to_owned(), @@ -50,11 +295,9 @@ impl<'app> ActionGraphBuilder<'app> { } pub fn sync_project(&mut self, project: &Project) -> miette::Result { - let runtime = self.get_target_runtime(project, None); - let node = ActionNode::SyncProject { project: project.id.clone(), - runtime: runtime.clone(), + runtime: self.get_runtime(project, None, true), }; if let Some(index) = self.get_index_from_node(&node) { @@ -63,7 +306,7 @@ impl<'app> ActionGraphBuilder<'app> { // Syncing depends on the language's tool to be installed let sync_workspace_index = self.sync_workspace(); - let setup_tool_index = self.setup_tool(&runtime); + let setup_tool_index = self.setup_tool(node.get_runtime()); let index = self.insert_node(node); self.graph.add_edge(index, sync_workspace_index, ()); diff --git a/nextgen/action-graph/src/action_node.rs b/nextgen/action-graph/src/action_node.rs index 70bea6bd6f2..a4884fdcc8b 100644 --- a/nextgen/action-graph/src/action_node.rs +++ b/nextgen/action-graph/src/action_node.rs @@ -2,21 +2,47 @@ use std::hash::{Hash, Hasher}; use moon_common::Id; use moon_platform_runtime::Runtime; +use moon_task::Target; #[derive(Clone, Debug, Eq)] pub enum ActionNode { - /// Sync the entire moon workspace. - /// Install system dependencies. - SyncWorkspace, + /// Install tool dependencies in the workspace root. + InstallDeps { runtime: Runtime }, + + /// Install tool dependencies in the project root. + InstallProjectDeps { project: Id, runtime: Runtime }, + + /// Run a project's task. + RunTask { + interactive: bool, // Interactively with stdin + persistent: bool, // Never terminates + runtime: Runtime, + target: Target, + }, /// Setup a tool + version for the provided platform. SetupTool { runtime: Runtime }, /// Sync a project with language specific semantics. SyncProject { project: Id, runtime: Runtime }, + + /// Sync the entire moon workspace. + /// Install system dependencies. + SyncWorkspace, } impl ActionNode { + pub fn get_runtime(&self) -> &Runtime { + match self { + ActionNode::InstallDeps { runtime } => runtime, + ActionNode::InstallProjectDeps { runtime, .. } => runtime, + ActionNode::RunTask { runtime, .. } => runtime, + ActionNode::SetupTool { runtime } => runtime, + ActionNode::SyncProject { runtime, .. } => runtime, + ActionNode::SyncWorkspace => unreachable!(), + } + } + pub fn label(&self) -> String { String::new() } diff --git a/nextgen/task/src/lib.rs b/nextgen/task/src/lib.rs index 6ad258d2350..7b3bb75e447 100644 --- a/nextgen/task/src/lib.rs +++ b/nextgen/task/src/lib.rs @@ -2,6 +2,6 @@ mod task; mod task_options; pub use moon_config::{TaskConfig, TaskOptionsConfig, TaskType}; -pub use moon_target::{Target, TargetScope}; +pub use moon_target::*; pub use task::*; pub use task_options::*; From be84581d9bf5c6bc1dd97d591f45bd7c3ea3ae81 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Mon, 2 Oct 2023 16:23:52 -0700 Subject: [PATCH 03/22] Add caching to lookups. --- nextgen/project-graph/src/project_graph.rs | 74 ++++++++++++---------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/nextgen/project-graph/src/project_graph.rs b/nextgen/project-graph/src/project_graph.rs index c06508c90db..012de71483a 100644 --- a/nextgen/project-graph/src/project_graph.rs +++ b/nextgen/project-graph/src/project_graph.rs @@ -38,6 +38,9 @@ pub struct ProjectNode { pub struct ProjectGraph { pub check_boundaries: bool, + /// Cache of file path lookups, mapped by starting path to project ID (as a string). + fs_cache: OnceMap, + /// Directed-acyclic graph (DAG) of non-expanded projects and their dependencies. graph: GraphType, @@ -67,6 +70,7 @@ impl ProjectGraph { projects: Arc::new(RwLock::new(FxHashMap::default())), working_dir: workspace_root.to_owned(), workspace_root: workspace_root.to_owned(), + fs_cache: OnceMap::new(), query_cache: OnceMap::new(), check_boundaries: false, } @@ -162,36 +166,7 @@ impl ProjectGraph { current_file }; - // Find the deepest matching path in case sub-projects are being used - let mut remaining_length = 1000; // Start with a really fake number - let mut possible_id = String::new(); - - for (id, node) in &self.nodes { - if !file.starts_with(node.source.as_str()) { - continue; - } - - if let Ok(diff) = file.relative_to(node.source.as_str()) { - let diff_comps = diff.components().count(); - - // Exact match, abort - if diff_comps == 0 { - possible_id = id.as_str().to_owned(); - break; - } - - if diff_comps < remaining_length { - remaining_length = diff_comps; - possible_id = id.as_str().to_owned(); - } - } - } - - if possible_id.is_empty() { - return Err(ProjectGraphError::MissingFromPath(file.to_path_buf()).into()); - } - - self.get(&possible_id) + self.get(self.internal_search(file)?) } /// Return a list of IDs for all projects currently within the graph. @@ -312,10 +287,10 @@ impl ProjectGraph { let query = query.as_ref(); let query_input = query .input - .clone() + .as_ref() .expect("Querying the project graph requires a query input string."); - self.query_cache.try_insert(query_input.clone(), |_| { + self.query_cache.try_insert(query_input.to_owned(), |_| { debug!("Querying projects with {}", color::shell(query_input)); let mut project_ids = vec![]; @@ -345,6 +320,41 @@ impl ProjectGraph { }) } + fn internal_search(&self, search: &Path) -> miette::Result<&str> { + self.fs_cache.try_insert(search.to_path_buf(), |_| { + // Find the deepest matching path in case sub-projects are being used + let mut remaining_length = 1000; // Start with a really fake number + let mut possible_id = String::new(); + + for (id, node) in &self.nodes { + if !search.starts_with(node.source.as_str()) { + continue; + } + + if let Ok(diff) = search.relative_to(node.source.as_str()) { + let diff_comps = diff.components().count(); + + // Exact match, abort + if diff_comps == 0 { + possible_id = id.as_str().to_owned(); + break; + } + + if diff_comps < remaining_length { + remaining_length = diff_comps; + possible_id = id.as_str().to_owned(); + } + } + } + + if possible_id.is_empty() { + return Err(ProjectGraphError::MissingFromPath(search.to_path_buf()).into()); + } + + Ok(possible_id) + }) + } + fn resolve_id(&self, alias_or_id: &str) -> Id { Id::raw(if self.nodes.contains_key(alias_or_id) { alias_or_id From d5454edc5ae5366a9a5b6a0a74221ac17aae9d98 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Mon, 2 Oct 2023 16:33:12 -0700 Subject: [PATCH 04/22] Add node labels. --- nextgen/action-graph/src/action_node.rs | 44 ++++++++++++++++++++----- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/nextgen/action-graph/src/action_node.rs b/nextgen/action-graph/src/action_node.rs index a4884fdcc8b..92100722289 100644 --- a/nextgen/action-graph/src/action_node.rs +++ b/nextgen/action-graph/src/action_node.rs @@ -4,7 +4,7 @@ use moon_common::Id; use moon_platform_runtime::Runtime; use moon_task::Target; -#[derive(Clone, Debug, Eq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub enum ActionNode { /// Install tool dependencies in the workspace root. InstallDeps { runtime: Runtime }, @@ -44,13 +44,41 @@ impl ActionNode { } pub fn label(&self) -> String { - String::new() - } -} - -impl PartialEq for ActionNode { - fn eq(&self, other: &Self) -> bool { - self.label() == other.label() + match self { + Self::InstallDeps { runtime } => { + format!("Install{runtime}Deps({})", runtime.requirement) + } + Self::InstallProjectDeps { runtime, project } => { + format!( + "Install{runtime}DepsInProject({}, {project})", + runtime.requirement + ) + } + Self::RunTask { + interactive, + persistent, + target, + .. + } => { + format!( + "Run{}Task({target})", + if *persistent { + "Persistent" + } else if *interactive { + "Interactive" + } else { + "" + } + ) + } + Self::SetupTool { runtime } => { + format!("Setup{runtime}Tool({})", runtime.requirement) + } + Self::SyncProject { runtime, project } => { + format!("Sync{runtime}Project({project})") + } + Self::SyncWorkspace => "SyncWorkspace".into(), + } } } From e896b5e39fa21789e8a52b0d9027202b0b406d2b Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Mon, 2 Oct 2023 18:41:45 -0700 Subject: [PATCH 05/22] Add graph. --- nextgen/action-graph/src/action_graph.rs | 86 +++++++++++++++++++ .../action-graph/src/action_graph_builder.rs | 5 ++ .../action-graph/src/action_graph_error.rs | 12 +++ nextgen/action-graph/src/action_node.rs | 3 +- nextgen/action-graph/src/lib.rs | 4 + 5 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 nextgen/action-graph/src/action_graph.rs create mode 100644 nextgen/action-graph/src/action_graph_error.rs diff --git a/nextgen/action-graph/src/action_graph.rs b/nextgen/action-graph/src/action_graph.rs new file mode 100644 index 00000000000..6bb9456334b --- /dev/null +++ b/nextgen/action-graph/src/action_graph.rs @@ -0,0 +1,86 @@ +use crate::action_graph_error::ActionGraphError; +use crate::action_node::ActionNode; +use petgraph::dot::{Config, Dot}; +use petgraph::prelude::*; +use rustc_hash::FxHashMap; + +pub struct ActionGraph { + graph: StableGraph, + indices: FxHashMap, +} + +impl ActionGraph { + pub fn new( + graph: StableGraph, + indices: FxHashMap, + ) -> Self { + ActionGraph { graph, indices } + } + + pub fn is_empty(&self) -> bool { + self.get_node_count() == 0 + } + + pub fn get_index_from_node(&self, node: &ActionNode) -> Option<&NodeIndex> { + self.indices.get(node) + } + + pub fn get_node_count(&self) -> usize { + self.graph.node_count() + } + + pub fn get_node_from_index(&self, index: &NodeIndex) -> Option<&ActionNode> { + self.graph.node_weight(*index) + } + + pub fn create_queue(&self) -> miette::Result<()> { + self.detect_cycle()?; + + Ok(()) + } + + pub fn to_dot(&self) -> String { + let graph = self.graph.map(|_, n| n.label(), |_, e| e); + + let dot = Dot::with_attr_getters( + &graph, + &[Config::EdgeNoLabel, Config::NodeNoLabel], + &|_, e| { + if e.source().index() == 0 { + String::from("arrowhead=none") + } else { + String::from("arrowhead=box, arrowtail=box") + } + }, + &|_, n| { + format!( + "label=\"{}\" style=filled, shape=oval, fillcolor=gray, fontcolor=black", + n.1 + ) + }, + ); + + format!("{dot:?}") + } + + pub fn to_labeled_graph(&self) -> StableGraph { + self.graph.map(|_, n| n.label(), |_, _| String::new()) + } + + fn detect_cycle(&self) -> miette::Result<()> { + let scc = petgraph::algo::kosaraju_scc(&self.graph); + + // The cycle is always the last sequence in the list + let Some(cycle) = scc.last() else { + return Err(ActionGraphError::CycleDetected("(unknown)".into()).into()); + }; + + let path = cycle + .iter() + .filter_map(|i| self.get_node_from_index(i).map(|n| n.label())) + .collect::>() + .join(" → "); + + Err(ActionGraphError::CycleDetected(path).into()) + } +} diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs index 849d88c2bd5..8d9ac50f85f 100644 --- a/nextgen/action-graph/src/action_graph_builder.rs +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -1,3 +1,4 @@ +use crate::action_graph::ActionGraph; use crate::action_node::ActionNode; use moon_common::{color, path::WorkspaceRelativePathBuf}; use moon_platform::{PlatformManager, Runtime}; @@ -30,6 +31,10 @@ impl<'app> ActionGraphBuilder<'app> { }) } + pub fn build(self) -> miette::Result { + Ok(ActionGraph::new(self.graph, self.indices)) + } + pub fn get_index_from_node(&self, node: &ActionNode) -> Option<&NodeIndex> { self.indices.get(node) } diff --git a/nextgen/action-graph/src/action_graph_error.rs b/nextgen/action-graph/src/action_graph_error.rs new file mode 100644 index 00000000000..581bbdff52f --- /dev/null +++ b/nextgen/action-graph/src/action_graph_error.rs @@ -0,0 +1,12 @@ +use miette::Diagnostic; +use moon_common::{Style, Stylize}; +use thiserror::Error; + +#[derive(Error, Debug, Diagnostic)] +pub enum ActionGraphError { + #[error("A dependency cycle has been detected for {}.", .0.style(Style::Label))] + CycleDetected(String), + + #[error("Unknown node {0} found in action graph. How did this get here?")] + UnknownNode(usize), +} diff --git a/nextgen/action-graph/src/action_node.rs b/nextgen/action-graph/src/action_node.rs index 92100722289..17753ebbcbb 100644 --- a/nextgen/action-graph/src/action_node.rs +++ b/nextgen/action-graph/src/action_node.rs @@ -1,8 +1,7 @@ -use std::hash::{Hash, Hasher}; - use moon_common::Id; use moon_platform_runtime::Runtime; use moon_task::Target; +use std::hash::{Hash, Hasher}; #[derive(Clone, Debug, Eq, PartialEq)] pub enum ActionNode { diff --git a/nextgen/action-graph/src/lib.rs b/nextgen/action-graph/src/lib.rs index cb43a04dedb..913205c5436 100644 --- a/nextgen/action-graph/src/lib.rs +++ b/nextgen/action-graph/src/lib.rs @@ -1,5 +1,9 @@ +mod action_graph; mod action_graph_builder; +mod action_graph_error; mod action_node; +pub use action_graph::*; pub use action_graph_builder::*; +pub use action_graph_error::*; pub use action_node::*; From 6d9b77365c80e064e73b4e464dbc7f75e78ccbc6 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Tue, 3 Oct 2023 12:47:51 -0700 Subject: [PATCH 06/22] Start on topo and tests. --- Cargo.lock | 1 + nextgen/action-graph/Cargo.toml | 3 + nextgen/action-graph/src/action_graph.rs | 130 ++++++++++++++---- .../action-graph/src/action_graph_builder.rs | 8 +- nextgen/action-graph/src/action_node.rs | 26 +++- .../action-graph/tests/action_graph_test.rs | 36 +++++ ..._action_graph__sync_workspace__graphs.snap | 8 ++ 7 files changed, 174 insertions(+), 38 deletions(-) create mode 100644 nextgen/action-graph/tests/action_graph_test.rs create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_workspace__graphs.snap diff --git a/Cargo.lock b/Cargo.lock index 2b48cd78465..a71d0c1388b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3040,6 +3040,7 @@ dependencies = [ "moon_task", "petgraph", "rustc-hash", + "starbase_sandbox", "thiserror", "tracing", ] diff --git a/nextgen/action-graph/Cargo.toml b/nextgen/action-graph/Cargo.toml index 75c79bccc97..bbf538c88db 100644 --- a/nextgen/action-graph/Cargo.toml +++ b/nextgen/action-graph/Cargo.toml @@ -21,3 +21,6 @@ petgraph = { workspace = true, features = ["stable_graph"] } rustc-hash = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } + +[dev-dependencies] +starbase_sandbox = { workspace = true } diff --git a/nextgen/action-graph/src/action_graph.rs b/nextgen/action-graph/src/action_graph.rs index 6bb9456334b..51db99dd172 100644 --- a/nextgen/action-graph/src/action_graph.rs +++ b/nextgen/action-graph/src/action_graph.rs @@ -1,20 +1,49 @@ use crate::action_graph_error::ActionGraphError; use crate::action_node::ActionNode; +use moon_common::is_test_env; use petgraph::dot::{Config, Dot}; use petgraph::prelude::*; -use rustc_hash::FxHashMap; +use petgraph::visit::{IntoEdgeReferences, IntoNodeReferences}; +use rustc_hash::{FxHashMap, FxHashSet}; +use std::collections::VecDeque; + +pub type GraphType = StableGraph; +pub type IndicesMap = FxHashMap; pub struct ActionGraph { - graph: StableGraph, - indices: FxHashMap, + graph: GraphType, + indices: IndicesMap, + + // States when iterating + queue: VecDeque, + visited: FxHashSet, } impl ActionGraph { - pub fn new( - graph: StableGraph, - indices: FxHashMap, - ) -> Self { - ActionGraph { graph, indices } + pub fn new(graph: GraphType, indices: IndicesMap) -> Self { + ActionGraph { + graph, + indices, + queue: VecDeque::default(), + visited: FxHashSet::default(), + } + } + + pub fn reset_iterator(&mut self) -> miette::Result<()> { + self.detect_cycle()?; + + self.queue.clear(); + self.visited.clear(); + + // Extract root/initial nodes (those without edges) + self.queue.extend(self.graph.node_indices().filter(|&idx| { + self.graph + .neighbors_directed(idx, Incoming) + .next() + .is_none() + })); + + Ok(()) } pub fn is_empty(&self) -> bool { @@ -33,31 +62,38 @@ impl ActionGraph { self.graph.node_weight(*index) } - pub fn create_queue(&self) -> miette::Result<()> { - self.detect_cycle()?; - - Ok(()) - } - pub fn to_dot(&self) -> String { - let graph = self.graph.map(|_, n| n.label(), |_, e| e); + type DotGraph = StableGraph; + + let is_test = is_test_env() || cfg!(debug_assertions); + let graph = self.graph.map(|_, n| n.label(), |_, _| ()); + + let edge = |_: &DotGraph, e: <&DotGraph as IntoEdgeReferences>::EdgeRef| { + if is_test { + String::new() + } else if e.source().index() == 0 { + String::from("arrowhead=none") + } else { + String::from("arrowhead=box, arrowtail=box") + } + }; - let dot = Dot::with_attr_getters( - &graph, - &[Config::EdgeNoLabel, Config::NodeNoLabel], - &|_, e| { - if e.source().index() == 0 { - String::from("arrowhead=none") - } else { - String::from("arrowhead=box, arrowtail=box") - } - }, - &|_, n| { + let node = |_: &DotGraph, n: <&DotGraph as IntoNodeReferences>::NodeRef| { + if is_test { + format!("label=\"{}\" ", n.1) + } else { format!( - "label=\"{}\" style=filled, shape=oval, fillcolor=gray, fontcolor=black", + "label=\"{}\" style=filled, shape=oval, fillcolor=gray, fontcolor=black ", n.1 ) - }, + } + }; + + let dot = Dot::with_attr_getters( + &graph, + &[Config::EdgeNoLabel, Config::NodeNoLabel], + &edge, + &node, ); format!("{dot:?}") @@ -68,8 +104,15 @@ impl ActionGraph { } fn detect_cycle(&self) -> miette::Result<()> { + if self.is_empty() || self.get_node_count() == 1 { + return Ok(()); + } + let scc = petgraph::algo::kosaraju_scc(&self.graph); + // TODO + dbg!(&scc); + // The cycle is always the last sequence in the list let Some(cycle) = scc.last() else { return Err(ActionGraphError::CycleDetected("(unknown)".into()).into()); @@ -84,3 +127,34 @@ impl ActionGraph { Err(ActionGraphError::CycleDetected(path).into()) } } + +// This is based on the `Topo` struct from petgraph! +impl Iterator for ActionGraph { + type Item = ActionNode; + + fn next(&mut self) -> Option { + while let Some(idx) = self.queue.pop_front() { + if self.visited.contains(&idx) { + continue; + } + + self.visited.insert(idx); + + for neighbor in self.graph.neighbors_directed(idx, Direction::Outgoing) { + // Look at each neighbor, and those that only have incoming edges + // from the already ordered list, they are the next to visit. + if self + .graph + .neighbors_directed(neighbor, Direction::Incoming) + .all(|b| self.visited.contains(&b)) + { + self.queue.push_back(neighbor); + } + } + + return self.graph.node_weight(idx).map(|n| n.to_owned()); + } + + None + } +} diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs index 8d9ac50f85f..a30b8cc34b8 100644 --- a/nextgen/action-graph/src/action_graph_builder.rs +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -15,19 +15,19 @@ type TouchedFilePaths = FxHashSet; // TODO: run task dependents pub struct ActionGraphBuilder<'app> { + all_query: Option, graph: StableGraph, indices: FxHashMap, project_graph: &'app ProjectGraph, - query: Option, } impl<'app> ActionGraphBuilder<'app> { pub fn new(project_graph: &'app ProjectGraph) -> miette::Result { Ok(ActionGraphBuilder { + all_query: None, graph: StableGraph::new(), indices: FxHashMap::default(), project_graph, - query: None, }) } @@ -60,7 +60,7 @@ impl<'app> ActionGraphBuilder<'app> { } pub fn set_query(&mut self, input: &str) -> miette::Result<()> { - self.query = Some(build_query(input)?); + self.all_query = Some(build_query(input)?); Ok(()) } @@ -213,7 +213,7 @@ impl<'app> ActionGraphBuilder<'app> { TargetScope::All => { let mut projects = vec![]; - if let Some(all_query) = &self.query { + if let Some(all_query) = &self.all_query { projects.extend(self.project_graph.query(all_query)?); } else { projects.extend(self.project_graph.get_all()?); diff --git a/nextgen/action-graph/src/action_node.rs b/nextgen/action-graph/src/action_node.rs index 17753ebbcbb..adb8de4e25f 100644 --- a/nextgen/action-graph/src/action_node.rs +++ b/nextgen/action-graph/src/action_node.rs @@ -33,12 +33,26 @@ pub enum ActionNode { impl ActionNode { pub fn get_runtime(&self) -> &Runtime { match self { - ActionNode::InstallDeps { runtime } => runtime, - ActionNode::InstallProjectDeps { runtime, .. } => runtime, - ActionNode::RunTask { runtime, .. } => runtime, - ActionNode::SetupTool { runtime } => runtime, - ActionNode::SyncProject { runtime, .. } => runtime, - ActionNode::SyncWorkspace => unreachable!(), + Self::InstallDeps { runtime } => runtime, + Self::InstallProjectDeps { runtime, .. } => runtime, + Self::RunTask { runtime, .. } => runtime, + Self::SetupTool { runtime } => runtime, + Self::SyncProject { runtime, .. } => runtime, + Self::SyncWorkspace => unreachable!(), + } + } + + pub fn is_interactive(&self) -> bool { + match self { + Self::RunTask { interactive, .. } => *interactive, + _ => false, + } + } + + pub fn is_persistent(&self) -> bool { + match self { + Self::RunTask { persistent, .. } => *persistent, + _ => false, } } diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs new file mode 100644 index 00000000000..8ef99205ae4 --- /dev/null +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -0,0 +1,36 @@ +use moon_action_graph::*; +use moon_project_graph::ProjectGraph; +use starbase_sandbox::assert_snapshot; + +fn topo(mut graph: ActionGraph) -> Vec { + let mut nodes = vec![]; + + graph.reset_iterator().unwrap(); + + for node in graph { + nodes.push(node); + } + + nodes +} + +mod action_graph { + use super::*; + + mod sync_workspace { + use super::*; + + #[test] + fn graphs() { + let pg = ProjectGraph::default(); + + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + builder.sync_workspace(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!(topo(graph), vec![ActionNode::SyncWorkspace]); + } + } +} diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_workspace__graphs.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_workspace__graphs.snap new file mode 100644 index 00000000000..35bf4b9bad6 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_workspace__graphs.snap @@ -0,0 +1,8 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] +} + From 8b839e3b1abda9e6cdeaa0e2fcf2db3cb6977a6a Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Tue, 3 Oct 2023 14:49:57 -0700 Subject: [PATCH 07/22] Add basic tests. --- Cargo.lock | 1 + nextgen/action-graph/Cargo.toml | 1 + nextgen/action-graph/src/action_graph.rs | 8 +- .../action-graph/src/action_graph_builder.rs | 46 ++- nextgen/action-graph/src/action_node.rs | 6 +- .../action-graph/tests/action_graph_test.rs | 285 +++++++++++++++++- ...est__action_graph__setup_tool__graphs.snap | 12 + ...aph__setup_tool__graphs_same_platform.snap | 14 + ...t__action_graph__sync_project__graphs.snap | 15 + ..._graph__sync_project__graphs_multiple.snap | 17 ++ ...on_graph__sync_project__graphs_single.snap | 12 + 11 files changed, 400 insertions(+), 17 deletions(-) create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs_same_platform.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap diff --git a/Cargo.lock b/Cargo.lock index a71d0c1388b..d6c530d18ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3032,6 +3032,7 @@ version = "0.1.0" dependencies = [ "miette", "moon_common", + "moon_config", "moon_platform", "moon_platform_runtime", "moon_project", diff --git a/nextgen/action-graph/Cargo.toml b/nextgen/action-graph/Cargo.toml index bbf538c88db..15d2d65b1dd 100644 --- a/nextgen/action-graph/Cargo.toml +++ b/nextgen/action-graph/Cargo.toml @@ -23,4 +23,5 @@ thiserror = { workspace = true } tracing = { workspace = true } [dev-dependencies] +moon_config = { path = "../config" } starbase_sandbox = { workspace = true } diff --git a/nextgen/action-graph/src/action_graph.rs b/nextgen/action-graph/src/action_graph.rs index 51db99dd172..189a938a2fe 100644 --- a/nextgen/action-graph/src/action_graph.rs +++ b/nextgen/action-graph/src/action_graph.rs @@ -30,7 +30,7 @@ impl ActionGraph { } pub fn reset_iterator(&mut self) -> miette::Result<()> { - self.detect_cycle()?; + // self.detect_cycle()?; self.queue.clear(); self.visited.clear(); @@ -38,7 +38,7 @@ impl ActionGraph { // Extract root/initial nodes (those without edges) self.queue.extend(self.graph.node_indices().filter(|&idx| { self.graph - .neighbors_directed(idx, Incoming) + .neighbors_directed(idx, Outgoing) .next() .is_none() })); @@ -140,12 +140,12 @@ impl Iterator for ActionGraph { self.visited.insert(idx); - for neighbor in self.graph.neighbors_directed(idx, Direction::Outgoing) { + for neighbor in self.graph.neighbors_directed(idx, Direction::Incoming) { // Look at each neighbor, and those that only have incoming edges // from the already ordered list, they are the next to visit. if self .graph - .neighbors_directed(neighbor, Direction::Incoming) + .neighbors_directed(neighbor, Direction::Outgoing) .all(|b| self.visited.contains(&b)) { self.queue.push_back(neighbor); diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs index a30b8cc34b8..e5f78ca712a 100644 --- a/nextgen/action-graph/src/action_graph_builder.rs +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -18,15 +18,24 @@ pub struct ActionGraphBuilder<'app> { all_query: Option, graph: StableGraph, indices: FxHashMap, + platform_manager: &'app PlatformManager, project_graph: &'app ProjectGraph, } impl<'app> ActionGraphBuilder<'app> { - pub fn new(project_graph: &'app ProjectGraph) -> miette::Result { + pub fn new(project_graph: &'app ProjectGraph) -> miette::Result { + ActionGraphBuilder::with_platforms(PlatformManager::read(), project_graph) + } + + pub fn with_platforms( + platform_manager: &'app PlatformManager, + project_graph: &'app ProjectGraph, + ) -> miette::Result { Ok(ActionGraphBuilder { all_query: None, graph: StableGraph::new(), indices: FxHashMap::default(), + platform_manager, project_graph, }) } @@ -45,7 +54,7 @@ impl<'app> ActionGraphBuilder<'app> { task: Option<&Task>, allow_override: bool, ) -> Runtime { - if let Some(platform) = PlatformManager::read().find(|p| match task { + if let Some(platform) = self.platform_manager.find(|p| match task { Some(task) => p.matches(&task.platform, None), None => p.matches(&project.language.clone().into(), None), }) { @@ -76,7 +85,7 @@ impl<'app> ActionGraphBuilder<'app> { // If project is NOT in the package manager workspace, then we should // install dependencies in the project, not the workspace root. - if let Ok(platform) = PlatformManager::read().get(project.language.clone()) { + if let Ok(platform) = self.platform_manager.get(project.language.clone()) { if !platform.is_project_in_dependency_workspace(project.source.as_str())? { in_project = true; @@ -309,13 +318,10 @@ impl<'app> ActionGraphBuilder<'app> { return Ok(*index); } - // Syncing depends on the language's tool to be installed - let sync_workspace_index = self.sync_workspace(); + // Syncing requires the language's tool to be installed let setup_tool_index = self.setup_tool(node.get_runtime()); let index = self.insert_node(node); - - self.graph.add_edge(index, sync_workspace_index, ()); - self.graph.add_edge(index, setup_tool_index, ()); + let mut reqs = vec![setup_tool_index]; // And we should also depend on other projects for dep_project_id in self.project_graph.dependencies_of(project)? { @@ -323,10 +329,12 @@ impl<'app> ActionGraphBuilder<'app> { let dep_project_index = self.sync_project(&dep_project)?; if index != dep_project_index { - self.graph.add_edge(index, dep_project_index, ()); + reqs.push(dep_project_index); } } + self.link_requirements(index, reqs); + Ok(index) } @@ -342,11 +350,27 @@ impl<'app> ActionGraphBuilder<'app> { // PRIVATE - fn insert_node(&mut self, node: ActionNode) -> NodeIndex { - debug!("Adding {} to graph", color::muted_light(node.label())); + fn link_requirements(&mut self, index: NodeIndex, reqs: Vec) { + trace!( + index = index.index(), + requires = ?reqs, + "Linking requirements for index" + ); + + for req in reqs { + self.graph.add_edge(index, req, ()); + } + } + fn insert_node(&mut self, node: ActionNode) -> NodeIndex { let index = self.graph.add_node(node.clone()); + debug!( + index = index.index(), + "Adding {} to graph", + color::muted_light(node.label()) + ); + self.indices.insert(node, index); index diff --git a/nextgen/action-graph/src/action_node.rs b/nextgen/action-graph/src/action_node.rs index adb8de4e25f..a27a44631a1 100644 --- a/nextgen/action-graph/src/action_node.rs +++ b/nextgen/action-graph/src/action_node.rs @@ -85,7 +85,11 @@ impl ActionNode { ) } Self::SetupTool { runtime } => { - format!("Setup{runtime}Tool({})", runtime.requirement) + if runtime.platform.is_system() { + format!("SetupSystemTool") + } else { + format!("Setup{runtime}Tool({})", runtime.requirement) + } } Self::SyncProject { runtime, project } => { format!("Sync{runtime}Project({project})") diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index 8ef99205ae4..b73683b2dbe 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -1,6 +1,89 @@ use moon_action_graph::*; -use moon_project_graph::ProjectGraph; +use moon_common::Id; +use moon_config::{DependencyConfig, DependencyScope, DependencySource}; +use moon_platform_runtime::*; +use moon_project::Project; +use moon_project_graph::{GraphType as ProjectGraphType, ProjectGraph, ProjectNode}; +use rustc_hash::FxHashMap; use starbase_sandbox::assert_snapshot; +use std::path::PathBuf; + +fn create_project(id: &str) -> Project { + let mut project = Project::default(); + project.id = Id::raw(id); + project +} + +fn create_project_graph() -> ProjectGraph { + // Create projects + let mut foo = create_project("foo"); + foo.dependencies.insert( + Id::raw("bar"), + DependencyConfig { + id: Id::raw("bar"), + scope: DependencyScope::Production, + source: DependencySource::Explicit, + via: None, + }, + ); + + let mut bar = create_project("bar"); + bar.platform = PlatformType::Node; + + let mut baz = create_project("baz"); + baz.platform = PlatformType::Node; + + let mut qux = create_project("qux"); + qux.platform = PlatformType::Rust; + + // Map nodes and create graph (in order of expected insertion) + let mut nodes = FxHashMap::default(); + let mut graph = ProjectGraphType::new(); + + let bi = graph.add_node(bar); + nodes.insert( + "bar".into(), + ProjectNode { + alias: None, + index: bi, + source: "bar".into(), + }, + ); + + let fi = graph.add_node(foo); + nodes.insert( + "foo".into(), + ProjectNode { + alias: None, + index: fi, + source: "foo".into(), + }, + ); + + graph.add_edge(fi, bi, DependencyScope::Production); + + let zi = graph.add_node(baz); + nodes.insert( + "baz".into(), + ProjectNode { + alias: None, + index: zi, + source: "baz".into(), + }, + ); + + let qi = graph.add_node(qux); + nodes.insert( + "qux".into(), + ProjectNode { + alias: None, + index: qi, + source: "qux".into(), + }, + ); + + ProjectGraph::new(graph, nodes, &PathBuf::from(".")) +} fn topo(mut graph: ActionGraph) -> Vec { let mut nodes = vec![]; @@ -17,6 +100,192 @@ fn topo(mut graph: ActionGraph) -> Vec { mod action_graph { use super::*; + mod setup_tool { + use super::*; + + #[test] + fn graphs() { + let pg = ProjectGraph::default(); + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + let system = Runtime::system(); + let node = Runtime::new( + PlatformType::Node, + RuntimeReq::Toolchain(UnresolvedVersionSpec::Version(Version::new(1, 2, 3))), + ); + + builder.setup_tool(&system); + builder.setup_tool(&node); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { runtime: node }, + ActionNode::SetupTool { runtime: system }, + ] + ); + } + + #[test] + fn graphs_same_platform() { + let pg = ProjectGraph::default(); + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + + let node1 = Runtime::new( + PlatformType::Node, + RuntimeReq::Toolchain(UnresolvedVersionSpec::Version(Version::new(1, 2, 3))), + ); + let node2 = Runtime::new_override( + PlatformType::Node, + RuntimeReq::Toolchain(UnresolvedVersionSpec::Version(Version::new(4, 5, 6))), + ); + let node3 = Runtime::new(PlatformType::Node, RuntimeReq::Global); + + builder.setup_tool(&node1); + builder.setup_tool(&node2); + builder.setup_tool(&node3); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { runtime: node3 }, + ActionNode::SetupTool { runtime: node2 }, + ActionNode::SetupTool { runtime: node1 }, + ] + ); + } + + #[test] + fn ignores_dupes() { + let pg = ProjectGraph::default(); + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + let system = Runtime::system(); + + builder.setup_tool(&system); + builder.setup_tool(&system); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { runtime: system }, + ] + ); + } + } + + mod sync_project { + use super::*; + + #[test] + fn graphs_single() { + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + + let bar = pg.get("bar").unwrap(); + + builder.sync_project(&bar).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: Runtime::system() + } + ] + ); + } + + #[test] + fn graphs_multiple() { + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + + let foo = pg.get("foo").unwrap(); + builder.sync_project(&foo).unwrap(); + + let bar = pg.get("bar").unwrap(); + builder.sync_project(&bar).unwrap(); + + let qux = pg.get("qux").unwrap(); + builder.sync_project(&qux).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("qux"), + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("foo"), + runtime: Runtime::system() + }, + ] + ); + } + + #[test] + fn ignores_dupes() { + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + + let foo = pg.get("foo").unwrap(); + + builder.sync_project(&foo).unwrap(); + builder.sync_project(&foo).unwrap(); + builder.sync_project(&foo).unwrap(); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("foo"), + runtime: Runtime::system() + } + ] + ); + } + } + mod sync_workspace { use super::*; @@ -32,5 +301,19 @@ mod action_graph { assert_snapshot!(graph.to_dot()); assert_eq!(topo(graph), vec![ActionNode::SyncWorkspace]); } + + #[test] + fn ignores_dupes() { + let pg = ProjectGraph::default(); + + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + builder.sync_workspace(); + builder.sync_workspace(); + builder.sync_workspace(); + + let graph = builder.build().unwrap(); + + assert_eq!(topo(graph), vec![ActionNode::SyncWorkspace]); + } } } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs.snap new file mode 100644 index 00000000000..e8e7634758e --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs.snap @@ -0,0 +1,12 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SetupNodeTool(1.2.3)" ] + 1 -> 0 [ ] + 2 -> 0 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs_same_platform.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs_same_platform.snap new file mode 100644 index 00000000000..1c0fe1c6ada --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs_same_platform.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(1.2.3)" ] + 2 [ label="SetupNodeTool(4.5.6)" ] + 3 [ label="SetupNodeTool(global)" ] + 1 -> 0 [ ] + 2 -> 0 [ ] + 3 -> 0 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs.snap new file mode 100644 index 00000000000..46d410389a3 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs.snap @@ -0,0 +1,15 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(foo)" ] + 3 [ label="SyncSystemProject(bar)" ] + 1 -> 0 [ ] + 3 -> 1 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap new file mode 100644 index 00000000000..6f9dda954a9 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap @@ -0,0 +1,17 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(foo)" ] + 3 [ label="SyncSystemProject(bar)" ] + 4 [ label="SyncSystemProject(qux)" ] + 1 -> 0 [ ] + 3 -> 1 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] + 4 -> 1 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap new file mode 100644 index 00000000000..ddfdb147fe1 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap @@ -0,0 +1,12 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(bar)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] +} + From 6d299a2d8d113f23b64c1014bce1b49973b89cac Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Tue, 3 Oct 2023 18:54:39 -0700 Subject: [PATCH 08/22] Add tests. --- Cargo.lock | 3 + nextgen/action-graph/Cargo.toml | 5 + .../action-graph/src/action_graph_builder.rs | 2 +- .../action-graph/src/action_graph_error.rs | 3 - .../action-graph/tests/action_graph_test.rs | 184 +++++++++++++++++- ..._sync_project__graphs_single_with_dep.snap | 15 ++ ..._sync_project__inherits_platform_tool.snap | 16 ++ ...c_project__supports_platform_override.snap | 16 ++ nextgen/platform-runtime/src/lib.rs | 4 + 9 files changed, 239 insertions(+), 9 deletions(-) create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap diff --git a/Cargo.lock b/Cargo.lock index d6c530d18ae..49a814d7452 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3033,13 +3033,16 @@ dependencies = [ "miette", "moon_common", "moon_config", + "moon_node_platform", "moon_platform", "moon_platform_runtime", "moon_project", "moon_project_graph", "moon_query", + "moon_rust_platform", "moon_task", "petgraph", + "proto_core", "rustc-hash", "starbase_sandbox", "thiserror", diff --git a/nextgen/action-graph/Cargo.toml b/nextgen/action-graph/Cargo.toml index 15d2d65b1dd..beb33a4e3d3 100644 --- a/nextgen/action-graph/Cargo.toml +++ b/nextgen/action-graph/Cargo.toml @@ -24,4 +24,9 @@ tracing = { workspace = true } [dev-dependencies] moon_config = { path = "../config" } +proto_core = { workspace = true } starbase_sandbox = { workspace = true } + +# TODO remove +moon_node_platform = { path = "../../crates/node/platform" } +moon_rust_platform = { path = "../../crates/rust/platform" } diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs index e5f78ca712a..3070875e7cb 100644 --- a/nextgen/action-graph/src/action_graph_builder.rs +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -56,7 +56,7 @@ impl<'app> ActionGraphBuilder<'app> { ) -> Runtime { if let Some(platform) = self.platform_manager.find(|p| match task { Some(task) => p.matches(&task.platform, None), - None => p.matches(&project.language.clone().into(), None), + None => p.matches(&project.platform, None), }) { return platform.get_runtime_from_config(if allow_override { Some(&project.config) diff --git a/nextgen/action-graph/src/action_graph_error.rs b/nextgen/action-graph/src/action_graph_error.rs index 581bbdff52f..59e37ddd11e 100644 --- a/nextgen/action-graph/src/action_graph_error.rs +++ b/nextgen/action-graph/src/action_graph_error.rs @@ -6,7 +6,4 @@ use thiserror::Error; pub enum ActionGraphError { #[error("A dependency cycle has been detected for {}.", .0.style(Style::Label))] CycleDetected(String), - - #[error("Unknown node {0} found in action graph. How did this get here?")] - UnknownNode(usize), } diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index b73683b2dbe..b296a7a696f 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -1,12 +1,20 @@ use moon_action_graph::*; use moon_common::Id; -use moon_config::{DependencyConfig, DependencyScope, DependencySource}; +use moon_config::{ + DependencyConfig, DependencyScope, DependencySource, LanguageType, NodeConfig, + ProjectToolchainCommonToolConfig, RustConfig, +}; +use moon_node_platform::NodePlatform; +use moon_platform::PlatformManager; use moon_platform_runtime::*; use moon_project::Project; use moon_project_graph::{GraphType as ProjectGraphType, ProjectGraph, ProjectNode}; +use moon_rust_platform::RustPlatform; +use proto_core::ProtoEnvironment; use rustc_hash::FxHashMap; use starbase_sandbox::assert_snapshot; use std::path::PathBuf; +use std::sync::Arc; fn create_project(id: &str) -> Project { let mut project = Project::default(); @@ -28,12 +36,18 @@ fn create_project_graph() -> ProjectGraph { ); let mut bar = create_project("bar"); + bar.language = LanguageType::JavaScript; bar.platform = PlatformType::Node; let mut baz = create_project("baz"); + baz.language = LanguageType::TypeScript; baz.platform = PlatformType::Node; + baz.config.toolchain.node = Some(ProjectToolchainCommonToolConfig { + version: Some(UnresolvedVersionSpec::Version(Version::new(18, 0, 0))), + }); let mut qux = create_project("qux"); + qux.language = LanguageType::Rust; qux.platform = PlatformType::Rust; // Map nodes and create graph (in order of expected insertion) @@ -85,6 +99,39 @@ fn create_project_graph() -> ProjectGraph { ProjectGraph::new(graph, nodes, &PathBuf::from(".")) } +fn create_platform_manager() -> PlatformManager { + let mut manager = PlatformManager::default(); + let root = PathBuf::from("."); + let proto = Arc::new(ProtoEnvironment::new_testing(&root)); + + manager.register( + PlatformType::Node, + Box::new(NodePlatform::new( + &NodeConfig { + version: Some(UnresolvedVersionSpec::Version(Version::new(20, 0, 0))), + ..Default::default() + }, + &None, + &root, + proto.clone(), + )), + ); + + manager.register( + PlatformType::Rust, + Box::new(RustPlatform::new( + &RustConfig { + version: Some(UnresolvedVersionSpec::Version(Version::new(1, 72, 0))), + ..Default::default() + }, + &root, + proto.clone(), + )), + ); + + manager +} + fn topo(mut graph: ActionGraph) -> Vec { let mut nodes = vec![]; @@ -110,7 +157,7 @@ mod action_graph { let system = Runtime::system(); let node = Runtime::new( PlatformType::Node, - RuntimeReq::Toolchain(UnresolvedVersionSpec::Version(Version::new(1, 2, 3))), + RuntimeReq::with_version(Version::new(1, 2, 3)), ); builder.setup_tool(&system); @@ -136,11 +183,11 @@ mod action_graph { let node1 = Runtime::new( PlatformType::Node, - RuntimeReq::Toolchain(UnresolvedVersionSpec::Version(Version::new(1, 2, 3))), + RuntimeReq::with_version(Version::new(1, 2, 3)), ); let node2 = Runtime::new_override( PlatformType::Node, - RuntimeReq::Toolchain(UnresolvedVersionSpec::Version(Version::new(4, 5, 6))), + RuntimeReq::with_version(Version::new(4, 5, 6)), ); let node3 = Runtime::new(PlatformType::Node, RuntimeReq::Global); @@ -192,7 +239,6 @@ mod action_graph { let mut builder = ActionGraphBuilder::new(&pg).unwrap(); let bar = pg.get("bar").unwrap(); - builder.sync_project(&bar).unwrap(); let graph = builder.build().unwrap(); @@ -213,6 +259,36 @@ mod action_graph { ); } + #[test] + fn graphs_single_with_dep() { + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + + let foo = pg.get("foo").unwrap(); + builder.sync_project(&foo).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("foo"), + runtime: Runtime::system() + } + ] + ); + } + #[test] fn graphs_multiple() { let pg = create_project_graph(); @@ -284,6 +360,104 @@ mod action_graph { ] ); } + + #[test] + fn inherits_platform_tool() { + let pm = create_platform_manager(); + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + + let bar = pg.get("bar").unwrap(); + builder.sync_project(&bar).unwrap(); + + let qux = pg.get("qux").unwrap(); + builder.sync_project(&qux).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::new( + PlatformType::Rust, + RuntimeReq::with_version(Version::new(1, 72, 0)) + ) + }, + ActionNode::SetupTool { + runtime: Runtime::new( + PlatformType::Node, + RuntimeReq::with_version(Version::new(20, 0, 0)) + ) + }, + ActionNode::SyncProject { + project: Id::raw("qux"), + runtime: Runtime::new( + PlatformType::Rust, + RuntimeReq::with_version(Version::new(1, 72, 0)) + ) + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: Runtime::new( + PlatformType::Node, + RuntimeReq::with_version(Version::new(20, 0, 0)) + ) + } + ] + ); + } + + #[test] + fn supports_platform_override() { + let pm = create_platform_manager(); + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + + let bar = pg.get("bar").unwrap(); + builder.sync_project(&bar).unwrap(); + + let baz = pg.get("baz").unwrap(); + builder.sync_project(&baz).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::new_override( + PlatformType::Node, + RuntimeReq::with_version(Version::new(18, 0, 0)) + ) + }, + ActionNode::SetupTool { + runtime: Runtime::new( + PlatformType::Node, + RuntimeReq::with_version(Version::new(20, 0, 0)) + ) + }, + ActionNode::SyncProject { + project: Id::raw("baz"), + runtime: Runtime::new_override( + PlatformType::Node, + RuntimeReq::with_version(Version::new(18, 0, 0)) + ) + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: Runtime::new( + PlatformType::Node, + RuntimeReq::with_version(Version::new(20, 0, 0)) + ) + }, + ] + ); + } } mod sync_workspace { diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap new file mode 100644 index 00000000000..46d410389a3 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap @@ -0,0 +1,15 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(foo)" ] + 3 [ label="SyncSystemProject(bar)" ] + 1 -> 0 [ ] + 3 -> 1 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap new file mode 100644 index 00000000000..c59a9643d06 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap @@ -0,0 +1,16 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="SyncNodeProject(bar)" ] + 3 [ label="SetupRustTool(1.72.0)" ] + 4 [ label="SyncRustProject(qux)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 0 [ ] + 4 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap new file mode 100644 index 00000000000..8f0bbd2e090 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap @@ -0,0 +1,16 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="SyncNodeProject(bar)" ] + 3 [ label="SetupNodeTool(18.0.0)" ] + 4 [ label="SyncNodeProject(baz)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 0 [ ] + 4 -> 3 [ ] +} + diff --git a/nextgen/platform-runtime/src/lib.rs b/nextgen/platform-runtime/src/lib.rs index f6683fae574..b3627dc60a5 100644 --- a/nextgen/platform-runtime/src/lib.rs +++ b/nextgen/platform-runtime/src/lib.rs @@ -12,6 +12,10 @@ pub enum RuntimeReq { } impl RuntimeReq { + pub fn with_version(version: Version) -> Self { + Self::Toolchain(UnresolvedVersionSpec::Version(version)) + } + pub fn is_global(&self) -> bool { matches!(self, Self::Global) } From af8ceb94a6d8b20159a928a26242625c3174651a Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Wed, 4 Oct 2023 15:42:36 -0700 Subject: [PATCH 09/22] Add run task. --- .../action-graph/src/action_graph_builder.rs | 32 +- nextgen/action-graph/src/action_node.rs | 2 +- .../action-graph/tests/action_graph_test.rs | 368 +++++++++++++++--- ...t__action_graph__install_deps__graphs.snap | 12 + ..._if_not_affected_by_touched_files.snap.new | 8 + ..._test__action_graph__run_task__graphs.snap | 17 + ...can_have_a_diff_platform_from_project.snap | 17 + ..._graph__sync_project__graphs_multiple.snap | 14 +- ...on_graph__sync_project__graphs_single.snap | 4 +- ..._sync_project__graphs_single_with_dep.snap | 10 +- ..._sync_project__inherits_platform_tool.snap | 10 +- ...c_project__supports_platform_override.snap | 10 +- 12 files changed, 402 insertions(+), 102 deletions(-) create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__graphs.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__doesnt_graph_if_not_affected_by_touched_files.snap.new create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__graphs.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__task_can_have_a_diff_platform_from_project.snap diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs index 3070875e7cb..914c3a8b3ba 100644 --- a/nextgen/action-graph/src/action_graph_builder.rs +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -49,7 +49,7 @@ impl<'app> ActionGraphBuilder<'app> { } pub fn get_runtime( - &mut self, + &self, project: &Project, task: Option<&Task>, allow_override: bool, @@ -78,8 +78,8 @@ impl<'app> ActionGraphBuilder<'app> { pub fn install_deps( &mut self, - runtime: &Runtime, project: &Project, + task: Option<&Task>, ) -> miette::Result { let mut in_project = false; @@ -90,7 +90,7 @@ impl<'app> ActionGraphBuilder<'app> { in_project = true; debug!( - "Project {} not within dependency manager workspace, dependencies will be installed within the project instead of the root", + "Project {} is not within the dependency manager workspace, dependencies will be installed within the project instead of the root", color::id(&project.id), ); } @@ -99,11 +99,11 @@ impl<'app> ActionGraphBuilder<'app> { let node = if in_project { ActionNode::InstallProjectDeps { project: project.id.to_owned(), - runtime: runtime.to_owned(), + runtime: self.get_runtime(project, task, true), } } else { ActionNode::InstallDeps { - runtime: self.get_runtime(project, None, false), + runtime: self.get_runtime(project, task, false), } }; @@ -115,7 +115,7 @@ impl<'app> ActionGraphBuilder<'app> { let setup_tool_index = self.setup_tool(node.get_runtime()); let index = self.insert_node(node); - self.graph.add_edge(index, setup_tool_index, ()); + self.link_requirements(index, vec![setup_tool_index]); Ok(index) } @@ -150,12 +150,10 @@ impl<'app> ActionGraphBuilder<'app> { } // We should install deps & sync projects *before* running targets - let install_deps_index = self.install_deps(node.get_runtime(), project)?; + let install_deps_index = self.install_deps(project, Some(task))?; let sync_project_index = self.sync_project(project)?; let index = self.insert_node(node); - - self.graph.add_edge(index, install_deps_index, ()); - self.graph.add_edge(index, sync_project_index, ()); + let mut reqs = vec![install_deps_index, sync_project_index]; // And we also need to create edges for task dependencies if !task.deps.is_empty() { @@ -167,11 +165,11 @@ impl<'app> ActionGraphBuilder<'app> { // We don't pass touched files to dependencies, because if the parent // task is affected/going to run, then so should all of these! - for dep_index in self.run_task_dependencies(task, None)? { - self.graph.add_edge(index, dep_index, ()); - } + reqs.extend(self.run_task_dependencies(task, None)?); } + self.link_requirements(index, reqs); + Ok(Some(index)) } @@ -194,7 +192,7 @@ impl<'app> ActionGraphBuilder<'app> { // When serial, next child depends on previous child } else if let Some(prev) = previous_target_index { - self.graph.add_edge(dep_index, prev, ()); + self.link_requirements(dep_index, vec![prev]); } previous_target_index = Some(dep_index); @@ -303,7 +301,7 @@ impl<'app> ActionGraphBuilder<'app> { let sync_workspace_index = self.sync_workspace(); let index = self.insert_node(node); - self.graph.add_edge(index, sync_workspace_index, ()); + self.link_requirements(index, vec![sync_workspace_index]); index } @@ -319,9 +317,9 @@ impl<'app> ActionGraphBuilder<'app> { } // Syncing requires the language's tool to be installed - let setup_tool_index = self.setup_tool(node.get_runtime()); + let sync_workspace_index = self.sync_workspace(); let index = self.insert_node(node); - let mut reqs = vec![setup_tool_index]; + let mut reqs = vec![sync_workspace_index]; // And we should also depend on other projects for dep_project_id in self.project_graph.dependencies_of(project)? { diff --git a/nextgen/action-graph/src/action_node.rs b/nextgen/action-graph/src/action_node.rs index a27a44631a1..a691c58de60 100644 --- a/nextgen/action-graph/src/action_node.rs +++ b/nextgen/action-graph/src/action_node.rs @@ -86,7 +86,7 @@ impl ActionNode { } Self::SetupTool { runtime } => { if runtime.platform.is_system() { - format!("SetupSystemTool") + "SetupSystemTool".into() } else { format!("Setup{runtime}Tool({})", runtime.requirement) } diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index b296a7a696f..ac0fd272e65 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -1,4 +1,7 @@ +#![allow(clippy::disallowed_names)] + use moon_action_graph::*; +use moon_common::path::WorkspaceRelativePathBuf; use moon_common::Id; use moon_config::{ DependencyConfig, DependencyScope, DependencySource, LanguageType, NodeConfig, @@ -10,16 +13,26 @@ use moon_platform_runtime::*; use moon_project::Project; use moon_project_graph::{GraphType as ProjectGraphType, ProjectGraph, ProjectNode}; use moon_rust_platform::RustPlatform; +use moon_task::{Target, Task}; use proto_core::ProtoEnvironment; -use rustc_hash::FxHashMap; +use rustc_hash::{FxHashMap, FxHashSet}; use starbase_sandbox::assert_snapshot; use std::path::PathBuf; use std::sync::Arc; +fn create_task(id: &str, project: &str) -> Task { + Task { + id: Id::raw(id), + target: Target::new(project, id).unwrap(), + ..Task::default() + } +} + fn create_project(id: &str) -> Project { - let mut project = Project::default(); - project.id = Id::raw(id); - project + Project { + id: Id::raw(id), + ..Project::default() + } } fn create_project_graph() -> ProjectGraph { @@ -99,6 +112,20 @@ fn create_project_graph() -> ProjectGraph { ProjectGraph::new(graph, nodes, &PathBuf::from(".")) } +fn create_node_runtime() -> Runtime { + Runtime::new( + PlatformType::Node, + RuntimeReq::with_version(Version::new(20, 0, 0)), + ) +} + +fn create_rust_runtime() -> Runtime { + Runtime::new( + PlatformType::Rust, + RuntimeReq::with_version(Version::new(1, 70, 0)), + ) +} + fn create_platform_manager() -> PlatformManager { let mut manager = PlatformManager::default(); let root = PathBuf::from("."); @@ -121,7 +148,7 @@ fn create_platform_manager() -> PlatformManager { PlatformType::Rust, Box::new(RustPlatform::new( &RustConfig { - version: Some(UnresolvedVersionSpec::Version(Version::new(1, 72, 0))), + version: Some(UnresolvedVersionSpec::Version(Version::new(1, 70, 0))), ..Default::default() }, &root, @@ -147,6 +174,286 @@ fn topo(mut graph: ActionGraph) -> Vec { mod action_graph { use super::*; + mod install_deps { + use super::*; + + #[test] + fn graphs() { + let pm = create_platform_manager(); + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + + let bar = pg.get("bar").unwrap(); + builder.install_deps(&bar, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::InstallDeps { + runtime: create_node_runtime() + } + ] + ); + } + + #[test] + fn ignores_dupes() { + let pm = create_platform_manager(); + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + + let bar = pg.get("bar").unwrap(); + builder.install_deps(&bar, None).unwrap(); + builder.install_deps(&bar, None).unwrap(); + builder.install_deps(&bar, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::InstallDeps { + runtime: create_node_runtime() + } + ] + ); + } + } + + mod run_task { + use super::*; + use starbase_sandbox::pretty_assertions::assert_eq; + + #[test] + fn graphs() { + let pm = create_platform_manager(); + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + + let project = pg.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.platform = PlatformType::Node; + + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: create_node_runtime() + }, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::InstallDeps { + runtime: create_node_runtime() + }, + ActionNode::RunTask { + interactive: false, + persistent: false, + runtime: create_node_runtime(), + target: task.target + } + ] + ); + } + + #[test] + fn ignores_dupes() { + let pm = create_platform_manager(); + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + + let project = pg.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.platform = PlatformType::Node; + + builder.run_task(&project, &task, None).unwrap(); + builder.run_task(&project, &task, None).unwrap(); + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: create_node_runtime() + }, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::InstallDeps { + runtime: create_node_runtime() + }, + ActionNode::RunTask { + interactive: false, + persistent: false, + runtime: create_node_runtime(), + target: task.target + } + ] + ); + } + + #[test] + fn doesnt_graph_if_not_affected_by_touched_files() { + let pm = create_platform_manager(); + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + + let project = pg.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.platform = PlatformType::Node; + + builder + // Empty set works fine, just needs to be some + .run_task(&project, &task, Some(&FxHashSet::default())) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert!(topo(graph).is_empty()); + } + + #[test] + fn graphs_if_affected_by_touched_files() { + let pm = create_platform_manager(); + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + + let file = WorkspaceRelativePathBuf::from("bar/file.js"); + + let project = pg.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.platform = PlatformType::Node; + task.input_files.insert(file.clone()); + + builder + .run_task(&project, &task, Some(&FxHashSet::from_iter([file]))) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert!(!topo(graph).is_empty()); + } + + #[test] + fn task_can_have_a_diff_platform_from_project() { + let pm = create_platform_manager(); + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + + // node + let project = pg.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.platform = PlatformType::Rust; + + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: create_node_runtime() + }, + ActionNode::SetupTool { + runtime: create_rust_runtime() + }, + ActionNode::InstallDeps { + runtime: create_rust_runtime() + }, + ActionNode::RunTask { + interactive: false, + persistent: false, + runtime: create_rust_runtime(), + target: task.target + } + ] + ); + } + + #[test] + fn sets_interactive() { + let pm = create_platform_manager(); + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + + let project = pg.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.options.interactive = true; + + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph).last().unwrap(), + &ActionNode::RunTask { + interactive: true, + persistent: false, + runtime: Runtime::system(), + target: task.target + } + ); + } + + #[test] + fn sets_persistent() { + let pm = create_platform_manager(); + let pg = create_project_graph(); + let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + + let project = pg.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.options.persistent = true; + + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph).last().unwrap(), + &ActionNode::RunTask { + interactive: false, + persistent: true, + runtime: Runtime::system(), + target: task.target + } + ); + } + } + mod setup_tool { use super::*; @@ -248,9 +555,6 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, - ActionNode::SetupTool { - runtime: Runtime::system() - }, ActionNode::SyncProject { project: Id::raw("bar"), runtime: Runtime::system() @@ -274,9 +578,6 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, - ActionNode::SetupTool { - runtime: Runtime::system() - }, ActionNode::SyncProject { project: Id::raw("bar"), runtime: Runtime::system() @@ -310,9 +611,6 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, - ActionNode::SetupTool { - runtime: Runtime::system() - }, ActionNode::SyncProject { project: Id::raw("qux"), runtime: Runtime::system() @@ -346,9 +644,6 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, - ActionNode::SetupTool { - runtime: Runtime::system() - }, ActionNode::SyncProject { project: Id::raw("bar"), runtime: Runtime::system() @@ -380,31 +675,13 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, - ActionNode::SetupTool { - runtime: Runtime::new( - PlatformType::Rust, - RuntimeReq::with_version(Version::new(1, 72, 0)) - ) - }, - ActionNode::SetupTool { - runtime: Runtime::new( - PlatformType::Node, - RuntimeReq::with_version(Version::new(20, 0, 0)) - ) - }, ActionNode::SyncProject { project: Id::raw("qux"), - runtime: Runtime::new( - PlatformType::Rust, - RuntimeReq::with_version(Version::new(1, 72, 0)) - ) + runtime: create_rust_runtime() }, ActionNode::SyncProject { project: Id::raw("bar"), - runtime: Runtime::new( - PlatformType::Node, - RuntimeReq::with_version(Version::new(20, 0, 0)) - ) + runtime: create_node_runtime() } ] ); @@ -429,18 +706,6 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, - ActionNode::SetupTool { - runtime: Runtime::new_override( - PlatformType::Node, - RuntimeReq::with_version(Version::new(18, 0, 0)) - ) - }, - ActionNode::SetupTool { - runtime: Runtime::new( - PlatformType::Node, - RuntimeReq::with_version(Version::new(20, 0, 0)) - ) - }, ActionNode::SyncProject { project: Id::raw("baz"), runtime: Runtime::new_override( @@ -450,10 +715,7 @@ mod action_graph { }, ActionNode::SyncProject { project: Id::raw("bar"), - runtime: Runtime::new( - PlatformType::Node, - RuntimeReq::with_version(Version::new(20, 0, 0)) - ) + runtime: create_node_runtime() }, ] ); diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__graphs.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__graphs.snap new file mode 100644 index 00000000000..10d4bd747e0 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__graphs.snap @@ -0,0 +1,12 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="InstallNodeDeps(20.0.0)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__doesnt_graph_if_not_affected_by_touched_files.snap.new b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__doesnt_graph_if_not_affected_by_touched_files.snap.new new file mode 100644 index 00000000000..c6fb3902fe1 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__doesnt_graph_if_not_affected_by_touched_files.snap.new @@ -0,0 +1,8 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +assertion_line: 296 +expression: graph.to_dot() +--- +digraph { +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__graphs.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__graphs.snap new file mode 100644 index 00000000000..bbb08d9a762 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__graphs.snap @@ -0,0 +1,17 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="InstallNodeDeps(20.0.0)" ] + 3 [ label="SyncNodeProject(bar)" ] + 4 [ label="RunTask(bar:build)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 0 [ ] + 4 -> 2 [ ] + 4 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__task_can_have_a_diff_platform_from_project.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__task_can_have_a_diff_platform_from_project.snap new file mode 100644 index 00000000000..ed6cd4af7c9 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__task_can_have_a_diff_platform_from_project.snap @@ -0,0 +1,17 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupRustTool(1.70.0)" ] + 2 [ label="InstallRustDeps(1.70.0)" ] + 3 [ label="SyncNodeProject(bar)" ] + 4 [ label="RunTask(bar:build)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 0 [ ] + 4 -> 2 [ ] + 4 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap index 6f9dda954a9..bb151659335 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap @@ -4,14 +4,12 @@ expression: graph.to_dot() --- digraph { 0 [ label="SyncWorkspace" ] - 1 [ label="SetupSystemTool" ] - 2 [ label="SyncSystemProject(foo)" ] - 3 [ label="SyncSystemProject(bar)" ] - 4 [ label="SyncSystemProject(qux)" ] + 1 [ label="SyncSystemProject(foo)" ] + 2 [ label="SyncSystemProject(bar)" ] + 3 [ label="SyncSystemProject(qux)" ] + 2 -> 0 [ ] 1 -> 0 [ ] - 3 -> 1 [ ] - 2 -> 1 [ ] - 2 -> 3 [ ] - 4 -> 1 [ ] + 1 -> 2 [ ] + 3 -> 0 [ ] } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap index ddfdb147fe1..2afc70a17f2 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap @@ -4,9 +4,7 @@ expression: graph.to_dot() --- digraph { 0 [ label="SyncWorkspace" ] - 1 [ label="SetupSystemTool" ] - 2 [ label="SyncSystemProject(bar)" ] + 1 [ label="SyncSystemProject(bar)" ] 1 -> 0 [ ] - 2 -> 1 [ ] } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap index 46d410389a3..4600b32bf02 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap @@ -4,12 +4,10 @@ expression: graph.to_dot() --- digraph { 0 [ label="SyncWorkspace" ] - 1 [ label="SetupSystemTool" ] - 2 [ label="SyncSystemProject(foo)" ] - 3 [ label="SyncSystemProject(bar)" ] + 1 [ label="SyncSystemProject(foo)" ] + 2 [ label="SyncSystemProject(bar)" ] + 2 -> 0 [ ] 1 -> 0 [ ] - 3 -> 1 [ ] - 2 -> 1 [ ] - 2 -> 3 [ ] + 1 -> 2 [ ] } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap index c59a9643d06..f01b2d0d799 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap @@ -4,13 +4,9 @@ expression: graph.to_dot() --- digraph { 0 [ label="SyncWorkspace" ] - 1 [ label="SetupNodeTool(20.0.0)" ] - 2 [ label="SyncNodeProject(bar)" ] - 3 [ label="SetupRustTool(1.72.0)" ] - 4 [ label="SyncRustProject(qux)" ] + 1 [ label="SyncNodeProject(bar)" ] + 2 [ label="SyncRustProject(qux)" ] 1 -> 0 [ ] - 2 -> 1 [ ] - 3 -> 0 [ ] - 4 -> 3 [ ] + 2 -> 0 [ ] } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap index 8f0bbd2e090..197a66ca35b 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap @@ -4,13 +4,9 @@ expression: graph.to_dot() --- digraph { 0 [ label="SyncWorkspace" ] - 1 [ label="SetupNodeTool(20.0.0)" ] - 2 [ label="SyncNodeProject(bar)" ] - 3 [ label="SetupNodeTool(18.0.0)" ] - 4 [ label="SyncNodeProject(baz)" ] + 1 [ label="SyncNodeProject(bar)" ] + 2 [ label="SyncNodeProject(baz)" ] 1 -> 0 [ ] - 2 -> 1 [ ] - 3 -> 0 [ ] - 4 -> 3 [ ] + 2 -> 0 [ ] } From 45fdc21a5ff05e300299dac790c6a9e9e735eac4 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Wed, 4 Oct 2023 15:47:31 -0700 Subject: [PATCH 10/22] Use digraph. --- nextgen/action-graph/Cargo.toml | 2 +- nextgen/action-graph/src/action_graph.rs | 6 +++--- nextgen/action-graph/src/action_graph_builder.rs | 4 ++-- ...doesnt_graph_if_not_affected_by_touched_files.snap.new | 8 -------- 4 files changed, 6 insertions(+), 14 deletions(-) delete mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__doesnt_graph_if_not_affected_by_touched_files.snap.new diff --git a/nextgen/action-graph/Cargo.toml b/nextgen/action-graph/Cargo.toml index beb33a4e3d3..baf9cd711d5 100644 --- a/nextgen/action-graph/Cargo.toml +++ b/nextgen/action-graph/Cargo.toml @@ -17,7 +17,7 @@ moon_project_graph = { path = "../project-graph" } moon_task = { path = "../task" } moon_query = { path = "../query" } miette = { workspace = true } -petgraph = { workspace = true, features = ["stable_graph"] } +petgraph = { workspace = true } rustc-hash = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } diff --git a/nextgen/action-graph/src/action_graph.rs b/nextgen/action-graph/src/action_graph.rs index 189a938a2fe..9fc70e3a77c 100644 --- a/nextgen/action-graph/src/action_graph.rs +++ b/nextgen/action-graph/src/action_graph.rs @@ -7,7 +7,7 @@ use petgraph::visit::{IntoEdgeReferences, IntoNodeReferences}; use rustc_hash::{FxHashMap, FxHashSet}; use std::collections::VecDeque; -pub type GraphType = StableGraph; +pub type GraphType = DiGraph; pub type IndicesMap = FxHashMap; pub struct ActionGraph { @@ -63,7 +63,7 @@ impl ActionGraph { } pub fn to_dot(&self) -> String { - type DotGraph = StableGraph; + type DotGraph = DiGraph; let is_test = is_test_env() || cfg!(debug_assertions); let graph = self.graph.map(|_, n| n.label(), |_, _| ()); @@ -99,7 +99,7 @@ impl ActionGraph { format!("{dot:?}") } - pub fn to_labeled_graph(&self) -> StableGraph { + pub fn to_labeled_graph(&self) -> DiGraph { self.graph.map(|_, n| n.label(), |_, _| String::new()) } diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs index 914c3a8b3ba..e991a78265c 100644 --- a/nextgen/action-graph/src/action_graph_builder.rs +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -16,7 +16,7 @@ type TouchedFilePaths = FxHashSet; pub struct ActionGraphBuilder<'app> { all_query: Option, - graph: StableGraph, + graph: DiGraph, indices: FxHashMap, platform_manager: &'app PlatformManager, project_graph: &'app ProjectGraph, @@ -33,7 +33,7 @@ impl<'app> ActionGraphBuilder<'app> { ) -> miette::Result { Ok(ActionGraphBuilder { all_query: None, - graph: StableGraph::new(), + graph: DiGraph::new(), indices: FxHashMap::default(), platform_manager, project_graph, diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__doesnt_graph_if_not_affected_by_touched_files.snap.new b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__doesnt_graph_if_not_affected_by_touched_files.snap.new deleted file mode 100644 index c6fb3902fe1..00000000000 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__doesnt_graph_if_not_affected_by_touched_files.snap.new +++ /dev/null @@ -1,8 +0,0 @@ ---- -source: nextgen/action-graph/tests/action_graph_test.rs -assertion_line: 296 -expression: graph.to_dot() ---- -digraph { -} - From 197aba5123238754141605ec995cbb99180c548e Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Wed, 4 Oct 2023 16:11:41 -0700 Subject: [PATCH 11/22] Update docs. --- .../action-graph/src/action_graph_builder.rs | 4 +- packages/report/tests/action.test.ts | 2 +- packages/report/tests/report.test.ts | 48 +++++++++---------- packages/visualizer/src/helpers/render.ts | 6 +-- website/docs/commands/dep-graph.mdx | 2 +- website/docs/concepts/project.mdx | 2 +- website/docs/concepts/task.mdx | 10 ++-- website/docs/config/project.mdx | 4 +- website/docs/editors/vscode.mdx | 2 +- .../{dep-graph.mdx => action-graph.mdx} | 44 ++++++++--------- website/docs/how-it-works/project-graph.mdx | 2 +- website/docs/run-task.mdx | 2 +- website/sidebars.js | 2 +- .../Docs/{DepGraph.tsx => ActionGraph.tsx} | 20 ++++++-- 14 files changed, 81 insertions(+), 69 deletions(-) rename website/docs/how-it-works/{dep-graph.mdx => action-graph.mdx} (68%) rename website/src/components/Docs/{DepGraph.tsx => ActionGraph.tsx} (87%) diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs index e991a78265c..70f178a1e32 100644 --- a/nextgen/action-graph/src/action_graph_builder.rs +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -317,9 +317,9 @@ impl<'app> ActionGraphBuilder<'app> { } // Syncing requires the language's tool to be installed - let sync_workspace_index = self.sync_workspace(); + let setup_tool_index = self.setup_tool(node.get_runtime()); let index = self.insert_node(node); - let mut reqs = vec![sync_workspace_index]; + let mut reqs = vec![setup_tool_index]; // And we should also depend on other projects for dep_project_id in self.project_graph.dependencies_of(project)? { diff --git a/packages/report/tests/action.test.ts b/packages/report/tests/action.test.ts index c1fa88e3a07..b20ce300f99 100644 --- a/packages/report/tests/action.test.ts +++ b/packages/report/tests/action.test.ts @@ -10,7 +10,7 @@ const action: Action = { }, error: null, flaky: false, - label: 'RunTarget(app:build)', + label: 'RunTask(app:build)', nodeIndex: 8, status: 'passed', finishedAt: '2022-09-12T22:50:12.932311Z', diff --git a/packages/report/tests/report.test.ts b/packages/report/tests/report.test.ts index 023d48c7b9a..789f54cc2c6 100644 --- a/packages/report/tests/report.test.ts +++ b/packages/report/tests/report.test.ts @@ -13,7 +13,7 @@ function mockReport(): RunReport { }, error: null, flaky: false, - label: 'RunTarget(types:build)', + label: 'RunTask(types:build)', nodeIndex: 5, status: 'cached', finishedAt: '2022-09-12T22:50:12.932311Z', @@ -28,7 +28,7 @@ function mockReport(): RunReport { }, error: null, flaky: true, - label: 'RunTarget(runtime:typecheck)', + label: 'RunTask(runtime:typecheck)', nodeIndex: 4, status: 'passed', finishedAt: '2022-09-12T22:50:12.932311Z', @@ -43,7 +43,7 @@ function mockReport(): RunReport { }, error: null, flaky: false, - label: 'RunTarget(types:typecheck)', + label: 'RunTask(types:typecheck)', nodeIndex: 6, status: 'passed', finishedAt: '2022-09-12T22:50:12.932311Z', @@ -58,7 +58,7 @@ function mockReport(): RunReport { }, error: null, flaky: false, - label: 'RunTarget(website:typecheck)', + label: 'RunTask(website:typecheck)', nodeIndex: 8, status: 'passed', finishedAt: '2022-09-12T22:50:12.932311Z', @@ -102,10 +102,10 @@ describe('sortReport()', () => { sortReport(report, 'time', 'asc'); expect(report.actions.map((a) => a.label)).toEqual([ - 'RunTarget(types:build)', - 'RunTarget(website:typecheck)', - 'RunTarget(types:typecheck)', - 'RunTarget(runtime:typecheck)', + 'RunTask(types:build)', + 'RunTask(website:typecheck)', + 'RunTask(types:typecheck)', + 'RunTask(runtime:typecheck)', ]); }); @@ -114,10 +114,10 @@ describe('sortReport()', () => { sortReport(report, 'time', 'desc'); expect(report.actions.map((a) => a.label)).toEqual([ - 'RunTarget(runtime:typecheck)', - 'RunTarget(types:typecheck)', - 'RunTarget(website:typecheck)', - 'RunTarget(types:build)', + 'RunTask(runtime:typecheck)', + 'RunTask(types:typecheck)', + 'RunTask(website:typecheck)', + 'RunTask(types:build)', ]); }); @@ -126,10 +126,10 @@ describe('sortReport()', () => { sortReport(report, 'label', 'asc'); expect(report.actions.map((a) => a.label)).toEqual([ - 'RunTarget(runtime:typecheck)', - 'RunTarget(types:build)', - 'RunTarget(types:typecheck)', - 'RunTarget(website:typecheck)', + 'RunTask(runtime:typecheck)', + 'RunTask(types:build)', + 'RunTask(types:typecheck)', + 'RunTask(website:typecheck)', ]); }); @@ -138,10 +138,10 @@ describe('sortReport()', () => { sortReport(report, 'label', 'desc'); expect(report.actions.map((a) => a.label)).toEqual([ - 'RunTarget(website:typecheck)', - 'RunTarget(types:typecheck)', - 'RunTarget(types:build)', - 'RunTarget(runtime:typecheck)', + 'RunTask(website:typecheck)', + 'RunTask(types:typecheck)', + 'RunTask(types:build)', + 'RunTask(runtime:typecheck)', ]); }); }); @@ -156,7 +156,7 @@ describe('prepareReportActions()', () => { secs: 0, }, icon: '🟪', - label: 'RunTarget(types:build)', + label: 'RunTask(types:build)', status: 'cached', time: '0s', }, @@ -167,7 +167,7 @@ describe('prepareReportActions()', () => { secs: 1922, }, icon: '🟩', - label: 'RunTarget(runtime:typecheck)', + label: 'RunTask(runtime:typecheck)', status: 'passed', time: '32m 2s', }, @@ -178,7 +178,7 @@ describe('prepareReportActions()', () => { secs: 64, }, icon: '🟩', - label: 'RunTarget(types:typecheck)', + label: 'RunTask(types:typecheck)', status: 'passed', time: '1m 4s', }, @@ -189,7 +189,7 @@ describe('prepareReportActions()', () => { secs: 34, }, icon: '🟩', - label: 'RunTarget(website:typecheck)', + label: 'RunTask(website:typecheck)', status: 'passed', time: '34.4s', }, diff --git a/packages/visualizer/src/helpers/render.ts b/packages/visualizer/src/helpers/render.ts index f755c987a17..f57f42dd8f8 100644 --- a/packages/visualizer/src/helpers/render.ts +++ b/packages/visualizer/src/helpers/render.ts @@ -9,8 +9,8 @@ function getActionType(label: string) { return 'sync-workspace'; } - if (label.startsWith('RunTarget') || label.startsWith('RunPersistentTarget')) { - return 'run-target'; + if (label.startsWith('Run') && (label.includes('Target') || label.includes('Task'))) { + return 'run-task'; } if (label.startsWith('Sync') && label.includes('Project')) { @@ -105,7 +105,7 @@ export function render(element: HTMLElement, data: GraphInfo) { }, }, { - selector: 'node[type="run-target"]', + selector: 'node[type="run-task"]', style: { // @ts-expect-error Types incorrect 'background-gradient-stop-colors': '#6e58d1 #4a2ec6 #3b259e', diff --git a/website/docs/commands/dep-graph.mdx b/website/docs/commands/dep-graph.mdx index f916862a1df..7ed5439c1eb 100644 --- a/website/docs/commands/dep-graph.mdx +++ b/website/docs/commands/dep-graph.mdx @@ -34,7 +34,7 @@ digraph { 0 [ label="SetupNodeTool" style=filled, shape=oval, fillcolor=black, fontcolor=white] 1 [ label="InstallNodeDeps" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 2 [ label="SyncNodeProject(node)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="RunTarget(node:standard)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] + 3 [ label="RunTask(node:standard)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 1 -> 0 [ arrowhead=box, arrowtail=box] 2 -> 0 [ arrowhead=box, arrowtail=box] 3 -> 1 [ arrowhead=box, arrowtail=box] diff --git a/website/docs/concepts/project.mdx b/website/docs/concepts/project.mdx index f49c31d5423..6dccd3206de 100644 --- a/website/docs/concepts/project.mdx +++ b/website/docs/concepts/project.mdx @@ -36,7 +36,7 @@ pattern that makes the most sense for your company or team! ## Dependencies Projects can depend on other projects within the [workspace](./workspace) to build a -[project graph](../how-it-works/dep-graph), and in turn, a dependency graph for executing +[project graph](../how-it-works/action-graph), and in turn, an action graph for executing [tasks](./task). Project dependencies are divided into 2 categories: - **Explicit dependencies** - These are dependencies that are explicitly defined in a project's diff --git a/website/docs/concepts/task.mdx b/website/docs/concepts/task.mdx index 74e4637fb7d..94449e36b45 100644 --- a/website/docs/concepts/task.mdx +++ b/website/docs/concepts/task.mdx @@ -30,7 +30,7 @@ Tasks are grouped into 1 of the following types based on their configured parame ## Modes Alongside types, tasks can also grouped into a special mode that provides unique handling within the -dependency graph and pipelines. +action graph and pipelines. ### Local only @@ -52,8 +52,8 @@ tasks: Tasks that need to interact with the user via terminal prompts are known as interactive tasks. Because interactive tasks require stdin, and it's not possible to have multiple parallel running -tasks interact with stdin, we isolate interactive tasks from other tasks in the dependency graph. -This ensures that only 1 interactive task is ran at a time. +tasks interact with stdin, we isolate interactive tasks from other tasks in the action graph. This +ensures that only 1 interactive task is ran at a time. To mark a task as interactive, enable the [`options.interactive`](../config/project#interactive) setting. @@ -72,8 +72,8 @@ Tasks that never complete, like servers and watchers, are known as persistent ta tasks are typically problematic when it comes to dependency graphs, because if they run in the middle of the graph, subsequent tasks will never run because the persistent task never completes! -However in moon, this is a non-issue, as we collect all persistent tasks within the dependency graph -and run them _last as a batch_. This is perfect for a few reasons: +However in moon, this is a non-issue, as we collect all persistent tasks within the action graph and +run them _last as a batch_. This is perfect for a few reasons: - All persistent tasks are ran in parallel, so they don't block each other. - Running both the backend API and frontend webapp in parallel is a breeze. diff --git a/website/docs/config/project.mdx b/website/docs/config/project.mdx index c34f5b84f06..3b55865fa2f 100644 --- a/website/docs/config/project.mdx +++ b/website/docs/config/project.mdx @@ -830,8 +830,8 @@ tasks: Marks the task as persistent (continuously running). [Persistent tasks](../concepts/task#persistent) -are handled differently than non-persistent tasks in the dependency graph. When running a target, -all persistent tasks are _ran last_ and _in parallel_, after all their dependencies have completed. +are handled differently than non-persistent tasks in the action graph. When running a target, all +persistent tasks are _ran last_ and _in parallel_, after all their dependencies have completed. This is extremely useful for running a server (or a watcher) in the background while other tasks are running. diff --git a/website/docs/editors/vscode.mdx b/website/docs/editors/vscode.mdx index 94cbcb22e9e..80acb0987d4 100644 --- a/website/docs/editors/vscode.mdx +++ b/website/docs/editors/vscode.mdx @@ -59,7 +59,7 @@ Information about the last ran target will be displayed in a beautiful table wit Only tasks ran from the [projects view](#projects) or on the command line will be displayed here. This table displays all actions that were ran alongside the running primary target(s). They are -ordered topologically via the dependency graph. +ordered topologically via the action graph. diff --git a/website/docs/how-it-works/dep-graph.mdx b/website/docs/how-it-works/action-graph.mdx similarity index 68% rename from website/docs/how-it-works/dep-graph.mdx rename to website/docs/how-it-works/action-graph.mdx index a7803c5ccb5..3c9cb38d2ea 100644 --- a/website/docs/how-it-works/dep-graph.mdx +++ b/website/docs/how-it-works/action-graph.mdx @@ -1,18 +1,17 @@ --- -title: Dependency graph +title: Action graph --- -import DepGraph from '@site/src/components/Docs/DepGraph'; +import ActionGraph from '@site/src/components/Docs/ActionGraph'; -When you run a [task](../config/project#tasks-1) on the command line, we generate a dependency graph -to ensure [dependencies](../config/project#deps) of tasks have ran before running run the primary -task. +When you run a [task](../config/project#tasks-1) on the command line, we generate an action graph to +ensure [dependencies](../config/project#deps) of tasks have ran before running run the primary task. -The dependency graph is a representation of all [tasks](../concepts/task), derived from the +The action graph is a representation of all [tasks](../concepts/task), derived from the [project graph](./project-graph), and is also represented internally as a directed acyclic graph (DAG). - + ## Actions @@ -21,7 +20,7 @@ represent each node in the graph as an action to perform. This allows us to be m efficient with how we run tasks, and allows us to provide more functionality and automation than other runners. -The following actions compose our dependency graph: +The following actions compose our action graph: ### Sync workspace @@ -36,12 +35,12 @@ tier 3 language into the toolchain. For other tiers, this is basically a no-oper - When the tool has already been installed, this action will be skipped. - Actions will be scoped by language and version, also known as a runtime. For example, `SetupNodeTool(18.1.0)` or `SetupDenoTool(1.31.0)`. -- Tools that require a global language binary will display the version as "global". For example, - `SetupNodeTool(global)`. +- Tools that require a global binary (found on `PATH`) will display the version as "global". For + example, `SetupNodeTool(global)`. ### Install dependencies -Before we run a target, we ensure that all language dependencies (`node_modules` for example) have +Before we run a task, we ensure that all language dependencies (`node_modules` for example) have been installed, by automatically installing them if we detect changes since the last run. We achieve this by comparing lockfile modified timestamps, parsing manifest files, and hashing resolved dependency versions. @@ -58,7 +57,7 @@ dependency versions. ### Sync project To ensure a consistently healthy project and repository, we run a process known as syncing -_everytime_ a target is ran. Actions will be scoped by language, for example, +_everytime_ a task is ran. Actions will be scoped by language, for example, `SyncNodeProject(example)`. What is synced or considered healthcare is dependent on the language and its ecosystem. @@ -74,26 +73,27 @@ What is synced or considered healthcare is dependent on the language and its eco > This action depends on the setup toolchain action, in case it requires binaries or functionality > that the toolchain provides. -### Run target +### Run task -The primary action in the graph is the run [target](../concepts/target) action, which runs a -project's task as a child process. Tasks can depend on other tasks, and they'll be effectively -orchestrated and executed by running in topological order _and_ in batches via a worker pool. +The primary action in the graph is the run [task](../concepts/task) action, which runs a project's +task as a child process, derived from a [target](../concepts/target). Tasks can depend on other +tasks, and they'll be effectively orchestrated and executed by running in topological order using a +thread pool. > This action depends on the previous actions, as the toolchain is used for running the task's > command, and the outcome of the task is best when the project state is healthy and deterministic. -### Run interactive target +### Run interactive task -Like the base run target, but runs the [task interactively](../concepts/task#interactive) with stdin +Like the base run task, but runs the [task interactively](../concepts/task#interactive) with stdin capabilities. All interactive tasks are run in isolation in the graph. -### Run persistent target +### Run persistent task -Like the base run target, but runs the [task in a persistent process](../concepts/task#persistent) +Like the base run task, but runs the [task in a persistent process](../concepts/task#persistent) that never exits. All persistent tasks are run in parallel as the last batch in the graph. ## What is the graph used for? -Without the dependency graph, tasks would not efficiently, or possibly at all! The graph helps to -run tasks in parallel, in the correct order, and to ensure a reliable outcome. +Without the action graph, tasks would not efficiently, or possibly at all! The graph helps to run +tasks in parallel, in the correct order, and to ensure a reliable outcome. diff --git a/website/docs/how-it-works/project-graph.mdx b/website/docs/how-it-works/project-graph.mdx index fddb4ca2814..1db3ddef951 100644 --- a/website/docs/how-it-works/project-graph.mdx +++ b/website/docs/how-it-works/project-graph.mdx @@ -82,7 +82,7 @@ integration. Great question, the project graph is used throughout the codebase to accomplish a variety of functions, but mainly: -- Is fed into the [dependency graph](./dep-graph) to determine relationships of tasks between other +- Is fed into the [action graph](./action-graph) to determine relationships of tasks between other tasks, and across projects. - Powers our [Docker](../guides/docker) layer caching and scaffolding implementations. - Utilized for [project syncing](../commands/sync) to ensure a healthy repository state. diff --git a/website/docs/run-task.mdx b/website/docs/run-task.mdx index df4cbd5b2fb..48f72e2e5ef 100644 --- a/website/docs/run-task.mdx +++ b/website/docs/run-task.mdx @@ -23,7 +23,7 @@ $ moon app:build When this command is ran, it will do the following: -- Generate a directed acyclic graph, known as the dependency graph. +- Generate a directed acyclic graph, known as the action (dependency) graph. - Insert [`deps`](./config/project#deps) as targets into the graph. - Insert the primary target into the graph. - Run all tasks in the graph in parallel and in topological order (the dependency chain). diff --git a/website/sidebars.js b/website/sidebars.js index 3ead0dd75f5..d9cfca902b6 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -10,7 +10,7 @@ const sidebars = { label: 'How it works', collapsed: true, collapsible: true, - items: ['how-it-works/languages', 'how-it-works/project-graph', 'how-it-works/dep-graph'], + items: ['how-it-works/languages', 'how-it-works/project-graph', 'how-it-works/action-graph'], link: { type: 'generated-index', title: 'How it works', diff --git a/website/src/components/Docs/DepGraph.tsx b/website/src/components/Docs/ActionGraph.tsx similarity index 87% rename from website/src/components/Docs/DepGraph.tsx rename to website/src/components/Docs/ActionGraph.tsx index 84b701602d8..a8a1e52764b 100644 --- a/website/src/components/Docs/DepGraph.tsx +++ b/website/src/components/Docs/ActionGraph.tsx @@ -1,13 +1,25 @@ import React, { useEffect, useRef } from 'react'; import { renderGraph } from '../../utils/renderGraph'; -export default function DepGraph() { +export default function ActionGraph() { const graphRef = useRef(null); useEffect(() => { if (graphRef.current) { renderGraph(graphRef.current, { edges: [ + { + data: { + source: 'sync-workspace', + target: 'node-tool', + }, + }, + { + data: { + source: 'sync-workspace', + target: 'system-tool', + }, + }, { data: { source: 'node-tool', @@ -125,21 +137,21 @@ export default function DepGraph() { { data: { id: 'target-clean', - label: 'RunTarget(example:clean)', + label: 'RunTask(example:clean)', type: 'sm', }, }, { data: { id: 'target-build', - label: 'RunTarget(example:build)', + label: 'RunTask(example:build)', type: 'sm', }, }, { data: { id: 'target-package', - label: 'RunTarget(example:package)', + label: 'RunTask(example:package)', type: 'sm', }, }, From 67581585083230c23190f71dd8b17451109bfb24 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Wed, 4 Oct 2023 16:17:30 -0700 Subject: [PATCH 12/22] Add back link. --- .../action-graph/tests/action_graph_test.rs | 46 +++++++++++++++---- ..._test__action_graph__run_task__graphs.snap | 2 +- ...can_have_a_diff_platform_from_project.snap | 8 ++-- ..._graph__sync_project__graphs_multiple.snap | 14 +++--- ...on_graph__sync_project__graphs_single.snap | 4 +- ..._sync_project__graphs_single_with_dep.snap | 10 ++-- ..._sync_project__inherits_platform_tool.snap | 10 ++-- ...c_project__supports_platform_override.snap | 10 ++-- 8 files changed, 75 insertions(+), 29 deletions(-) diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index ac0fd272e65..987bc415fd0 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -255,11 +255,11 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, - ActionNode::SyncProject { - project: Id::raw("bar"), + ActionNode::SetupTool { runtime: create_node_runtime() }, - ActionNode::SetupTool { + ActionNode::SyncProject { + project: Id::raw("bar"), runtime: create_node_runtime() }, ActionNode::InstallDeps { @@ -296,11 +296,11 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, - ActionNode::SyncProject { - project: Id::raw("bar"), + ActionNode::SetupTool { runtime: create_node_runtime() }, - ActionNode::SetupTool { + ActionNode::SyncProject { + project: Id::raw("bar"), runtime: create_node_runtime() }, ActionNode::InstallDeps { @@ -381,13 +381,16 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, - ActionNode::SyncProject { - project: Id::raw("bar"), + ActionNode::SetupTool { runtime: create_node_runtime() }, ActionNode::SetupTool { runtime: create_rust_runtime() }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: create_node_runtime() + }, ActionNode::InstallDeps { runtime: create_rust_runtime() }, @@ -555,6 +558,9 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, ActionNode::SyncProject { project: Id::raw("bar"), runtime: Runtime::system() @@ -578,6 +584,9 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, ActionNode::SyncProject { project: Id::raw("bar"), runtime: Runtime::system() @@ -611,6 +620,9 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, ActionNode::SyncProject { project: Id::raw("qux"), runtime: Runtime::system() @@ -644,6 +656,9 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, ActionNode::SyncProject { project: Id::raw("bar"), runtime: Runtime::system() @@ -675,6 +690,12 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_rust_runtime() + }, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, ActionNode::SyncProject { project: Id::raw("qux"), runtime: create_rust_runtime() @@ -706,6 +727,15 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::new_override( + PlatformType::Node, + RuntimeReq::with_version(Version::new(18, 0, 0)) + ) + }, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, ActionNode::SyncProject { project: Id::raw("baz"), runtime: Runtime::new_override( diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__graphs.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__graphs.snap index bbb08d9a762..cb49bf8219b 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__graphs.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__graphs.snap @@ -10,7 +10,7 @@ digraph { 4 [ label="RunTask(bar:build)" ] 1 -> 0 [ ] 2 -> 1 [ ] - 3 -> 0 [ ] + 3 -> 1 [ ] 4 -> 2 [ ] 4 -> 3 [ ] } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__task_can_have_a_diff_platform_from_project.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__task_can_have_a_diff_platform_from_project.snap index ed6cd4af7c9..38f0da434df 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__task_can_have_a_diff_platform_from_project.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__task_can_have_a_diff_platform_from_project.snap @@ -6,12 +6,14 @@ digraph { 0 [ label="SyncWorkspace" ] 1 [ label="SetupRustTool(1.70.0)" ] 2 [ label="InstallRustDeps(1.70.0)" ] - 3 [ label="SyncNodeProject(bar)" ] - 4 [ label="RunTask(bar:build)" ] + 3 [ label="SetupNodeTool(20.0.0)" ] + 4 [ label="SyncNodeProject(bar)" ] + 5 [ label="RunTask(bar:build)" ] 1 -> 0 [ ] 2 -> 1 [ ] 3 -> 0 [ ] - 4 -> 2 [ ] 4 -> 3 [ ] + 5 -> 2 [ ] + 5 -> 4 [ ] } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap index bb151659335..6f9dda954a9 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap @@ -4,12 +4,14 @@ expression: graph.to_dot() --- digraph { 0 [ label="SyncWorkspace" ] - 1 [ label="SyncSystemProject(foo)" ] - 2 [ label="SyncSystemProject(bar)" ] - 3 [ label="SyncSystemProject(qux)" ] - 2 -> 0 [ ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(foo)" ] + 3 [ label="SyncSystemProject(bar)" ] + 4 [ label="SyncSystemProject(qux)" ] 1 -> 0 [ ] - 1 -> 2 [ ] - 3 -> 0 [ ] + 3 -> 1 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] + 4 -> 1 [ ] } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap index 2afc70a17f2..ddfdb147fe1 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap @@ -4,7 +4,9 @@ expression: graph.to_dot() --- digraph { 0 [ label="SyncWorkspace" ] - 1 [ label="SyncSystemProject(bar)" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(bar)" ] 1 -> 0 [ ] + 2 -> 1 [ ] } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap index 4600b32bf02..46d410389a3 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap @@ -4,10 +4,12 @@ expression: graph.to_dot() --- digraph { 0 [ label="SyncWorkspace" ] - 1 [ label="SyncSystemProject(foo)" ] - 2 [ label="SyncSystemProject(bar)" ] - 2 -> 0 [ ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(foo)" ] + 3 [ label="SyncSystemProject(bar)" ] 1 -> 0 [ ] - 1 -> 2 [ ] + 3 -> 1 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap index f01b2d0d799..8a93b55531b 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap @@ -4,9 +4,13 @@ expression: graph.to_dot() --- digraph { 0 [ label="SyncWorkspace" ] - 1 [ label="SyncNodeProject(bar)" ] - 2 [ label="SyncRustProject(qux)" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="SyncNodeProject(bar)" ] + 3 [ label="SetupRustTool(1.70.0)" ] + 4 [ label="SyncRustProject(qux)" ] 1 -> 0 [ ] - 2 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 0 [ ] + 4 -> 3 [ ] } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap index 197a66ca35b..8f0bbd2e090 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap @@ -4,9 +4,13 @@ expression: graph.to_dot() --- digraph { 0 [ label="SyncWorkspace" ] - 1 [ label="SyncNodeProject(bar)" ] - 2 [ label="SyncNodeProject(baz)" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="SyncNodeProject(bar)" ] + 3 [ label="SetupNodeTool(18.0.0)" ] + 4 [ label="SyncNodeProject(baz)" ] 1 -> 0 [ ] - 2 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 0 [ ] + 4 -> 3 [ ] } From 07099e2f617e002700e33a25e68b4aa10bce91fd Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Wed, 4 Oct 2023 16:35:44 -0700 Subject: [PATCH 13/22] Add cycle stuff. --- nextgen/action-graph/src/action_graph.rs | 31 +++++++------------ .../action-graph/tests/action_graph_test.rs | 25 +++++++++++++++ nextgen/project-graph/src/project_graph.rs | 9 ++++++ 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/nextgen/action-graph/src/action_graph.rs b/nextgen/action-graph/src/action_graph.rs index 9fc70e3a77c..7cb832ab873 100644 --- a/nextgen/action-graph/src/action_graph.rs +++ b/nextgen/action-graph/src/action_graph.rs @@ -30,7 +30,7 @@ impl ActionGraph { } pub fn reset_iterator(&mut self) -> miette::Result<()> { - // self.detect_cycle()?; + self.detect_cycle()?; self.queue.clear(); self.visited.clear(); @@ -104,27 +104,18 @@ impl ActionGraph { } fn detect_cycle(&self) -> miette::Result<()> { - if self.is_empty() || self.get_node_count() == 1 { - return Ok(()); + if self.get_node_count() > 1 { + if let Err(cycle) = petgraph::algo::toposort(&self.graph, None) { + return Err(ActionGraphError::CycleDetected( + self.get_node_from_index(&cycle.node_id()) + .map(|n| n.label()) + .unwrap_or_else(|| "(unknown)".into()), + ) + .into()); + } } - let scc = petgraph::algo::kosaraju_scc(&self.graph); - - // TODO - dbg!(&scc); - - // The cycle is always the last sequence in the list - let Some(cycle) = scc.last() else { - return Err(ActionGraphError::CycleDetected("(unknown)".into()).into()); - }; - - let path = cycle - .iter() - .filter_map(|i| self.get_node_from_index(i).map(|n| n.label())) - .collect::>() - .join(" → "); - - Err(ActionGraphError::CycleDetected(path).into()) + Ok(()) } } diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index 987bc415fd0..de9a843f356 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -174,6 +174,31 @@ fn topo(mut graph: ActionGraph) -> Vec { mod action_graph { use super::*; + // #[test] + // fn errors_on_cycle() { + // let mut graph = ProjectGraphType::new(); + // let a = graph.add_node(create_project("a")); + // let b = graph.add_node(create_project("b")); + // graph.add_edge(a, b, DependencyScope::Build); + // graph.add_edge(b, a, DependencyScope::Build); + + // let pg = ProjectGraph::new( + // graph, + // FxHashMap::from_iter([ + // ("a".into(), ProjectNode::new(0)), + // ("b".into(), ProjectNode::new(1)), + // ]), + // &PathBuf::from("."), + // ); + + // let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + + // builder.sync_project(&pg.get("a").unwrap()).unwrap(); + // builder.sync_project(&pg.get("b").unwrap()).unwrap(); + + // builder.build().unwrap().reset_iterator().unwrap(); + // } + mod install_deps { use super::*; diff --git a/nextgen/project-graph/src/project_graph.rs b/nextgen/project-graph/src/project_graph.rs index 012de71483a..38b15e90294 100644 --- a/nextgen/project-graph/src/project_graph.rs +++ b/nextgen/project-graph/src/project_graph.rs @@ -34,6 +34,15 @@ pub struct ProjectNode { pub source: WorkspaceRelativePathBuf, } +impl ProjectNode { + pub fn new(index: usize) -> Self { + ProjectNode { + index: NodeIndex::new(index), + ..ProjectNode::default() + } + } +} + #[derive(Default)] pub struct ProjectGraph { pub check_boundaries: bool, From 5f3d8fbbcff5899713f508269075b2ea51570469 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Thu, 5 Oct 2023 14:51:46 -0700 Subject: [PATCH 14/22] Add test utils crate. --- Cargo.lock | 13 ++ nextgen/project-graph/Cargo.toml | 1 + .../project-graph/tests/project_graph_test.rs | 164 +++--------------- nextgen/test-utils/Cargo.toml | 16 ++ nextgen/test-utils/src/lib.rs | 3 + nextgen/test-utils/src/project_graph.rs | 127 ++++++++++++++ 6 files changed, 186 insertions(+), 138 deletions(-) create mode 100644 nextgen/test-utils/Cargo.toml create mode 100644 nextgen/test-utils/src/lib.rs create mode 100644 nextgen/test-utils/src/project_graph.rs diff --git a/Cargo.lock b/Cargo.lock index 49a814d7452..88d62d1bb26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3756,6 +3756,7 @@ dependencies = [ "moon_query", "moon_task", "moon_task_builder", + "moon_test_utils2", "moon_vcs", "once_map", "petgraph", @@ -3992,6 +3993,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "moon_test_utils2" +version = "0.1.0" +dependencies = [ + "miette", + "moon_config", + "moon_project_graph", + "moon_vcs", + "starbase_events", + "starbase_sandbox", +] + [[package]] name = "moon_time" version = "0.1.0" diff --git a/nextgen/project-graph/Cargo.toml b/nextgen/project-graph/Cargo.toml index a7f09387974..d0ad40416a8 100644 --- a/nextgen/project-graph/Cargo.toml +++ b/nextgen/project-graph/Cargo.toml @@ -32,5 +32,6 @@ thiserror = { workspace = true } tracing = { workspace = true } [dev-dependencies] +moon_test_utils2 = { path = "../test-utils" } starbase_sandbox = { workspace = true } tokio = { workspace = true } diff --git a/nextgen/project-graph/tests/project_graph_test.rs b/nextgen/project-graph/tests/project_graph_test.rs index 380722c6b97..96bd361e5d4 100644 --- a/nextgen/project-graph/tests/project_graph_test.rs +++ b/nextgen/project-graph/tests/project_graph_test.rs @@ -1,124 +1,26 @@ use moon_common::{path::WorkspaceRelativePathBuf, Id}; -use moon_config::PartialTaskConfig; use moon_config::{ - DependencyConfig, DependencyScope, DependencySource, InheritedTasksEntry, - InheritedTasksManager, NodeConfig, PartialInheritedTasksConfig, ToolchainConfig, - WorkspaceConfig, WorkspaceProjects, WorkspaceProjectsConfig, + DependencyConfig, DependencyScope, DependencySource, InheritedTasksManager, WorkspaceProjects, + WorkspaceProjectsConfig, }; use moon_project::{FileGroup, Project}; -use moon_project_builder::DetectLanguageEvent; use moon_project_graph::{ ExtendProjectData, ExtendProjectEvent, ExtendProjectGraphData, ExtendProjectGraphEvent, - ProjectGraph, ProjectGraphBuilder, ProjectGraphBuilderContext, + ProjectGraph, ProjectGraphBuilder, }; use moon_query::build_query; use moon_task::Target; -use moon_task_builder::DetectPlatformEvent; -use moon_vcs::{BoxedVcs, Git}; +use moon_test_utils2::*; use rustc_hash::{FxHashMap, FxHashSet}; -use starbase_events::{Emitter, EventState}; +use starbase_events::EventState; use starbase_sandbox::{assert_snapshot, create_sandbox, Sandbox}; -use starbase_utils::string_vec; -use starbase_utils::{fs, json}; -use std::collections::BTreeMap; +use starbase_utils::{fs, json, string_vec}; use std::fs::OpenOptions; use std::io::Write; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::Arc; use tokio::sync::RwLock; -#[derive(Default)] -struct GraphContainer { - pub inherited_tasks: InheritedTasksManager, - pub toolchain_config: ToolchainConfig, - pub workspace_config: WorkspaceConfig, - pub workspace_root: PathBuf, - pub vcs: Option, -} - -impl GraphContainer { - pub fn new(root: &Path) -> Self { - let mut graph = Self { - workspace_root: root.to_path_buf(), - ..Default::default() - }; - - // Add a global task to all projects - graph.inherited_tasks.configs.insert( - "*".into(), - InheritedTasksEntry { - input: ".moon/tasks.yml".into(), - config: PartialInheritedTasksConfig { - tasks: Some(BTreeMap::from_iter([( - "global".into(), - PartialTaskConfig::default(), - )])), - ..PartialInheritedTasksConfig::default() - }, - }, - ); - - // Always use the node platform - graph.toolchain_config.node = Some(NodeConfig::default()); - - // Use folders as project names - graph.workspace_config.projects = WorkspaceProjects::Globs(string_vec!["*"]); - - graph - } - - pub fn new_with_vcs(root: &Path) -> Self { - let mut container = Self::new(root); - container.vcs = Some(Box::new(Git::load(root, "master", &[]).unwrap())); - container - } - - pub fn create_context(&self) -> ProjectGraphBuilderContext { - ProjectGraphBuilderContext { - extend_project: Emitter::::new(), - extend_project_graph: Emitter::::new(), - detect_language: Emitter::::new(), - detect_platform: Emitter::::new(), - inherited_tasks: &self.inherited_tasks, - toolchain_config: &self.toolchain_config, - vcs: self.vcs.as_ref(), - working_dir: &self.workspace_root, - workspace_config: &self.workspace_config, - workspace_root: &self.workspace_root, - } - } - - pub async fn build_graph<'l>(&self, context: ProjectGraphBuilderContext<'l>) -> ProjectGraph { - let mut builder = ProjectGraphBuilder::new(context).await.unwrap(); - builder.load_all().await.unwrap(); - - let mut graph = builder.build().await.unwrap(); - graph.check_boundaries = true; - graph.get_all().unwrap(); - graph - } - - pub async fn build_graph_for<'l>( - &self, - context: ProjectGraphBuilderContext<'l>, - ids: &[&str], - ) -> ProjectGraph { - let mut builder = ProjectGraphBuilder::new(context).await.unwrap(); - - for id in ids { - builder.load(id).await.unwrap(); - } - - let graph = builder.build().await.unwrap(); - - for id in ids { - graph.get(id).unwrap(); - } - - graph - } -} - pub fn append_file>(path: P, data: &str) { let mut file = OpenOptions::new() .write(true) @@ -145,19 +47,6 @@ fn get_ids_from_projects(projects: Vec>) -> Vec { mod project_graph { use super::*; - async fn generate_project_graph(fixture: &str) -> ProjectGraph { - let sandbox = create_sandbox(fixture); - - generate_project_graph_from_sandbox(sandbox.path()).await - } - - async fn generate_project_graph_from_sandbox(path: &Path) -> ProjectGraph { - let container = GraphContainer::new(path); - let context = container.create_context(); - - container.build_graph(context).await - } - #[tokio::test] async fn gets_by_id() { let graph = generate_project_graph("dependencies").await; @@ -219,7 +108,7 @@ mod project_graph { // Move files so that we can infer a compatible root project name fs::copy_dir_all(sandbox.path(), sandbox.path(), &root).unwrap(); - let mut container = GraphContainer::new(&root); + let mut container = ProjectGraphContainer::new(&root); container.workspace_config.projects = WorkspaceProjects::Globs(string_vec!["*", "."]); @@ -235,7 +124,7 @@ mod project_graph { #[tokio::test] async fn paths() { let sandbox = create_sandbox("dependencies"); - let mut container = GraphContainer::new(sandbox.path()); + let mut container = ProjectGraphContainer::new(sandbox.path()); container.workspace_config.projects = WorkspaceProjects::Sources(FxHashMap::from_iter([ @@ -252,7 +141,7 @@ mod project_graph { #[tokio::test] async fn paths_and_globs() { let sandbox = create_sandbox("dependencies"); - let mut container = GraphContainer::new(sandbox.path()); + let mut container = ProjectGraphContainer::new(sandbox.path()); container.workspace_config.projects = WorkspaceProjects::Both(WorkspaceProjectsConfig { @@ -308,7 +197,7 @@ mod project_graph { sandbox.enable_git(); sandbox.create_file(".gitignore", "*-other"); - let container = GraphContainer::new_with_vcs(sandbox.path()); + let container = ProjectGraphContainer::with_vcs(sandbox.path()); let context = container.create_context(); let graph = container.build_graph(context).await; @@ -348,7 +237,7 @@ mod project_graph { async fn do_generate(root: &Path) -> ProjectGraph { let cache_engine = CacheEngine::new(root).unwrap(); let hash_engine = HashEngine::new(&cache_engine.cache_dir).unwrap(); - let container = GraphContainer::new_with_vcs(root); + let container = ProjectGraphContainer::with_vcs(root); let context = container.create_context(); let mut builder = ProjectGraphBuilder::generate(context, &cache_engine, &hash_engine) @@ -538,13 +427,12 @@ mod project_graph { async fn generate_inheritance_project_graph(fixture: &str) -> ProjectGraph { let sandbox = create_sandbox(fixture); - let mut container = GraphContainer::new(sandbox.path()); - container.inherited_tasks = - InheritedTasksManager::load(sandbox.path(), sandbox.path().join(".moon")).unwrap(); - - let context = container.create_context(); - - container.build_graph(context).await + generate_project_graph_with_changes(sandbox.path(), |container| { + container.inherited_tasks = + InheritedTasksManager::load(sandbox.path(), sandbox.path().join(".moon")) + .unwrap(); + }) + .await } #[tokio::test] @@ -802,7 +690,7 @@ mod project_graph { #[tokio::test] async fn no_depends_on() { let sandbox = create_sandbox("dependency-types"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); let graph = container.build_graph_for(context, &["no-depends-on"]).await; @@ -812,7 +700,7 @@ mod project_graph { #[tokio::test] async fn some_depends_on() { let sandbox = create_sandbox("dependency-types"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); let graph = container .build_graph_for(context, &["some-depends-on"]) @@ -824,7 +712,7 @@ mod project_graph { #[tokio::test] async fn from_task_deps() { let sandbox = create_sandbox("dependency-types"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); let graph = container .build_graph_for(context, &["from-task-deps"]) @@ -836,7 +724,7 @@ mod project_graph { #[tokio::test] async fn self_task_deps() { let sandbox = create_sandbox("dependency-types"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); let graph = container .build_graph_for(context, &["self-task-deps"]) @@ -852,7 +740,7 @@ mod project_graph { async fn generate_aliases_project_graph() -> ProjectGraph { let sandbox = create_sandbox("aliases"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); // Set aliases for projects @@ -1036,7 +924,7 @@ mod project_graph { func(&sandbox); - let mut container = GraphContainer::new(sandbox.path()); + let mut container = ProjectGraphContainer::new(sandbox.path()); container .workspace_config @@ -1178,7 +1066,7 @@ mod project_graph { func(&sandbox); - let mut container = GraphContainer::new(sandbox.path()); + let mut container = ProjectGraphContainer::new(sandbox.path()); container .workspace_config @@ -1464,7 +1352,7 @@ mod project_graph { #[tokio::test] async fn renders_partial() { let sandbox = create_sandbox("dependencies"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); let graph = container.build_graph_for(context, &["b"]).await; diff --git a/nextgen/test-utils/Cargo.toml b/nextgen/test-utils/Cargo.toml new file mode 100644 index 00000000000..1056fb1ca62 --- /dev/null +++ b/nextgen/test-utils/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "moon_test_utils2" +version = "0.1.0" +edition = "2021" +license = "MIT" +description = "Testing utilities." +homepage = "https://moonrepo.dev/moon" +repository = "https://github.com/moonrepo/moon" + +[dependencies] +moon_config = { path = "../config" } +moon_project_graph = { path = "../project-graph" } +moon_vcs = { path = "../vcs" } +miette = { workspace = true } +starbase_events = { workspace = true } +starbase_sandbox = { workspace = true } diff --git a/nextgen/test-utils/src/lib.rs b/nextgen/test-utils/src/lib.rs new file mode 100644 index 00000000000..acb8deedbf9 --- /dev/null +++ b/nextgen/test-utils/src/lib.rs @@ -0,0 +1,3 @@ +mod project_graph; + +pub use project_graph::*; diff --git a/nextgen/test-utils/src/project_graph.rs b/nextgen/test-utils/src/project_graph.rs new file mode 100644 index 00000000000..8d25437fdfa --- /dev/null +++ b/nextgen/test-utils/src/project_graph.rs @@ -0,0 +1,127 @@ +use moon_config::{ + InheritedTasksEntry, InheritedTasksManager, NodeConfig, PartialInheritedTasksConfig, + PartialTaskConfig, ToolchainConfig, WorkspaceConfig, WorkspaceProjects, +}; +use moon_project_graph::{ + DetectLanguageEvent, DetectPlatformEvent, ExtendProjectEvent, ExtendProjectGraphEvent, + ProjectGraph, ProjectGraphBuilder, ProjectGraphBuilderContext, +}; +use moon_vcs::{BoxedVcs, Git}; +use starbase_events::Emitter; +use starbase_sandbox::create_sandbox; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +#[derive(Default)] +pub struct ProjectGraphContainer { + pub inherited_tasks: InheritedTasksManager, + pub toolchain_config: ToolchainConfig, + pub workspace_config: WorkspaceConfig, + pub workspace_root: PathBuf, + pub vcs: Option, +} + +impl ProjectGraphContainer { + pub fn new(root: &Path) -> Self { + let mut graph = Self { + workspace_root: root.to_path_buf(), + ..Default::default() + }; + + // Add a global task to all projects + graph.inherited_tasks.configs.insert( + "*".into(), + InheritedTasksEntry { + input: ".moon/tasks.yml".into(), + config: PartialInheritedTasksConfig { + tasks: Some(BTreeMap::from_iter([( + "global".into(), + PartialTaskConfig::default(), + )])), + ..PartialInheritedTasksConfig::default() + }, + }, + ); + + // Always use the node platform + graph.toolchain_config.node = Some(NodeConfig::default()); + + // Use folders as project names + graph.workspace_config.projects = WorkspaceProjects::Globs(vec!["*".into()]); + + graph + } + + pub fn with_vcs(root: &Path) -> Self { + let mut container = Self::new(root); + container.vcs = Some(Box::new(Git::load(root, "master", &[]).unwrap())); + container + } + + pub fn create_context(&self) -> ProjectGraphBuilderContext { + ProjectGraphBuilderContext { + extend_project: Emitter::::new(), + extend_project_graph: Emitter::::new(), + detect_language: Emitter::::new(), + detect_platform: Emitter::::new(), + inherited_tasks: &self.inherited_tasks, + toolchain_config: &self.toolchain_config, + vcs: self.vcs.as_ref(), + working_dir: &self.workspace_root, + workspace_config: &self.workspace_config, + workspace_root: &self.workspace_root, + } + } + + pub async fn build_graph<'l>(&self, context: ProjectGraphBuilderContext<'l>) -> ProjectGraph { + let mut builder = ProjectGraphBuilder::new(context).await.unwrap(); + builder.load_all().await.unwrap(); + + let mut graph = builder.build().await.unwrap(); + graph.check_boundaries = true; + graph.get_all().unwrap(); + graph + } + + pub async fn build_graph_for<'l>( + &self, + context: ProjectGraphBuilderContext<'l>, + ids: &[&str], + ) -> ProjectGraph { + let mut builder = ProjectGraphBuilder::new(context).await.unwrap(); + + for id in ids { + builder.load(id).await.unwrap(); + } + + let mut graph = builder.build().await.unwrap(); + graph.check_boundaries = true; + + for id in ids { + graph.get(id).unwrap(); + } + + graph + } +} + +pub async fn generate_project_graph(fixture: &str) -> ProjectGraph { + generate_project_graph_from_sandbox(create_sandbox(fixture).path()).await +} + +pub async fn generate_project_graph_from_sandbox(path: &Path) -> ProjectGraph { + generate_project_graph_with_changes(path, |_| {}).await +} + +pub async fn generate_project_graph_with_changes(path: &Path, mut op: F) -> ProjectGraph +where + F: FnMut(&mut ProjectGraphContainer), +{ + let mut container = ProjectGraphContainer::new(path); + + op(&mut container); + + let context = container.create_context(); + + container.build_graph(context).await +} From 9818c255b01d531add957ece0a0551beb9f63043 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Fri, 6 Oct 2023 10:30:24 -0700 Subject: [PATCH 15/22] Start on new test utils crate. --- Cargo.lock | 7 + nextgen/action-graph/Cargo.toml | 2 + .../dep-workspace/.moon/toolchain.yml | 2 + .../__fixtures__/dep-workspace/in/moon.yml | 1 + .../__fixtures__/dep-workspace/out/moon.yml | 1 + .../__fixtures__/dep-workspace/package.json | 3 + .../__fixtures__/projects/.moon/toolchain.yml | 6 + .../tests/__fixtures__/projects/bar/moon.yml | 1 + .../tests/__fixtures__/projects/baz/moon.yml | 5 + .../tests/__fixtures__/projects/foo/moon.yml | 1 + .../tests/__fixtures__/projects/package.json | 3 + .../tests/__fixtures__/projects/qux/moon.yml | 1 + .../action-graph/tests/action_graph_test.rs | 312 ++++++++---------- ..._project_when_not_in_depman_workspace.snap | 14 + nextgen/action-graph/tests/utils.rs | 27 ++ .../tests/inherited_tasks_config_test.rs | 2 - nextgen/test-utils/Cargo.toml | 7 + nextgen/test-utils/src/lib.rs | 2 + nextgen/test-utils/src/platform_manager.rs | 32 ++ nextgen/test-utils/src/project_graph.rs | 17 +- 20 files changed, 258 insertions(+), 188 deletions(-) create mode 100644 nextgen/action-graph/tests/__fixtures__/dep-workspace/.moon/toolchain.yml create mode 100644 nextgen/action-graph/tests/__fixtures__/dep-workspace/in/moon.yml create mode 100644 nextgen/action-graph/tests/__fixtures__/dep-workspace/out/moon.yml create mode 100644 nextgen/action-graph/tests/__fixtures__/dep-workspace/package.json create mode 100644 nextgen/action-graph/tests/__fixtures__/projects/.moon/toolchain.yml create mode 100644 nextgen/action-graph/tests/__fixtures__/projects/bar/moon.yml create mode 100644 nextgen/action-graph/tests/__fixtures__/projects/baz/moon.yml create mode 100644 nextgen/action-graph/tests/__fixtures__/projects/foo/moon.yml create mode 100644 nextgen/action-graph/tests/__fixtures__/projects/package.json create mode 100644 nextgen/action-graph/tests/__fixtures__/projects/qux/moon.yml create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__installs_in_project_when_not_in_depman_workspace.snap create mode 100644 nextgen/action-graph/tests/utils.rs create mode 100644 nextgen/test-utils/src/platform_manager.rs diff --git a/Cargo.lock b/Cargo.lock index 88d62d1bb26..4789162576b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3041,11 +3041,13 @@ dependencies = [ "moon_query", "moon_rust_platform", "moon_task", + "moon_test_utils2", "petgraph", "proto_core", "rustc-hash", "starbase_sandbox", "thiserror", + "tokio", "tracing", ] @@ -3999,8 +4001,13 @@ version = "0.1.0" dependencies = [ "miette", "moon_config", + "moon_node_platform", + "moon_platform", "moon_project_graph", + "moon_rust_platform", + "moon_system_platform", "moon_vcs", + "proto_core", "starbase_events", "starbase_sandbox", ] diff --git a/nextgen/action-graph/Cargo.toml b/nextgen/action-graph/Cargo.toml index baf9cd711d5..89535036b6a 100644 --- a/nextgen/action-graph/Cargo.toml +++ b/nextgen/action-graph/Cargo.toml @@ -24,8 +24,10 @@ tracing = { workspace = true } [dev-dependencies] moon_config = { path = "../config" } +moon_test_utils2 = { path = "../test-utils" } proto_core = { workspace = true } starbase_sandbox = { workspace = true } +tokio = { workspace = true } # TODO remove moon_node_platform = { path = "../../crates/node/platform" } diff --git a/nextgen/action-graph/tests/__fixtures__/dep-workspace/.moon/toolchain.yml b/nextgen/action-graph/tests/__fixtures__/dep-workspace/.moon/toolchain.yml new file mode 100644 index 00000000000..88c6fbe7e3e --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/dep-workspace/.moon/toolchain.yml @@ -0,0 +1,2 @@ +node: + version: '20.0.0' diff --git a/nextgen/action-graph/tests/__fixtures__/dep-workspace/in/moon.yml b/nextgen/action-graph/tests/__fixtures__/dep-workspace/in/moon.yml new file mode 100644 index 00000000000..f1de67fbd7f --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/dep-workspace/in/moon.yml @@ -0,0 +1 @@ +language: javascript diff --git a/nextgen/action-graph/tests/__fixtures__/dep-workspace/out/moon.yml b/nextgen/action-graph/tests/__fixtures__/dep-workspace/out/moon.yml new file mode 100644 index 00000000000..30d34adef1c --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/dep-workspace/out/moon.yml @@ -0,0 +1 @@ +language: typescript diff --git a/nextgen/action-graph/tests/__fixtures__/dep-workspace/package.json b/nextgen/action-graph/tests/__fixtures__/dep-workspace/package.json new file mode 100644 index 00000000000..ab0250ab062 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/dep-workspace/package.json @@ -0,0 +1,3 @@ +{ + "workspaces": ["in"] +} diff --git a/nextgen/action-graph/tests/__fixtures__/projects/.moon/toolchain.yml b/nextgen/action-graph/tests/__fixtures__/projects/.moon/toolchain.yml new file mode 100644 index 00000000000..edf14aa87de --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/.moon/toolchain.yml @@ -0,0 +1,6 @@ +node: + version: '20.0.0' + packageManager: 'npm' + +rust: + version: '1.70.0' diff --git a/nextgen/action-graph/tests/__fixtures__/projects/bar/moon.yml b/nextgen/action-graph/tests/__fixtures__/projects/bar/moon.yml new file mode 100644 index 00000000000..f1de67fbd7f --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/bar/moon.yml @@ -0,0 +1 @@ +language: javascript diff --git a/nextgen/action-graph/tests/__fixtures__/projects/baz/moon.yml b/nextgen/action-graph/tests/__fixtures__/projects/baz/moon.yml new file mode 100644 index 00000000000..0a620fec5d0 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/baz/moon.yml @@ -0,0 +1,5 @@ +language: typescript + +toolchain: + node: + version: '18.0.0' diff --git a/nextgen/action-graph/tests/__fixtures__/projects/foo/moon.yml b/nextgen/action-graph/tests/__fixtures__/projects/foo/moon.yml new file mode 100644 index 00000000000..38118b204e1 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/foo/moon.yml @@ -0,0 +1 @@ +dependsOn: [bar] diff --git a/nextgen/action-graph/tests/__fixtures__/projects/package.json b/nextgen/action-graph/tests/__fixtures__/projects/package.json new file mode 100644 index 00000000000..b203b4c6c45 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/package.json @@ -0,0 +1,3 @@ +{ + "workspaces": ["*"] +} diff --git a/nextgen/action-graph/tests/__fixtures__/projects/qux/moon.yml b/nextgen/action-graph/tests/__fixtures__/projects/qux/moon.yml new file mode 100644 index 00000000000..22761ba7ee1 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/qux/moon.yml @@ -0,0 +1 @@ +language: rust diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index de9a843f356..b04b22dd4e5 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -1,24 +1,24 @@ #![allow(clippy::disallowed_names)] +mod utils; + use moon_action_graph::*; use moon_common::path::WorkspaceRelativePathBuf; use moon_common::Id; -use moon_config::{ - DependencyConfig, DependencyScope, DependencySource, LanguageType, NodeConfig, - ProjectToolchainCommonToolConfig, RustConfig, -}; +use moon_config::{NodeConfig, RustConfig}; use moon_node_platform::NodePlatform; use moon_platform::PlatformManager; use moon_platform_runtime::*; -use moon_project::Project; -use moon_project_graph::{GraphType as ProjectGraphType, ProjectGraph, ProjectNode}; +use moon_project_graph::ProjectGraph; use moon_rust_platform::RustPlatform; use moon_task::{Target, Task}; +use moon_test_utils2::generate_project_graph; use proto_core::ProtoEnvironment; -use rustc_hash::{FxHashMap, FxHashSet}; -use starbase_sandbox::assert_snapshot; -use std::path::PathBuf; +use rustc_hash::FxHashSet; +use starbase_sandbox::{assert_snapshot, create_sandbox}; +use std::path::Path; use std::sync::Arc; +use utils::ActionGraphContainer; fn create_task(id: &str, project: &str) -> Task { Task { @@ -28,88 +28,8 @@ fn create_task(id: &str, project: &str) -> Task { } } -fn create_project(id: &str) -> Project { - Project { - id: Id::raw(id), - ..Project::default() - } -} - -fn create_project_graph() -> ProjectGraph { - // Create projects - let mut foo = create_project("foo"); - foo.dependencies.insert( - Id::raw("bar"), - DependencyConfig { - id: Id::raw("bar"), - scope: DependencyScope::Production, - source: DependencySource::Explicit, - via: None, - }, - ); - - let mut bar = create_project("bar"); - bar.language = LanguageType::JavaScript; - bar.platform = PlatformType::Node; - - let mut baz = create_project("baz"); - baz.language = LanguageType::TypeScript; - baz.platform = PlatformType::Node; - baz.config.toolchain.node = Some(ProjectToolchainCommonToolConfig { - version: Some(UnresolvedVersionSpec::Version(Version::new(18, 0, 0))), - }); - - let mut qux = create_project("qux"); - qux.language = LanguageType::Rust; - qux.platform = PlatformType::Rust; - - // Map nodes and create graph (in order of expected insertion) - let mut nodes = FxHashMap::default(); - let mut graph = ProjectGraphType::new(); - - let bi = graph.add_node(bar); - nodes.insert( - "bar".into(), - ProjectNode { - alias: None, - index: bi, - source: "bar".into(), - }, - ); - - let fi = graph.add_node(foo); - nodes.insert( - "foo".into(), - ProjectNode { - alias: None, - index: fi, - source: "foo".into(), - }, - ); - - graph.add_edge(fi, bi, DependencyScope::Production); - - let zi = graph.add_node(baz); - nodes.insert( - "baz".into(), - ProjectNode { - alias: None, - index: zi, - source: "baz".into(), - }, - ); - - let qi = graph.add_node(qux); - nodes.insert( - "qux".into(), - ProjectNode { - alias: None, - index: qi, - source: "qux".into(), - }, - ); - - ProjectGraph::new(graph, nodes, &PathBuf::from(".")) +async fn create_project_graph() -> ProjectGraph { + generate_project_graph("projects").await } fn create_node_runtime() -> Runtime { @@ -126,10 +46,9 @@ fn create_rust_runtime() -> Runtime { ) } -fn create_platform_manager() -> PlatformManager { +fn create_platform_manager(root: &Path) -> PlatformManager { let mut manager = PlatformManager::default(); - let root = PathBuf::from("."); - let proto = Arc::new(ProtoEnvironment::new_testing(&root)); + let proto = Arc::new(ProtoEnvironment::new_testing(root)); manager.register( PlatformType::Node, @@ -139,7 +58,7 @@ fn create_platform_manager() -> PlatformManager { ..Default::default() }, &None, - &root, + root, proto.clone(), )), ); @@ -151,7 +70,7 @@ fn create_platform_manager() -> PlatformManager { version: Some(UnresolvedVersionSpec::Version(Version::new(1, 70, 0))), ..Default::default() }, - &root, + root, proto.clone(), )), ); @@ -202,13 +121,13 @@ mod action_graph { mod install_deps { use super::*; - #[test] - fn graphs() { - let pm = create_platform_manager(); - let pg = create_project_graph(); - let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + #[tokio::test] + async fn graphs() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); - let bar = pg.get("bar").unwrap(); + let bar = container.project_graph.get("bar").unwrap(); builder.install_deps(&bar, None).unwrap(); let graph = builder.build().unwrap(); @@ -228,13 +147,13 @@ mod action_graph { ); } - #[test] - fn ignores_dupes() { - let pm = create_platform_manager(); - let pg = create_project_graph(); - let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + #[tokio::test] + async fn ignores_dupes() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); - let bar = pg.get("bar").unwrap(); + let bar = container.project_graph.get("bar").unwrap(); builder.install_deps(&bar, None).unwrap(); builder.install_deps(&bar, None).unwrap(); builder.install_deps(&bar, None).unwrap(); @@ -254,19 +173,52 @@ mod action_graph { ] ); } + + #[tokio::test] + async fn installs_in_project_when_not_in_depman_workspace() { + let sandbox = create_sandbox("dep-workspace"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let inside = container.project_graph.get("in").unwrap(); + builder.install_deps(&inside, None).unwrap(); + + let outside = container.project_graph.get("out").unwrap(); + builder.install_deps(&outside, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::InstallProjectDeps { + project: Id::raw("out"), + runtime: create_node_runtime() + }, + ActionNode::InstallDeps { + runtime: create_node_runtime() + }, + ] + ); + } } mod run_task { use super::*; use starbase_sandbox::pretty_assertions::assert_eq; - #[test] - fn graphs() { - let pm = create_platform_manager(); - let pg = create_project_graph(); - let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + #[tokio::test] + async fn graphs() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); - let project = pg.get("bar").unwrap(); + let project = container.project_graph.get("bar").unwrap(); let mut task = create_task("build", "bar"); task.platform = PlatformType::Node; @@ -300,13 +252,13 @@ mod action_graph { ); } - #[test] - fn ignores_dupes() { - let pm = create_platform_manager(); - let pg = create_project_graph(); - let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + #[tokio::test] + async fn ignores_dupes() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); - let project = pg.get("bar").unwrap(); + let project = container.project_graph.get("bar").unwrap(); let mut task = create_task("build", "bar"); task.platform = PlatformType::Node; @@ -341,13 +293,13 @@ mod action_graph { ); } - #[test] - fn doesnt_graph_if_not_affected_by_touched_files() { - let pm = create_platform_manager(); - let pg = create_project_graph(); - let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + #[tokio::test] + async fn doesnt_graph_if_not_affected_by_touched_files() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); - let project = pg.get("bar").unwrap(); + let project = container.project_graph.get("bar").unwrap(); let mut task = create_task("build", "bar"); task.platform = PlatformType::Node; @@ -362,15 +314,15 @@ mod action_graph { assert!(topo(graph).is_empty()); } - #[test] - fn graphs_if_affected_by_touched_files() { - let pm = create_platform_manager(); - let pg = create_project_graph(); - let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + #[tokio::test] + async fn graphs_if_affected_by_touched_files() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); let file = WorkspaceRelativePathBuf::from("bar/file.js"); - let project = pg.get("bar").unwrap(); + let project = container.project_graph.get("bar").unwrap(); let mut task = create_task("build", "bar"); task.platform = PlatformType::Node; @@ -385,14 +337,14 @@ mod action_graph { assert!(!topo(graph).is_empty()); } - #[test] - fn task_can_have_a_diff_platform_from_project() { - let pm = create_platform_manager(); - let pg = create_project_graph(); - let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + #[tokio::test] + async fn task_can_have_a_diff_platform_from_project() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); // node - let project = pg.get("bar").unwrap(); + let project = container.project_graph.get("bar").unwrap(); let mut task = create_task("build", "bar"); task.platform = PlatformType::Rust; @@ -429,13 +381,13 @@ mod action_graph { ); } - #[test] - fn sets_interactive() { - let pm = create_platform_manager(); - let pg = create_project_graph(); - let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + #[tokio::test] + async fn sets_interactive() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); - let project = pg.get("bar").unwrap(); + let project = container.project_graph.get("bar").unwrap(); let mut task = create_task("build", "bar"); task.options.interactive = true; @@ -455,13 +407,13 @@ mod action_graph { ); } - #[test] - fn sets_persistent() { - let pm = create_platform_manager(); - let pg = create_project_graph(); - let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + #[tokio::test] + async fn sets_persistent() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); - let project = pg.get("bar").unwrap(); + let project = container.project_graph.get("bar").unwrap(); let mut task = create_task("build", "bar"); task.options.persistent = true; @@ -485,8 +437,8 @@ mod action_graph { mod setup_tool { use super::*; - #[test] - fn graphs() { + #[tokio::test] + async fn graphs() { let pg = ProjectGraph::default(); let mut builder = ActionGraphBuilder::new(&pg).unwrap(); let system = Runtime::system(); @@ -511,8 +463,8 @@ mod action_graph { ); } - #[test] - fn graphs_same_platform() { + #[tokio::test] + async fn graphs_same_platform() { let pg = ProjectGraph::default(); let mut builder = ActionGraphBuilder::new(&pg).unwrap(); @@ -544,8 +496,8 @@ mod action_graph { ); } - #[test] - fn ignores_dupes() { + #[tokio::test] + async fn ignores_dupes() { let pg = ProjectGraph::default(); let mut builder = ActionGraphBuilder::new(&pg).unwrap(); let system = Runtime::system(); @@ -568,9 +520,9 @@ mod action_graph { mod sync_project { use super::*; - #[test] - fn graphs_single() { - let pg = create_project_graph(); + #[tokio::test] + async fn graphs_single() { + let pg = create_project_graph().await; let mut builder = ActionGraphBuilder::new(&pg).unwrap(); let bar = pg.get("bar").unwrap(); @@ -594,9 +546,9 @@ mod action_graph { ); } - #[test] - fn graphs_single_with_dep() { - let pg = create_project_graph(); + #[tokio::test] + async fn graphs_single_with_dep() { + let pg = create_project_graph().await; let mut builder = ActionGraphBuilder::new(&pg).unwrap(); let foo = pg.get("foo").unwrap(); @@ -624,9 +576,9 @@ mod action_graph { ); } - #[test] - fn graphs_multiple() { - let pg = create_project_graph(); + #[tokio::test] + async fn graphs_multiple() { + let pg = create_project_graph().await; let mut builder = ActionGraphBuilder::new(&pg).unwrap(); let foo = pg.get("foo").unwrap(); @@ -664,9 +616,9 @@ mod action_graph { ); } - #[test] - fn ignores_dupes() { - let pg = create_project_graph(); + #[tokio::test] + async fn ignores_dupes() { + let pg = create_project_graph().await; let mut builder = ActionGraphBuilder::new(&pg).unwrap(); let foo = pg.get("foo").unwrap(); @@ -696,10 +648,10 @@ mod action_graph { ); } - #[test] - fn inherits_platform_tool() { - let pm = create_platform_manager(); - let pg = create_project_graph(); + #[tokio::test] + async fn inherits_platform_tool() { + let pg = create_project_graph().await; + let pm = create_platform_manager(&pg.workspace_root); let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); let bar = pg.get("bar").unwrap(); @@ -733,10 +685,10 @@ mod action_graph { ); } - #[test] - fn supports_platform_override() { - let pm = create_platform_manager(); - let pg = create_project_graph(); + #[tokio::test] + async fn supports_platform_override() { + let pg = create_project_graph().await; + let pm = create_platform_manager(&pg.workspace_root); let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); let bar = pg.get("bar").unwrap(); @@ -780,8 +732,8 @@ mod action_graph { mod sync_workspace { use super::*; - #[test] - fn graphs() { + #[tokio::test] + async fn graphs() { let pg = ProjectGraph::default(); let mut builder = ActionGraphBuilder::new(&pg).unwrap(); @@ -793,8 +745,8 @@ mod action_graph { assert_eq!(topo(graph), vec![ActionNode::SyncWorkspace]); } - #[test] - fn ignores_dupes() { + #[tokio::test] + async fn ignores_dupes() { let pg = ProjectGraph::default(); let mut builder = ActionGraphBuilder::new(&pg).unwrap(); diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__installs_in_project_when_not_in_depman_workspace.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__installs_in_project_when_not_in_depman_workspace.snap new file mode 100644 index 00000000000..3e423cd537e --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__installs_in_project_when_not_in_depman_workspace.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="InstallNodeDeps(20.0.0)" ] + 3 [ label="InstallNodeDepsInProject(20.0.0, out)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 1 [ ] +} + diff --git a/nextgen/action-graph/tests/utils.rs b/nextgen/action-graph/tests/utils.rs new file mode 100644 index 00000000000..0c61e18096c --- /dev/null +++ b/nextgen/action-graph/tests/utils.rs @@ -0,0 +1,27 @@ +use moon_action_graph::ActionGraphBuilder; +use moon_platform::PlatformManager; +use moon_project_graph::ProjectGraph; +use moon_test_utils2::{ + generate_platform_manager_from_sandbox, generate_project_graph_from_sandbox, +}; +use std::path::{Path, PathBuf}; + +pub struct ActionGraphContainer { + pub platform_manager: PlatformManager, + pub project_graph: ProjectGraph, + pub workspace_root: PathBuf, +} + +impl ActionGraphContainer { + pub async fn new(root: &Path) -> Self { + Self { + platform_manager: generate_platform_manager_from_sandbox(root).await, + project_graph: generate_project_graph_from_sandbox(root).await, + workspace_root: root.to_path_buf(), + } + } + + pub fn create_builder(&self) -> ActionGraphBuilder { + ActionGraphBuilder::with_platforms(&self.platform_manager, &self.project_graph).unwrap() + } +} diff --git a/nextgen/config/tests/inherited_tasks_config_test.rs b/nextgen/config/tests/inherited_tasks_config_test.rs index bdf2be65f73..b77ab251d92 100644 --- a/nextgen/config/tests/inherited_tasks_config_test.rs +++ b/nextgen/config/tests/inherited_tasks_config_test.rs @@ -198,8 +198,6 @@ fileGroups: let url = server.url("/config.yml"); - dbg!(&url); - sandbox.create_file( "tasks.yml", format!( diff --git a/nextgen/test-utils/Cargo.toml b/nextgen/test-utils/Cargo.toml index 1056fb1ca62..f290514b133 100644 --- a/nextgen/test-utils/Cargo.toml +++ b/nextgen/test-utils/Cargo.toml @@ -12,5 +12,12 @@ moon_config = { path = "../config" } moon_project_graph = { path = "../project-graph" } moon_vcs = { path = "../vcs" } miette = { workspace = true } +proto_core = { workspace = true } starbase_events = { workspace = true } starbase_sandbox = { workspace = true } + +# TODO +moon_platform = { path = "../../crates/core/platform" } +moon_node_platform = { path = "../../crates/node/platform" } +moon_rust_platform = { path = "../../crates/rust/platform" } +moon_system_platform = { path = "../../crates/system/platform" } diff --git a/nextgen/test-utils/src/lib.rs b/nextgen/test-utils/src/lib.rs index acb8deedbf9..5b73bce286f 100644 --- a/nextgen/test-utils/src/lib.rs +++ b/nextgen/test-utils/src/lib.rs @@ -1,3 +1,5 @@ +mod platform_manager; mod project_graph; +pub use platform_manager::*; pub use project_graph::*; diff --git a/nextgen/test-utils/src/platform_manager.rs b/nextgen/test-utils/src/platform_manager.rs new file mode 100644 index 00000000000..50fffc38380 --- /dev/null +++ b/nextgen/test-utils/src/platform_manager.rs @@ -0,0 +1,32 @@ +use moon_config::{PlatformType, ToolchainConfig, ToolsConfig}; +use moon_node_platform::NodePlatform; +use moon_platform::PlatformManager; +use moon_rust_platform::RustPlatform; +use moon_system_platform::SystemPlatform; +use proto_core::ProtoEnvironment; +use std::path::Path; +use std::sync::Arc; + +pub async fn generate_platform_manager_from_sandbox(root: &Path) -> PlatformManager { + let proto = Arc::new(ProtoEnvironment::new_testing(root)); + let config = ToolchainConfig::load_from(root, &ToolsConfig::default()).unwrap(); + let mut manager = PlatformManager::default(); + + if let Some(node_config) = &config.node { + manager.register( + PlatformType::Node, + Box::new(NodePlatform::new(node_config, &None, root, proto.clone())), + ); + } + + if let Some(rust_config) = &config.rust { + manager.register( + PlatformType::Rust, + Box::new(RustPlatform::new(rust_config, root, proto.clone())), + ); + } + + manager.register(PlatformType::System, Box::::default()); + + manager +} diff --git a/nextgen/test-utils/src/project_graph.rs b/nextgen/test-utils/src/project_graph.rs index 8d25437fdfa..5478968b90d 100644 --- a/nextgen/test-utils/src/project_graph.rs +++ b/nextgen/test-utils/src/project_graph.rs @@ -1,6 +1,6 @@ use moon_config::{ InheritedTasksEntry, InheritedTasksManager, NodeConfig, PartialInheritedTasksConfig, - PartialTaskConfig, ToolchainConfig, WorkspaceConfig, WorkspaceProjects, + PartialTaskConfig, ToolchainConfig, ToolsConfig, WorkspaceConfig, WorkspaceProjects, }; use moon_project_graph::{ DetectLanguageEvent, DetectPlatformEvent, ExtendProjectEvent, ExtendProjectGraphEvent, @@ -23,7 +23,10 @@ pub struct ProjectGraphContainer { impl ProjectGraphContainer { pub fn new(root: &Path) -> Self { + let proto = ToolsConfig::default(); let mut graph = Self { + inherited_tasks: InheritedTasksManager::load_from(root).unwrap(), + toolchain_config: ToolchainConfig::load_from(root, &proto).unwrap(), workspace_root: root.to_path_buf(), ..Default::default() }; @@ -44,7 +47,9 @@ impl ProjectGraphContainer { ); // Always use the node platform - graph.toolchain_config.node = Some(NodeConfig::default()); + if graph.toolchain_config.node.is_none() { + graph.toolchain_config.node = Some(NodeConfig::default()); + } // Use folders as project names graph.workspace_config.projects = WorkspaceProjects::Globs(vec!["*".into()]); @@ -109,15 +114,15 @@ pub async fn generate_project_graph(fixture: &str) -> ProjectGraph { generate_project_graph_from_sandbox(create_sandbox(fixture).path()).await } -pub async fn generate_project_graph_from_sandbox(path: &Path) -> ProjectGraph { - generate_project_graph_with_changes(path, |_| {}).await +pub async fn generate_project_graph_from_sandbox(root: &Path) -> ProjectGraph { + generate_project_graph_with_changes(root, |_| {}).await } -pub async fn generate_project_graph_with_changes(path: &Path, mut op: F) -> ProjectGraph +pub async fn generate_project_graph_with_changes(root: &Path, mut op: F) -> ProjectGraph where F: FnMut(&mut ProjectGraphContainer), { - let mut container = ProjectGraphContainer::new(path); + let mut container = ProjectGraphContainer::new(root); op(&mut container); From 26b53cc27bb14329ac41275e295e70705324a222 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Fri, 6 Oct 2023 14:32:15 -0700 Subject: [PATCH 16/22] Add target tests. --- .../action-graph/src/action_graph_builder.rs | 21 +- .../tests/__fixtures__/tasks/base/moon.yml | 3 + .../tests/__fixtures__/tasks/client/moon.yml | 14 ++ .../tests/__fixtures__/tasks/common/moon.yml | 9 + .../tests/__fixtures__/tasks/server/moon.yml | 10 + .../action-graph/tests/action_graph_test.rs | 207 +++++++++++++++++- ...n_graph__run_task_by_target__runs_all.snap | 31 +++ ...n_task_by_target__runs_all_with_query.snap | 14 ++ ...__run_task_by_target__runs_by_project.snap | 23 ++ ...n_graph__run_task_by_target__runs_tag.snap | 25 +++ ..._by_target_locator__runs_by_file_path.snap | 14 ++ ...ask_by_target_locator__runs_by_target.snap | 14 ++ 12 files changed, 378 insertions(+), 7 deletions(-) create mode 100644 nextgen/action-graph/tests/__fixtures__/tasks/base/moon.yml create mode 100644 nextgen/action-graph/tests/__fixtures__/tasks/client/moon.yml create mode 100644 nextgen/action-graph/tests/__fixtures__/tasks/common/moon.yml create mode 100644 nextgen/action-graph/tests/__fixtures__/tasks/server/moon.yml create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all_with_query.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_by_project.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_tag.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_file_path.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_target.snap diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs index 70f178a1e32..1f2a7f495d5 100644 --- a/nextgen/action-graph/src/action_graph_builder.rs +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -80,7 +80,7 @@ impl<'app> ActionGraphBuilder<'app> { &mut self, project: &Project, task: Option<&Task>, - ) -> miette::Result { + ) -> miette::Result> { let mut in_project = false; // If project is NOT in the package manager workspace, then we should @@ -107,8 +107,12 @@ impl<'app> ActionGraphBuilder<'app> { } }; + if node.get_runtime().platform.is_system() { + return Ok(None); + } + if let Some(index) = self.get_index_from_node(&node) { - return Ok(*index); + return Ok(Some(*index)); } // Before we install deps, we must ensure the language has been installed @@ -117,7 +121,7 @@ impl<'app> ActionGraphBuilder<'app> { self.link_requirements(index, vec![setup_tool_index]); - Ok(index) + Ok(Some(index)) } pub fn run_task( @@ -150,10 +154,15 @@ impl<'app> ActionGraphBuilder<'app> { } // We should install deps & sync projects *before* running targets - let install_deps_index = self.install_deps(project, Some(task))?; - let sync_project_index = self.sync_project(project)?; + let mut reqs = vec![]; + + if let Some(install_deps_index) = self.install_deps(project, Some(task))? { + reqs.push(install_deps_index); + } + + reqs.push(self.sync_project(project)?); + let index = self.insert_node(node); - let mut reqs = vec![install_deps_index, sync_project_index]; // And we also need to create edges for task dependencies if !task.deps.is_empty() { diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/base/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/base/moon.yml new file mode 100644 index 00000000000..606e4cef104 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/base/moon.yml @@ -0,0 +1,3 @@ +tasks: + build: + command: 'build' diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/client/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/client/moon.yml new file mode 100644 index 00000000000..59d5f8f9696 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/client/moon.yml @@ -0,0 +1,14 @@ +language: 'javascript' + +dependsOn: ['common', 'server'] + +tags: ['frontend'] + +tasks: + build: + command: 'build' + deps: ['^:build'] + lint: + command: 'lint' + test: + command: 'test' diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/common/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/common/moon.yml new file mode 100644 index 00000000000..6a36e76ede4 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/common/moon.yml @@ -0,0 +1,9 @@ +dependsOn: ['base'] + +tags: ['frontend'] + +tasks: + build: + command: 'build' + lint: + command: 'lint' diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/server/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/server/moon.yml new file mode 100644 index 00000000000..1cdb6dfa0c2 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/server/moon.yml @@ -0,0 +1,10 @@ +language: 'rust' + +tasks: + build: + command: 'build' + deps: ['^:build'] + lint: + command: 'lint' + test: + command: 'test' diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index b04b22dd4e5..30f183ac0f3 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -11,7 +11,7 @@ use moon_platform::PlatformManager; use moon_platform_runtime::*; use moon_project_graph::ProjectGraph; use moon_rust_platform::RustPlatform; -use moon_task::{Target, Task}; +use moon_task::{Target, TargetLocator, Task}; use moon_test_utils2::generate_project_graph; use proto_core::ProtoEnvironment; use rustc_hash::FxHashSet; @@ -434,6 +434,211 @@ mod action_graph { } } + mod run_task_by_target { + use super::*; + + #[tokio::test] + #[should_panic(expected = "Dependencies scope (^:) is not supported in run contexts.")] + async fn errors_on_parent_scope() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("^:build").unwrap(), None) + .unwrap(); + } + + #[tokio::test] + #[should_panic(expected = "Self scope (~:) is not supported in run contexts.")] + async fn errors_on_self_scope() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("~:build").unwrap(), None) + .unwrap(); + } + + #[tokio::test] + async fn runs_all() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse(":build").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn runs_all_with_query() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder.set_query("language=rust").unwrap(); + + builder + .run_task_by_target(Target::parse(":build").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn runs_all_no_nodes() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse(":unknown").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert!(graph.is_empty()); + } + + #[tokio::test] + async fn runs_by_project() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("client:lint").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + #[should_panic(expected = "No project has been configured with the name or alias unknown.")] + async fn errors_for_unknown_project() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("unknown:build").unwrap(), None) + .unwrap(); + } + + #[tokio::test] + #[should_panic(expected = "Unknown task unknown for project server.")] + async fn errors_for_unknown_project_task() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("server:unknown").unwrap(), None) + .unwrap(); + } + + #[tokio::test] + async fn runs_tag() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("#frontend:lint").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn runs_tag_no_nodes() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("#unknown:lint").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert!(graph.is_empty()); + } + } + + mod run_task_by_target_locator { + use super::*; + + #[tokio::test] + async fn runs_by_target() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target_locator( + TargetLocator::Qualified(Target::parse("server:build").unwrap()), + None, + ) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn runs_by_file_path() { + let sandbox = create_sandbox("tasks"); + let mut container = ActionGraphContainer::new(sandbox.path()).await; + + container.project_graph.working_dir = sandbox.path().join("server/nested"); + + let mut builder = container.create_builder(); + + builder + .run_task_by_target_locator( + TargetLocator::TaskFromWorkingDir(Id::raw("lint")), + None, + ) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + #[should_panic(expected = "No project could be located starting from path unknown/path.")] + async fn errors_if_no_project_by_path() { + let sandbox = create_sandbox("tasks"); + let mut container = ActionGraphContainer::new(sandbox.path()).await; + + container.project_graph.working_dir = sandbox.path().join("unknown/path"); + + let mut builder = container.create_builder(); + + builder + .run_task_by_target_locator( + TargetLocator::TaskFromWorkingDir(Id::raw("lint")), + None, + ) + .unwrap(); + } + } + mod setup_tool { use super::*; diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap new file mode 100644 index 00000000000..86a4829917c --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap @@ -0,0 +1,31 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(base)" ] + 3 [ label="RunTask(base:build)" ] + 4 [ label="SyncSystemProject(common)" ] + 5 [ label="RunTask(common:build)" ] + 6 [ label="SyncSystemProject(client)" ] + 7 [ label="SyncSystemProject(server)" ] + 8 [ label="RunTask(client:build)" ] + 9 [ label="RunTask(server:build)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] + 4 -> 1 [ ] + 4 -> 2 [ ] + 5 -> 4 [ ] + 7 -> 1 [ ] + 6 -> 1 [ ] + 6 -> 7 [ ] + 6 -> 4 [ ] + 9 -> 7 [ ] + 8 -> 6 [ ] + 8 -> 5 [ ] + 8 -> 9 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all_with_query.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all_with_query.snap new file mode 100644 index 00000000000..6dc1e1b0df3 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all_with_query.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(server)" ] + 3 [ label="RunTask(server:build)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_by_project.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_by_project.snap new file mode 100644 index 00000000000..6e78aa95cfd --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_by_project.snap @@ -0,0 +1,23 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(client)" ] + 3 [ label="SyncSystemProject(server)" ] + 4 [ label="SyncSystemProject(common)" ] + 5 [ label="SyncSystemProject(base)" ] + 6 [ label="RunTask(client:lint)" ] + 1 -> 0 [ ] + 3 -> 1 [ ] + 5 -> 1 [ ] + 4 -> 1 [ ] + 4 -> 5 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] + 2 -> 4 [ ] + 6 -> 2 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_tag.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_tag.snap new file mode 100644 index 00000000000..8943f0f12b9 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_tag.snap @@ -0,0 +1,25 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(client)" ] + 3 [ label="SyncSystemProject(server)" ] + 4 [ label="SyncSystemProject(common)" ] + 5 [ label="SyncSystemProject(base)" ] + 6 [ label="RunTask(client:lint)" ] + 7 [ label="RunTask(common:lint)" ] + 1 -> 0 [ ] + 3 -> 1 [ ] + 5 -> 1 [ ] + 4 -> 1 [ ] + 4 -> 5 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] + 2 -> 4 [ ] + 6 -> 2 [ ] + 7 -> 4 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_file_path.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_file_path.snap new file mode 100644 index 00000000000..d074f59cf86 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_file_path.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(server)" ] + 3 [ label="RunTask(server:lint)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_target.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_target.snap new file mode 100644 index 00000000000..6dc1e1b0df3 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_target.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(server)" ] + 3 [ label="RunTask(server:build)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] +} + From b7cde169af8a4fb877c95b683560b0d5adff08fd Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Fri, 6 Oct 2023 14:40:16 -0700 Subject: [PATCH 17/22] Test deps. --- .../tests/__fixtures__/tasks/cases/moon.yml | 15 ++++++++ .../action-graph/tests/action_graph_test.rs | 36 +++++++++++++++++++ ...k_dependencies__runs_deps_in_parallel.snap | 23 ++++++++++++ ...ask_dependencies__runs_deps_in_serial.snap | 23 ++++++++++++ 4 files changed, 97 insertions(+) create mode 100644 nextgen/action-graph/tests/__fixtures__/tasks/cases/moon.yml create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_parallel.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_serial.snap diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/cases/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/cases/moon.yml new file mode 100644 index 00000000000..be0e0b3cb2e --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/cases/moon.yml @@ -0,0 +1,15 @@ +tasks: + a: + command: 'a' + b: + command: 'b' + c: + command: 'd' + + parallel: + deps: [c, a, b] + + serial: + deps: [b, c, a] + options: + runDepsInParallel: false diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index 30f183ac0f3..9e67261dc52 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -434,6 +434,42 @@ mod action_graph { } } + mod run_task_dependencies { + use super::*; + + #[tokio::test] + async fn runs_deps_in_parallel() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("cases").unwrap(); + let task = project.get_task("parallel").unwrap(); + + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn runs_deps_in_serial() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("cases").unwrap(); + let task = project.get_task("serial").unwrap(); + + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + } + mod run_task_by_target { use super::*; diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_parallel.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_parallel.snap new file mode 100644 index 00000000000..4e3221ce9b1 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_parallel.snap @@ -0,0 +1,23 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(cases)" ] + 3 [ label="RunTask(cases:parallel)" ] + 4 [ label="RunTask(cases:c)" ] + 5 [ label="RunTask(cases:a)" ] + 6 [ label="RunTask(cases:b)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 4 -> 2 [ ] + 5 -> 2 [ ] + 6 -> 2 [ ] + 3 -> 2 [ ] + 3 -> 4 [ ] + 3 -> 5 [ ] + 3 -> 6 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_serial.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_serial.snap new file mode 100644 index 00000000000..26ec9e0deb6 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_serial.snap @@ -0,0 +1,23 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(cases)" ] + 3 [ label="RunTask(cases:serial)" ] + 4 [ label="RunTask(cases:b)" ] + 5 [ label="RunTask(cases:c)" ] + 6 [ label="RunTask(cases:a)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 4 -> 2 [ ] + 5 -> 2 [ ] + 5 -> 4 [ ] + 6 -> 2 [ ] + 6 -> 5 [ ] + 3 -> 2 [ ] + 3 -> 6 [ ] +} + From 2b57577c86d35590b324c4e22d19a50622a4ee10 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Fri, 6 Oct 2023 15:03:12 -0700 Subject: [PATCH 18/22] Rework peer dep. --- .../action-graph/tests/action_graph_test.rs | 4 +- .../project-builder/src/project_builder.rs | 52 ++++++++++++++++--- .../src/project_graph_builder.rs | 41 ++------------- packages/cli/CHANGELOG.md | 26 ++++++++++ 4 files changed, 77 insertions(+), 46 deletions(-) diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index 9e67261dc52..2066cb8c29d 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -446,7 +446,7 @@ mod action_graph { let project = container.project_graph.get("cases").unwrap(); let task = project.get_task("parallel").unwrap(); - builder.run_task(&project, &task, None).unwrap(); + builder.run_task(&project, task, None).unwrap(); let graph = builder.build().unwrap(); @@ -462,7 +462,7 @@ mod action_graph { let project = container.project_graph.get("cases").unwrap(); let task = project.get_task("serial").unwrap(); - builder.run_task(&project, &task, None).unwrap(); + builder.run_task(&project, task, None).unwrap(); let graph = builder.build().unwrap(); diff --git a/nextgen/project-builder/src/project_builder.rs b/nextgen/project-builder/src/project_builder.rs index 5fbbbf469ef..51aec8e3699 100644 --- a/nextgen/project-builder/src/project_builder.rs +++ b/nextgen/project-builder/src/project_builder.rs @@ -1,12 +1,13 @@ use moon_common::path::WorkspaceRelativePath; use moon_common::{color, consts, Id}; use moon_config::{ - DependencyConfig, DependencySource, InheritedTasksManager, InheritedTasksResult, LanguageType, - PlatformType, ProjectConfig, ProjectDependsOn, TaskConfig, ToolchainConfig, + DependencyConfig, DependencyScope, DependencySource, InheritedTasksManager, + InheritedTasksResult, LanguageType, PlatformType, ProjectConfig, ProjectDependsOn, TaskConfig, + ToolchainConfig, }; use moon_file_group::FileGroup; use moon_project::Project; -use moon_task::Task; +use moon_task::{TargetScope, Task}; use moon_task_builder::{DetectPlatformEvent, TasksBuilder, TasksBuilderContext}; use rustc_hash::FxHashMap; use starbase_events::{Emitter, Event}; @@ -197,11 +198,13 @@ impl<'app> ProjectBuilder<'app> { #[tracing::instrument(name = "project", skip_all)] pub async fn build(mut self) -> miette::Result { + let tasks = self.build_tasks().await?; + let mut project = Project { alias: self.alias.map(|a| a.to_owned()), - dependencies: self.build_dependencies()?, + dependencies: self.build_dependencies(&tasks)?, file_groups: self.build_file_groups()?, - tasks: self.build_tasks().await?, + tasks, id: self.id.to_owned(), language: self.language, platform: self.platform, @@ -220,7 +223,10 @@ impl<'app> ProjectBuilder<'app> { Ok(project) } - fn build_dependencies(&self) -> miette::Result> { + fn build_dependencies( + &self, + tasks: &BTreeMap, + ) -> miette::Result> { let mut deps = FxHashMap::default(); trace!(id = self.id.as_str(), "Building project dependencies"); @@ -237,7 +243,41 @@ impl<'app> ProjectBuilder<'app> { deps.insert(dep_config.id.clone(), dep_config); } + } + + // Tasks can depend on arbitray projects, so include them also + for task_config in tasks.values() { + for task_dep in &task_config.deps { + if let TargetScope::Project(dep_id) = &task_dep.scope { + // Already a dependency, or references self + if deps.contains_key(dep_id) + || self.id == dep_id + || self.alias.as_ref().is_some_and(|a| *a == dep_id.as_str()) + { + continue; + } + + trace!( + id = self.id.as_str(), + dep = dep_id.as_str(), + task = task_config.target.as_str(), + "Marking arbitrary project as a peer dependency because of a task dependency" + ); + + deps.insert( + dep_id.to_owned(), + DependencyConfig { + id: dep_id.to_owned(), + scope: DependencyScope::Peer, + source: DependencySource::Implicit, + via: Some(task_config.target.to_string()), + }, + ); + } + } + } + if !deps.is_empty() { trace!( id = self.id.as_str(), deps = ?deps.keys().map(|k| k.as_str()).collect::>(), diff --git a/nextgen/project-graph/src/project_graph_builder.rs b/nextgen/project-graph/src/project_graph_builder.rs index 8e02337e838..b3a721cc6f7 100644 --- a/nextgen/project-graph/src/project_graph_builder.rs +++ b/nextgen/project-graph/src/project_graph_builder.rs @@ -1,5 +1,4 @@ -use crate::project_events::ExtendProjectEvent; -use crate::project_events::ExtendProjectGraphEvent; +use crate::project_events::{ExtendProjectEvent, ExtendProjectGraphEvent}; use crate::project_graph::{GraphType, ProjectGraph, ProjectNode}; use crate::project_graph_cache::ProjectsState; use crate::project_graph_error::ProjectGraphError; @@ -7,17 +6,13 @@ use crate::project_graph_hash::ProjectGraphHash; use crate::projects_locator::locate_projects_with_globs; use async_recursion::async_recursion; use moon_cache::CacheEngine; -use moon_common::is_test_env; use moon_common::path::{to_virtual_string, WorkspaceRelativePath, WorkspaceRelativePathBuf}; -use moon_common::{color, consts, Id}; -use moon_config::{ - DependencyScope, InheritedTasksManager, ToolchainConfig, WorkspaceConfig, WorkspaceProjects, -}; +use moon_common::{color, consts, is_test_env, Id}; +use moon_config::{InheritedTasksManager, ToolchainConfig, WorkspaceConfig, WorkspaceProjects}; use moon_hash::HashEngine; use moon_project::Project; use moon_project_builder::{DetectLanguageEvent, ProjectBuilder, ProjectBuilderContext}; use moon_project_constraints::{enforce_project_type_relationships, enforce_tag_relationships}; -use moon_task::TargetScope; use moon_task_builder::DetectPlatformEvent; use moon_vcs::BoxedVcs; use petgraph::graph::DiGraph; @@ -256,36 +251,6 @@ impl<'app> ProjectGraphBuilder<'app> { } } - // Tasks can depend on arbitray projects, so include them also - for (task_id, task_config) in &project.tasks { - for task_dep in &task_config.deps { - if let TargetScope::Project(dep_id) = &task_dep.scope { - if - // Already a dependency - project.dependencies.contains_key(dep_id) || - // Don't reference itself - project.matches_locator(dep_id.as_str()) - { - continue; - } - - if cycle.contains(dep_id) { - warn!( - id = id.as_str(), - dependency_id = dep_id.as_str(), - task_id = task_id.as_str(), - "Encountered a dependency cycle (from task); will disconnect nodes to avoid recursion", - ); - } else { - edges.push(( - self.internal_load(dep_id, cycle).await?, - DependencyScope::Peer, - )); - } - } - } - } - // Insert into the graph and connect edges let index = self.graph.add_node(project); diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index c966243c309..7fd477bdb97 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -10,6 +10,32 @@ - More accurately monitors signals (ctrl+c) and shutdowns. - Tasks can now be configured with a timeout. +## Unreleased + +#### 💥 Breaking + +- Tasks that depend (via `deps`) on other tasks from arbitrary projects (the parent project doesn't + implicitly or explicitly depend on the other project) will now automatically mark that other + project as a "peer" dependency. For example, "b" becomes a peer dependency for "a". + + ```yaml + tasks: + build: + deps: ['b:build'] + + # Now internally becomes: + dependsOn: + - id: 'b' + scope: 'peer' + + tasks: + build: + deps: ['b:build'] + ``` + + We're marking this as a breaking change as this could subtly introduce cycles in the project graph + that weren't present before, and for Node.js projects, this may inject `peerDependencies`. + ## 1.14.5 #### 🐞 Fixes From e579e30617e4ec6f8a16ead6f0ea30049cfa5d48 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Fri, 6 Oct 2023 15:45:27 -0700 Subject: [PATCH 19/22] Add dependents. --- .../action-graph/src/action_graph_builder.rs | 58 +++++++++++++++---- .../__fixtures__/tasks/deps-external/moon.yml | 3 + .../tasks/{cases => deps}/moon.yml | 11 ++++ .../action-graph/tests/action_graph_test.rs | 54 ++++++++++++++++- ...n_graph__run_task_by_target__runs_all.snap | 26 ++++----- ...task_dependencies__can_create_a_chain.snap | 20 +++++++ ...pendencies__doesnt_include_dependents.snap | 14 +++++ ...ask_dependencies__includes_dependents.snap | 23 ++++++++ ...k_dependencies__runs_deps_in_parallel.snap | 10 ++-- ...ask_dependencies__runs_deps_in_serial.snap | 10 ++-- packages/cli/CHANGELOG.md | 14 +++++ 11 files changed, 206 insertions(+), 37 deletions(-) create mode 100644 nextgen/action-graph/tests/__fixtures__/tasks/deps-external/moon.yml rename nextgen/action-graph/tests/__fixtures__/tasks/{cases => deps}/moon.yml (59%) create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__can_create_a_chain.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__doesnt_include_dependents.snap create mode 100644 nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__includes_dependents.snap diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs index 1f2a7f495d5..ee51775871d 100644 --- a/nextgen/action-graph/src/action_graph_builder.rs +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -12,9 +12,9 @@ use tracing::{debug, trace}; type TouchedFilePaths = FxHashSet; -// TODO: run task dependents - pub struct ActionGraphBuilder<'app> { + pub include_dependents: bool, + all_query: Option, graph: DiGraph, indices: FxHashMap, @@ -34,6 +34,7 @@ impl<'app> ActionGraphBuilder<'app> { Ok(ActionGraphBuilder { all_query: None, graph: DiGraph::new(), + include_dependents: false, indices: FxHashMap::default(), platform_manager, project_graph, @@ -171,28 +172,28 @@ impl<'app> ActionGraphBuilder<'app> { "Adding dependencies for task {}", color::label(&task.target), ); - - // We don't pass touched files to dependencies, because if the parent - // task is affected/going to run, then so should all of these! - reqs.extend(self.run_task_dependencies(task, None)?); + reqs.extend(self.run_task_dependencies(task)?); } self.link_requirements(index, reqs); + // And possibly dependents + if self.include_dependents { + self.run_task_dependents(task)?; + } + Ok(Some(index)) } - pub fn run_task_dependencies( - &mut self, - task: &Task, - touched_files: Option<&TouchedFilePaths>, - ) -> miette::Result> { + // We don't pass touched files to dependencies, because if the parent + // task is affected/going to run, then so should all of these! + pub fn run_task_dependencies(&mut self, task: &Task) -> miette::Result> { let parallel = task.options.run_deps_in_parallel; let mut indices = vec![]; let mut previous_target_index = None; for dep_target in &task.deps { - let (_, dep_indices) = self.run_task_by_target(dep_target, touched_files)?; + let (_, dep_indices) = self.run_task_by_target(dep_target, None)?; for dep_index in dep_indices { // When parallel, parent depends on child @@ -215,6 +216,39 @@ impl<'app> ActionGraphBuilder<'app> { Ok(indices) } + // This is costly, is there a better way to do this? + pub fn run_task_dependents(&mut self, task: &Task) -> miette::Result> { + let mut indices = vec![]; + + if let TargetScope::Project(project_locator) = &task.target.scope { + let project = self.project_graph.get(project_locator)?; + + // From self project + for dep_task in project.tasks.values() { + if dep_task.deps.contains(&task.target) { + if let Some(index) = self.run_task(&project, dep_task, None)? { + indices.push(index); + } + } + } + + // From other projects + for dependent_id in self.project_graph.dependents_of(&project)? { + let dep_project = self.project_graph.get(dependent_id)?; + + for dep_task in dep_project.tasks.values() { + if dep_task.deps.contains(&task.target) { + if let Some(index) = self.run_task(&dep_project, dep_task, None)? { + indices.push(index); + } + } + } + } + } + + Ok(indices) + } + pub fn run_task_by_target>( &mut self, target: T, diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/deps-external/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/deps-external/moon.yml new file mode 100644 index 00000000000..bd188ab4774 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/deps-external/moon.yml @@ -0,0 +1,3 @@ +tasks: + external: + deps: ['deps:base'] diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/cases/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/deps/moon.yml similarity index 59% rename from nextgen/action-graph/tests/__fixtures__/tasks/cases/moon.yml rename to nextgen/action-graph/tests/__fixtures__/tasks/deps/moon.yml index be0e0b3cb2e..98363d5b0a0 100644 --- a/nextgen/action-graph/tests/__fixtures__/tasks/cases/moon.yml +++ b/nextgen/action-graph/tests/__fixtures__/tasks/deps/moon.yml @@ -13,3 +13,14 @@ tasks: deps: [b, c, a] options: runDepsInParallel: false + + chain1: + deps: ['chain2'] + chain2: + deps: ['chain3'] + chain3: {} + + base: {} + + internal: + deps: ['~:base'] diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index 2066cb8c29d..b99da4cdadf 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -443,7 +443,7 @@ mod action_graph { let container = ActionGraphContainer::new(sandbox.path()).await; let mut builder = container.create_builder(); - let project = container.project_graph.get("cases").unwrap(); + let project = container.project_graph.get("deps").unwrap(); let task = project.get_task("parallel").unwrap(); builder.run_task(&project, task, None).unwrap(); @@ -459,7 +459,7 @@ mod action_graph { let container = ActionGraphContainer::new(sandbox.path()).await; let mut builder = container.create_builder(); - let project = container.project_graph.get("cases").unwrap(); + let project = container.project_graph.get("deps").unwrap(); let task = project.get_task("serial").unwrap(); builder.run_task(&project, task, None).unwrap(); @@ -468,6 +468,56 @@ mod action_graph { assert_snapshot!(graph.to_dot()); } + + #[tokio::test] + async fn can_create_a_chain() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("deps").unwrap(); + let task = project.get_task("chain1").unwrap(); + + builder.run_task(&project, task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn doesnt_include_dependents() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("deps").unwrap(); + let task = project.get_task("base").unwrap(); + + builder.run_task(&project, task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn includes_dependents() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder.include_dependents = true; + + let project = container.project_graph.get("deps").unwrap(); + let task = project.get_task("base").unwrap(); + + builder.run_task(&project, task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } } mod run_task_by_target { diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap index 86a4829917c..e03fe506210 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap @@ -7,25 +7,25 @@ digraph { 1 [ label="SetupSystemTool" ] 2 [ label="SyncSystemProject(base)" ] 3 [ label="RunTask(base:build)" ] - 4 [ label="SyncSystemProject(common)" ] - 5 [ label="RunTask(common:build)" ] - 6 [ label="SyncSystemProject(client)" ] - 7 [ label="SyncSystemProject(server)" ] - 8 [ label="RunTask(client:build)" ] - 9 [ label="RunTask(server:build)" ] + 4 [ label="SyncSystemProject(server)" ] + 5 [ label="RunTask(server:build)" ] + 6 [ label="SyncSystemProject(common)" ] + 7 [ label="RunTask(common:build)" ] + 8 [ label="SyncSystemProject(client)" ] + 9 [ label="RunTask(client:build)" ] 1 -> 0 [ ] 2 -> 1 [ ] 3 -> 2 [ ] 4 -> 1 [ ] - 4 -> 2 [ ] 5 -> 4 [ ] - 7 -> 1 [ ] 6 -> 1 [ ] - 6 -> 7 [ ] - 6 -> 4 [ ] - 9 -> 7 [ ] + 6 -> 2 [ ] + 7 -> 6 [ ] + 8 -> 1 [ ] + 8 -> 4 [ ] 8 -> 6 [ ] - 8 -> 5 [ ] - 8 -> 9 [ ] + 9 -> 8 [ ] + 9 -> 7 [ ] + 9 -> 5 [ ] } diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__can_create_a_chain.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__can_create_a_chain.snap new file mode 100644 index 00000000000..290953fed5f --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__can_create_a_chain.snap @@ -0,0 +1,20 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(deps)" ] + 3 [ label="RunTask(deps:chain1)" ] + 4 [ label="RunTask(deps:chain2)" ] + 5 [ label="RunTask(deps:chain3)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 5 -> 2 [ ] + 4 -> 2 [ ] + 4 -> 5 [ ] + 3 -> 2 [ ] + 3 -> 4 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__doesnt_include_dependents.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__doesnt_include_dependents.snap new file mode 100644 index 00000000000..7de75775a1d --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__doesnt_include_dependents.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(deps)" ] + 3 [ label="RunTask(deps:base)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__includes_dependents.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__includes_dependents.snap new file mode 100644 index 00000000000..7375595d48b --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__includes_dependents.snap @@ -0,0 +1,23 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(deps)" ] + 3 [ label="RunTask(deps:base)" ] + 4 [ label="RunTask(deps:internal)" ] + 5 [ label="SyncSystemProject(deps-external)" ] + 6 [ label="RunTask(deps-external:external)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] + 4 -> 2 [ ] + 4 -> 3 [ ] + 5 -> 1 [ ] + 5 -> 2 [ ] + 6 -> 5 [ ] + 6 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_parallel.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_parallel.snap index 4e3221ce9b1..f5d5002c747 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_parallel.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_parallel.snap @@ -5,11 +5,11 @@ expression: graph.to_dot() digraph { 0 [ label="SyncWorkspace" ] 1 [ label="SetupSystemTool" ] - 2 [ label="SyncSystemProject(cases)" ] - 3 [ label="RunTask(cases:parallel)" ] - 4 [ label="RunTask(cases:c)" ] - 5 [ label="RunTask(cases:a)" ] - 6 [ label="RunTask(cases:b)" ] + 2 [ label="SyncSystemProject(deps)" ] + 3 [ label="RunTask(deps:parallel)" ] + 4 [ label="RunTask(deps:c)" ] + 5 [ label="RunTask(deps:a)" ] + 6 [ label="RunTask(deps:b)" ] 1 -> 0 [ ] 2 -> 1 [ ] 4 -> 2 [ ] diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_serial.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_serial.snap index 26ec9e0deb6..e444a36cc07 100644 --- a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_serial.snap +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_serial.snap @@ -5,11 +5,11 @@ expression: graph.to_dot() digraph { 0 [ label="SyncWorkspace" ] 1 [ label="SetupSystemTool" ] - 2 [ label="SyncSystemProject(cases)" ] - 3 [ label="RunTask(cases:serial)" ] - 4 [ label="RunTask(cases:b)" ] - 5 [ label="RunTask(cases:c)" ] - 6 [ label="RunTask(cases:a)" ] + 2 [ label="SyncSystemProject(deps)" ] + 3 [ label="RunTask(deps:serial)" ] + 4 [ label="RunTask(deps:b)" ] + 5 [ label="RunTask(deps:c)" ] + 6 [ label="RunTask(deps:a)" ] 1 -> 0 [ ] 2 -> 1 [ ] 4 -> 2 [ ] diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 7fd477bdb97..235fb309fbf 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -36,6 +36,20 @@ We're marking this as a breaking change as this could subtly introduce cycles in the project graph that weren't present before, and for Node.js projects, this may inject `peerDependencies`. +#### 🚀 Updates + +- Rewrote the dependency graph ground the ground-up: + - Now known as the action graph. + - All actions now depend on the `SyncWorkspace` action, instead of this action running + arbitrarily. + - Cleaned up dependency chains between actions, greatly reducing the number of nodes in the graph. + - Renamed `RunTarget` to `RunTask`, including interactive and persistent variants. + +#### 🐞 Fixes + +- Fixed an issue where task dependents (via `moon ci` or `moon run --dependents`) wouldn't always + locate all downstream tasks. + ## 1.14.5 #### 🐞 Fixes From 21c89d060ac8f98c6d19e788b76bfd76117b3231 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Fri, 6 Oct 2023 16:05:11 -0700 Subject: [PATCH 20/22] Polish. --- Cargo.lock | 3 - nextgen/action-graph/Cargo.toml | 5 - .../tests/__fixtures__/tasks/deps/moon.yml | 11 ++- .../action-graph/tests/action_graph_test.rs | 95 +++++-------------- packages/cli/CHANGELOG.md | 12 ++- website/docusaurus.config.js | 4 + 6 files changed, 48 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4789162576b..fb827287c7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3033,17 +3033,14 @@ dependencies = [ "miette", "moon_common", "moon_config", - "moon_node_platform", "moon_platform", "moon_platform_runtime", "moon_project", "moon_project_graph", "moon_query", - "moon_rust_platform", "moon_task", "moon_test_utils2", "petgraph", - "proto_core", "rustc-hash", "starbase_sandbox", "thiserror", diff --git a/nextgen/action-graph/Cargo.toml b/nextgen/action-graph/Cargo.toml index 89535036b6a..72b0af7abc1 100644 --- a/nextgen/action-graph/Cargo.toml +++ b/nextgen/action-graph/Cargo.toml @@ -25,10 +25,5 @@ tracing = { workspace = true } [dev-dependencies] moon_config = { path = "../config" } moon_test_utils2 = { path = "../test-utils" } -proto_core = { workspace = true } starbase_sandbox = { workspace = true } tokio = { workspace = true } - -# TODO remove -moon_node_platform = { path = "../../crates/node/platform" } -moon_rust_platform = { path = "../../crates/rust/platform" } diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/deps/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/deps/moon.yml index 98363d5b0a0..0bead8f5582 100644 --- a/nextgen/action-graph/tests/__fixtures__/tasks/deps/moon.yml +++ b/nextgen/action-graph/tests/__fixtures__/tasks/deps/moon.yml @@ -1,4 +1,6 @@ tasks: + base: {} + a: command: 'a' b: @@ -20,7 +22,10 @@ tasks: deps: ['chain3'] chain3: {} - base: {} - internal: - deps: ['~:base'] + deps: ['base'] + + cycle1: + deps: ['cycle2'] + cycle2: + deps: ['cycle1'] diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index b99da4cdadf..de0b9e52d44 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -5,19 +5,12 @@ mod utils; use moon_action_graph::*; use moon_common::path::WorkspaceRelativePathBuf; use moon_common::Id; -use moon_config::{NodeConfig, RustConfig}; -use moon_node_platform::NodePlatform; -use moon_platform::PlatformManager; use moon_platform_runtime::*; use moon_project_graph::ProjectGraph; -use moon_rust_platform::RustPlatform; use moon_task::{Target, TargetLocator, Task}; use moon_test_utils2::generate_project_graph; -use proto_core::ProtoEnvironment; use rustc_hash::FxHashSet; use starbase_sandbox::{assert_snapshot, create_sandbox}; -use std::path::Path; -use std::sync::Arc; use utils::ActionGraphContainer; fn create_task(id: &str, project: &str) -> Task { @@ -46,38 +39,6 @@ fn create_rust_runtime() -> Runtime { ) } -fn create_platform_manager(root: &Path) -> PlatformManager { - let mut manager = PlatformManager::default(); - let proto = Arc::new(ProtoEnvironment::new_testing(root)); - - manager.register( - PlatformType::Node, - Box::new(NodePlatform::new( - &NodeConfig { - version: Some(UnresolvedVersionSpec::Version(Version::new(20, 0, 0))), - ..Default::default() - }, - &None, - root, - proto.clone(), - )), - ); - - manager.register( - PlatformType::Rust, - Box::new(RustPlatform::new( - &RustConfig { - version: Some(UnresolvedVersionSpec::Version(Version::new(1, 70, 0))), - ..Default::default() - }, - root, - proto.clone(), - )), - ); - - manager -} - fn topo(mut graph: ActionGraph) -> Vec { let mut nodes = vec![]; @@ -93,30 +54,24 @@ fn topo(mut graph: ActionGraph) -> Vec { mod action_graph { use super::*; - // #[test] - // fn errors_on_cycle() { - // let mut graph = ProjectGraphType::new(); - // let a = graph.add_node(create_project("a")); - // let b = graph.add_node(create_project("b")); - // graph.add_edge(a, b, DependencyScope::Build); - // graph.add_edge(b, a, DependencyScope::Build); + #[tokio::test] + #[should_panic(expected = "A dependency cycle has been detected for RunTask(deps:cycle2).")] + async fn errors_on_cycle() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); - // let pg = ProjectGraph::new( - // graph, - // FxHashMap::from_iter([ - // ("a".into(), ProjectNode::new(0)), - // ("b".into(), ProjectNode::new(1)), - // ]), - // &PathBuf::from("."), - // ); + let project = container.project_graph.get("deps").unwrap(); - // let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + builder + .run_task(&project, project.get_task("cycle1").unwrap(), None) + .unwrap(); + builder + .run_task(&project, project.get_task("cycle2").unwrap(), None) + .unwrap(); - // builder.sync_project(&pg.get("a").unwrap()).unwrap(); - // builder.sync_project(&pg.get("b").unwrap()).unwrap(); - - // builder.build().unwrap().reset_iterator().unwrap(); - // } + builder.build().unwrap().reset_iterator().unwrap(); + } mod install_deps { use super::*; @@ -941,14 +896,14 @@ mod action_graph { #[tokio::test] async fn inherits_platform_tool() { - let pg = create_project_graph().await; - let pm = create_platform_manager(&pg.workspace_root); - let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); - let bar = pg.get("bar").unwrap(); + let bar = container.project_graph.get("bar").unwrap(); builder.sync_project(&bar).unwrap(); - let qux = pg.get("qux").unwrap(); + let qux = container.project_graph.get("qux").unwrap(); builder.sync_project(&qux).unwrap(); let graph = builder.build().unwrap(); @@ -978,14 +933,14 @@ mod action_graph { #[tokio::test] async fn supports_platform_override() { - let pg = create_project_graph().await; - let pm = create_platform_manager(&pg.workspace_root); - let mut builder = ActionGraphBuilder::with_platforms(&pm, &pg).unwrap(); + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); - let bar = pg.get("bar").unwrap(); + let bar = container.project_graph.get("bar").unwrap(); builder.sync_project(&bar).unwrap(); - let baz = pg.get("baz").unwrap(); + let baz = container.project_graph.get("baz").unwrap(); builder.sync_project(&baz).unwrap(); let graph = builder.build().unwrap(); diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 235fb309fbf..dd8bfe8f2a2 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -38,18 +38,28 @@ #### 🚀 Updates -- Rewrote the dependency graph ground the ground-up: +- Rewrote the dependency graph from the ground-up: - Now known as the action graph. - All actions now depend on the `SyncWorkspace` action, instead of this action running arbitrarily. - Cleaned up dependency chains between actions, greatly reducing the number of nodes in the graph. - Renamed `RunTarget` to `RunTask`, including interactive and persistent variants. +- Updated the action graph to iterate using a topological queue, which executes ready-to-run actions + in the thread pool. Previously, we would sort topologically _into batches_, which worked, but + resulted in many threads uselessly waiting for an action to run, which was blocked waiting for the + current batch to complete. + - For large graphs, this should result in a significant performance improvement, upwards of 10x. + - Persistent tasks will still be ran as a batch, but since it's the last operation, it's fine. #### 🐞 Fixes - Fixed an issue where task dependents (via `moon ci` or `moon run --dependents`) wouldn't always locate all downstream tasks. +#### ⚙️ Internal + +- Added in-memory caching to project graph file system lookup operations. + ## 1.14.5 #### 🐞 Fixes diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 7eb8d9187b4..aa683167fb3 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -211,6 +211,10 @@ const config = { '@docusaurus/plugin-client-redirects', { redirects: [ + { + from: '/docs/how-it-works/dep-graph', + to: '/docs/how-it-works/action-graph', + }, { from: '/docs/config/global-project', to: '/docs/config/tasks', From 9e3686b9c7cf572c41f7b20e54eb72f4bbc7dd1d Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Fri, 6 Oct 2023 16:11:19 -0700 Subject: [PATCH 21/22] Bump visualizer. --- .yarn/versions/22a0caa2.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.yarn/versions/22a0caa2.yml b/.yarn/versions/22a0caa2.yml index 28fac241db0..ead60d33b1b 100644 --- a/.yarn/versions/22a0caa2.yml +++ b/.yarn/versions/22a0caa2.yml @@ -1,9 +1,10 @@ releases: - "@moonrepo/cli": minor - "@moonrepo/core-linux-arm64-gnu": minor - "@moonrepo/core-linux-arm64-musl": minor - "@moonrepo/core-linux-x64-gnu": minor - "@moonrepo/core-linux-x64-musl": minor - "@moonrepo/core-macos-arm64": minor - "@moonrepo/core-macos-x64": minor - "@moonrepo/core-windows-x64-msvc": minor + '@moonrepo/cli': minor + '@moonrepo/core-linux-arm64-gnu': minor + '@moonrepo/core-linux-arm64-musl': minor + '@moonrepo/core-linux-x64-gnu': minor + '@moonrepo/core-linux-x64-musl': minor + '@moonrepo/core-macos-arm64': minor + '@moonrepo/core-macos-x64': minor + '@moonrepo/core-windows-x64-msvc': minor + '@moonrepo/visualizer': minor From ddc0e98440558833504f7ea99a6e3388bc3af7ca Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Fri, 6 Oct 2023 16:36:34 -0700 Subject: [PATCH 22/22] Fix test. --- crates/core/dep-graph/tests/dep_graph_test.rs | 4 ++-- ..._run_target__moves_persistent_tasks_last.snap | 16 ++++++++-------- packages/cli/CHANGELOG.md | 9 ++++++++- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/crates/core/dep-graph/tests/dep_graph_test.rs b/crates/core/dep-graph/tests/dep_graph_test.rs index 33115869642..309dcaa57ba 100644 --- a/crates/core/dep-graph/tests/dep_graph_test.rs +++ b/crates/core/dep-graph/tests/dep_graph_test.rs @@ -262,11 +262,11 @@ mod run_target { vec![NodeIndex::new(1)], vec![ NodeIndex::new(2), - NodeIndex::new(5), + NodeIndex::new(4), NodeIndex::new(6), NodeIndex::new(7) ], - vec![NodeIndex::new(4), NodeIndex::new(12), NodeIndex::new(13)], + vec![NodeIndex::new(5), NodeIndex::new(12), NodeIndex::new(13)], vec![NodeIndex::new(3), NodeIndex::new(11)], vec![NodeIndex::new(0)], vec![ diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap index f79a3d06563..2bf3c186a08 100644 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap +++ b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap @@ -7,8 +7,8 @@ digraph { 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 2 [ label="InstallNodeDeps(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 3 [ label="SyncNodeProject(persistent)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="SyncNodeProject(buildA)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="SyncNodeProject(buildC)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] + 4 [ label="SyncNodeProject(buildC)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] + 5 [ label="SyncNodeProject(buildA)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 6 [ label="SyncNodeProject(basic)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 7 [ label="SyncNodeProject(noTasks)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 8 [ label="RunPersistentTarget(persistent:dev)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] @@ -21,13 +21,13 @@ digraph { 2 -> 1 [ arrowhead=box, arrowtail=box] 3 -> 1 [ arrowhead=box, arrowtail=box] 4 -> 1 [ arrowhead=box, arrowtail=box] + 3 -> 4 [ arrowhead=box, arrowtail=box] 5 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 5 [ arrowhead=box, arrowtail=box] + 5 -> 4 [ arrowhead=box, arrowtail=box] 6 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 6 [ arrowhead=box, arrowtail=box] + 5 -> 6 [ arrowhead=box, arrowtail=box] 7 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 7 [ arrowhead=box, arrowtail=box] - 3 -> 4 [ arrowhead=box, arrowtail=box] + 5 -> 7 [ arrowhead=box, arrowtail=box] 3 -> 5 [ arrowhead=box, arrowtail=box] 8 -> 2 [ arrowhead=box, arrowtail=box] 8 -> 3 [ arrowhead=box, arrowtail=box] @@ -36,11 +36,11 @@ digraph { 10 -> 2 [ arrowhead=box, arrowtail=box] 10 -> 3 [ arrowhead=box, arrowtail=box] 11 -> 2 [ arrowhead=box, arrowtail=box] - 11 -> 4 [ arrowhead=box, arrowtail=box] + 11 -> 5 [ arrowhead=box, arrowtail=box] 12 -> 2 [ arrowhead=box, arrowtail=box] 12 -> 6 [ arrowhead=box, arrowtail=box] 13 -> 2 [ arrowhead=box, arrowtail=box] - 13 -> 5 [ arrowhead=box, arrowtail=box] + 13 -> 4 [ arrowhead=box, arrowtail=box] 11 -> 12 [ arrowhead=box, arrowtail=box] 11 -> 13 [ arrowhead=box, arrowtail=box] 10 -> 11 [ arrowhead=box, arrowtail=box] diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index dd8bfe8f2a2..6cdc6a341b4 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -36,7 +36,7 @@ We're marking this as a breaking change as this could subtly introduce cycles in the project graph that weren't present before, and for Node.js projects, this may inject `peerDependencies`. -#### 🚀 Updates +#### 🎉 Release - Rewrote the dependency graph from the ground-up: - Now known as the action graph. @@ -50,6 +50,13 @@ current batch to complete. - For large graphs, this should result in a significant performance improvement, upwards of 10x. - Persistent tasks will still be ran as a batch, but since it's the last operation, it's fine. +- Released a new GitHub action, + [`moonrepo/setup-toolchain`](https://github.com/marketplace/actions/setup-proto-and-moon-toolchains), + that replaces both `setup-moon-action` and `setup-proto`. + +#### 🚀 Updates + +- Added a `moon action-graph` command and deprecated `moon dep-graph`. #### 🐞 Fixes