diff --git a/.yarn/versions/22a0caa2.yml b/.yarn/versions/22a0caa2.yml index 28fac241db0..ead60d33b1b 100644 --- a/.yarn/versions/22a0caa2.yml +++ b/.yarn/versions/22a0caa2.yml @@ -1,9 +1,10 @@ releases: - "@moonrepo/cli": minor - "@moonrepo/core-linux-arm64-gnu": minor - "@moonrepo/core-linux-arm64-musl": minor - "@moonrepo/core-linux-x64-gnu": minor - "@moonrepo/core-linux-x64-musl": minor - "@moonrepo/core-macos-arm64": minor - "@moonrepo/core-macos-x64": minor - "@moonrepo/core-windows-x64-msvc": minor + '@moonrepo/cli': minor + '@moonrepo/core-linux-arm64-gnu': minor + '@moonrepo/core-linux-arm64-musl': minor + '@moonrepo/core-linux-x64-gnu': minor + '@moonrepo/core-linux-x64-musl': minor + '@moonrepo/core-macos-arm64': minor + '@moonrepo/core-macos-x64': minor + '@moonrepo/core-windows-x64-msvc': minor + '@moonrepo/visualizer': minor diff --git a/Cargo.lock b/Cargo.lock index e1e0b9db207..fb827287c7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3026,6 +3026,28 @@ dependencies = [ "serde", ] +[[package]] +name = "moon_action_graph" +version = "0.1.0" +dependencies = [ + "miette", + "moon_common", + "moon_config", + "moon_platform", + "moon_platform_runtime", + "moon_project", + "moon_project_graph", + "moon_query", + "moon_task", + "moon_test_utils2", + "petgraph", + "rustc-hash", + "starbase_sandbox", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "moon_action_pipeline" version = "0.1.0" @@ -3733,6 +3755,7 @@ dependencies = [ "moon_query", "moon_task", "moon_task_builder", + "moon_test_utils2", "moon_vcs", "once_map", "petgraph", @@ -3969,6 +3992,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "moon_test_utils2" +version = "0.1.0" +dependencies = [ + "miette", + "moon_config", + "moon_node_platform", + "moon_platform", + "moon_project_graph", + "moon_rust_platform", + "moon_system_platform", + "moon_vcs", + "proto_core", + "starbase_events", + "starbase_sandbox", +] + [[package]] name = "moon_time" version = "0.1.0" diff --git a/crates/core/dep-graph/tests/dep_graph_test.rs b/crates/core/dep-graph/tests/dep_graph_test.rs index 33115869642..309dcaa57ba 100644 --- a/crates/core/dep-graph/tests/dep_graph_test.rs +++ b/crates/core/dep-graph/tests/dep_graph_test.rs @@ -262,11 +262,11 @@ mod run_target { vec![NodeIndex::new(1)], vec![ NodeIndex::new(2), - NodeIndex::new(5), + NodeIndex::new(4), NodeIndex::new(6), NodeIndex::new(7) ], - vec![NodeIndex::new(4), NodeIndex::new(12), NodeIndex::new(13)], + vec![NodeIndex::new(5), NodeIndex::new(12), NodeIndex::new(13)], vec![NodeIndex::new(3), NodeIndex::new(11)], vec![NodeIndex::new(0)], vec![ diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap index f79a3d06563..2bf3c186a08 100644 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap +++ b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap @@ -7,8 +7,8 @@ digraph { 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 2 [ label="InstallNodeDeps(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 3 [ label="SyncNodeProject(persistent)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="SyncNodeProject(buildA)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="SyncNodeProject(buildC)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] + 4 [ label="SyncNodeProject(buildC)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] + 5 [ label="SyncNodeProject(buildA)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 6 [ label="SyncNodeProject(basic)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 7 [ label="SyncNodeProject(noTasks)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 8 [ label="RunPersistentTarget(persistent:dev)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] @@ -21,13 +21,13 @@ digraph { 2 -> 1 [ arrowhead=box, arrowtail=box] 3 -> 1 [ arrowhead=box, arrowtail=box] 4 -> 1 [ arrowhead=box, arrowtail=box] + 3 -> 4 [ arrowhead=box, arrowtail=box] 5 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 5 [ arrowhead=box, arrowtail=box] + 5 -> 4 [ arrowhead=box, arrowtail=box] 6 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 6 [ arrowhead=box, arrowtail=box] + 5 -> 6 [ arrowhead=box, arrowtail=box] 7 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 7 [ arrowhead=box, arrowtail=box] - 3 -> 4 [ arrowhead=box, arrowtail=box] + 5 -> 7 [ arrowhead=box, arrowtail=box] 3 -> 5 [ arrowhead=box, arrowtail=box] 8 -> 2 [ arrowhead=box, arrowtail=box] 8 -> 3 [ arrowhead=box, arrowtail=box] @@ -36,11 +36,11 @@ digraph { 10 -> 2 [ arrowhead=box, arrowtail=box] 10 -> 3 [ arrowhead=box, arrowtail=box] 11 -> 2 [ arrowhead=box, arrowtail=box] - 11 -> 4 [ arrowhead=box, arrowtail=box] + 11 -> 5 [ arrowhead=box, arrowtail=box] 12 -> 2 [ arrowhead=box, arrowtail=box] 12 -> 6 [ arrowhead=box, arrowtail=box] 13 -> 2 [ arrowhead=box, arrowtail=box] - 13 -> 5 [ arrowhead=box, arrowtail=box] + 13 -> 4 [ arrowhead=box, arrowtail=box] 11 -> 12 [ arrowhead=box, arrowtail=box] 11 -> 13 [ arrowhead=box, arrowtail=box] 10 -> 11 [ arrowhead=box, arrowtail=box] diff --git a/nextgen/action-graph/Cargo.toml b/nextgen/action-graph/Cargo.toml new file mode 100644 index 00000000000..72b0af7abc1 --- /dev/null +++ b/nextgen/action-graph/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "moon_action_graph" +version = "0.1.0" +edition = "2021" +license = "MIT" +description = "Dependency graph for actions (tasks)." +homepage = "https://moonrepo.dev/moon" +repository = "https://github.com/moonrepo/moon" + +[dependencies] +moon_common = { path = "../common" } +# TODO remove +moon_platform = { path = "../../crates/core/platform" } +moon_platform_runtime = { path = "../platform-runtime" } +moon_project = { path = "../project" } +moon_project_graph = { path = "../project-graph" } +moon_task = { path = "../task" } +moon_query = { path = "../query" } +miette = { workspace = true } +petgraph = { workspace = true } +rustc-hash = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +moon_config = { path = "../config" } +moon_test_utils2 = { path = "../test-utils" } +starbase_sandbox = { workspace = true } +tokio = { workspace = true } diff --git a/nextgen/action-graph/src/action_graph.rs b/nextgen/action-graph/src/action_graph.rs new file mode 100644 index 00000000000..7cb832ab873 --- /dev/null +++ b/nextgen/action-graph/src/action_graph.rs @@ -0,0 +1,151 @@ +use crate::action_graph_error::ActionGraphError; +use crate::action_node::ActionNode; +use moon_common::is_test_env; +use petgraph::dot::{Config, Dot}; +use petgraph::prelude::*; +use petgraph::visit::{IntoEdgeReferences, IntoNodeReferences}; +use rustc_hash::{FxHashMap, FxHashSet}; +use std::collections::VecDeque; + +pub type GraphType = DiGraph; +pub type IndicesMap = FxHashMap; + +pub struct ActionGraph { + graph: GraphType, + indices: IndicesMap, + + // States when iterating + queue: VecDeque, + visited: FxHashSet, +} + +impl ActionGraph { + pub fn new(graph: GraphType, indices: IndicesMap) -> Self { + ActionGraph { + graph, + indices, + queue: VecDeque::default(), + visited: FxHashSet::default(), + } + } + + pub fn reset_iterator(&mut self) -> miette::Result<()> { + self.detect_cycle()?; + + self.queue.clear(); + self.visited.clear(); + + // Extract root/initial nodes (those without edges) + self.queue.extend(self.graph.node_indices().filter(|&idx| { + self.graph + .neighbors_directed(idx, Outgoing) + .next() + .is_none() + })); + + Ok(()) + } + + pub fn is_empty(&self) -> bool { + self.get_node_count() == 0 + } + + pub fn get_index_from_node(&self, node: &ActionNode) -> Option<&NodeIndex> { + self.indices.get(node) + } + + pub fn get_node_count(&self) -> usize { + self.graph.node_count() + } + + pub fn get_node_from_index(&self, index: &NodeIndex) -> Option<&ActionNode> { + self.graph.node_weight(*index) + } + + pub fn to_dot(&self) -> String { + type DotGraph = DiGraph; + + let is_test = is_test_env() || cfg!(debug_assertions); + let graph = self.graph.map(|_, n| n.label(), |_, _| ()); + + let edge = |_: &DotGraph, e: <&DotGraph as IntoEdgeReferences>::EdgeRef| { + if is_test { + String::new() + } else if e.source().index() == 0 { + String::from("arrowhead=none") + } else { + String::from("arrowhead=box, arrowtail=box") + } + }; + + let node = |_: &DotGraph, n: <&DotGraph as IntoNodeReferences>::NodeRef| { + if is_test { + format!("label=\"{}\" ", n.1) + } else { + format!( + "label=\"{}\" style=filled, shape=oval, fillcolor=gray, fontcolor=black ", + n.1 + ) + } + }; + + let dot = Dot::with_attr_getters( + &graph, + &[Config::EdgeNoLabel, Config::NodeNoLabel], + &edge, + &node, + ); + + format!("{dot:?}") + } + + pub fn to_labeled_graph(&self) -> DiGraph { + self.graph.map(|_, n| n.label(), |_, _| String::new()) + } + + fn detect_cycle(&self) -> miette::Result<()> { + if self.get_node_count() > 1 { + if let Err(cycle) = petgraph::algo::toposort(&self.graph, None) { + return Err(ActionGraphError::CycleDetected( + self.get_node_from_index(&cycle.node_id()) + .map(|n| n.label()) + .unwrap_or_else(|| "(unknown)".into()), + ) + .into()); + } + } + + Ok(()) + } +} + +// This is based on the `Topo` struct from petgraph! +impl Iterator for ActionGraph { + type Item = ActionNode; + + fn next(&mut self) -> Option { + while let Some(idx) = self.queue.pop_front() { + if self.visited.contains(&idx) { + continue; + } + + self.visited.insert(idx); + + for neighbor in self.graph.neighbors_directed(idx, Direction::Incoming) { + // Look at each neighbor, and those that only have incoming edges + // from the already ordered list, they are the next to visit. + if self + .graph + .neighbors_directed(neighbor, Direction::Outgoing) + .all(|b| self.visited.contains(&b)) + { + self.queue.push_back(neighbor); + } + } + + return self.graph.node_weight(idx).map(|n| n.to_owned()); + } + + None + } +} diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs new file mode 100644 index 00000000000..ee51775871d --- /dev/null +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -0,0 +1,419 @@ +use crate::action_graph::ActionGraph; +use crate::action_node::ActionNode; +use moon_common::{color, path::WorkspaceRelativePathBuf}; +use moon_platform::{PlatformManager, Runtime}; +use moon_project::Project; +use moon_project_graph::ProjectGraph; +use moon_query::{build_query, Criteria}; +use moon_task::{Target, TargetError, TargetLocator, TargetScope, Task}; +use petgraph::prelude::*; +use rustc_hash::{FxHashMap, FxHashSet}; +use tracing::{debug, trace}; + +type TouchedFilePaths = FxHashSet; + +pub struct ActionGraphBuilder<'app> { + pub include_dependents: bool, + + all_query: Option, + graph: DiGraph, + indices: FxHashMap, + platform_manager: &'app PlatformManager, + project_graph: &'app ProjectGraph, +} + +impl<'app> ActionGraphBuilder<'app> { + pub fn new(project_graph: &'app ProjectGraph) -> miette::Result { + ActionGraphBuilder::with_platforms(PlatformManager::read(), project_graph) + } + + pub fn with_platforms( + platform_manager: &'app PlatformManager, + project_graph: &'app ProjectGraph, + ) -> miette::Result { + Ok(ActionGraphBuilder { + all_query: None, + graph: DiGraph::new(), + include_dependents: false, + indices: FxHashMap::default(), + platform_manager, + project_graph, + }) + } + + pub fn build(self) -> miette::Result { + Ok(ActionGraph::new(self.graph, self.indices)) + } + + pub fn get_index_from_node(&self, node: &ActionNode) -> Option<&NodeIndex> { + self.indices.get(node) + } + + pub fn get_runtime( + &self, + project: &Project, + task: Option<&Task>, + allow_override: bool, + ) -> Runtime { + if let Some(platform) = self.platform_manager.find(|p| match task { + Some(task) => p.matches(&task.platform, None), + None => p.matches(&project.platform, None), + }) { + return platform.get_runtime_from_config(if allow_override { + Some(&project.config) + } else { + None + }); + } + + Runtime::system() + } + + pub fn set_query(&mut self, input: &str) -> miette::Result<()> { + self.all_query = Some(build_query(input)?); + + Ok(()) + } + + // ACTIONS + + pub fn install_deps( + &mut self, + project: &Project, + task: Option<&Task>, + ) -> miette::Result> { + let mut in_project = false; + + // If project is NOT in the package manager workspace, then we should + // install dependencies in the project, not the workspace root. + if let Ok(platform) = self.platform_manager.get(project.language.clone()) { + if !platform.is_project_in_dependency_workspace(project.source.as_str())? { + in_project = true; + + debug!( + "Project {} is not within the dependency manager workspace, dependencies will be installed within the project instead of the root", + color::id(&project.id), + ); + } + } + + let node = if in_project { + ActionNode::InstallProjectDeps { + project: project.id.to_owned(), + runtime: self.get_runtime(project, task, true), + } + } else { + ActionNode::InstallDeps { + runtime: self.get_runtime(project, task, false), + } + }; + + if node.get_runtime().platform.is_system() { + return Ok(None); + } + + if let Some(index) = self.get_index_from_node(&node) { + return Ok(Some(*index)); + } + + // Before we install deps, we must ensure the language has been installed + let setup_tool_index = self.setup_tool(node.get_runtime()); + let index = self.insert_node(node); + + self.link_requirements(index, vec![setup_tool_index]); + + Ok(Some(index)) + } + + pub fn run_task( + &mut self, + project: &Project, + task: &Task, + touched_files: Option<&TouchedFilePaths>, + ) -> miette::Result> { + let node = ActionNode::RunTask { + interactive: task.is_interactive(), + persistent: task.is_persistent(), + runtime: self.get_runtime(project, Some(task), true), + target: task.target.to_owned(), + }; + + if let Some(index) = self.get_index_from_node(&node) { + return Ok(Some(*index)); + } + + // Compare against touched files if provided + if let Some(touched) = touched_files { + if !task.is_affected(touched)? { + trace!( + "Task {} not affected based on touched files, skipping", + color::label(&task.target), + ); + + return Ok(None); + } + } + + // We should install deps & sync projects *before* running targets + let mut reqs = vec![]; + + if let Some(install_deps_index) = self.install_deps(project, Some(task))? { + reqs.push(install_deps_index); + } + + reqs.push(self.sync_project(project)?); + + let index = self.insert_node(node); + + // And we also need to create edges for task dependencies + if !task.deps.is_empty() { + trace!( + deps = ?task.deps.iter().map(|d| d.as_str()).collect::>(), + "Adding dependencies for task {}", + color::label(&task.target), + ); + reqs.extend(self.run_task_dependencies(task)?); + } + + self.link_requirements(index, reqs); + + // And possibly dependents + if self.include_dependents { + self.run_task_dependents(task)?; + } + + Ok(Some(index)) + } + + // We don't pass touched files to dependencies, because if the parent + // task is affected/going to run, then so should all of these! + pub fn run_task_dependencies(&mut self, task: &Task) -> miette::Result> { + let parallel = task.options.run_deps_in_parallel; + let mut indices = vec![]; + let mut previous_target_index = None; + + for dep_target in &task.deps { + let (_, dep_indices) = self.run_task_by_target(dep_target, None)?; + + for dep_index in dep_indices { + // When parallel, parent depends on child + if parallel { + indices.push(dep_index); + + // When serial, next child depends on previous child + } else if let Some(prev) = previous_target_index { + self.link_requirements(dep_index, vec![prev]); + } + + previous_target_index = Some(dep_index); + } + } + + if !parallel { + indices.push(previous_target_index.unwrap()); + } + + Ok(indices) + } + + // This is costly, is there a better way to do this? + pub fn run_task_dependents(&mut self, task: &Task) -> miette::Result> { + let mut indices = vec![]; + + if let TargetScope::Project(project_locator) = &task.target.scope { + let project = self.project_graph.get(project_locator)?; + + // From self project + for dep_task in project.tasks.values() { + if dep_task.deps.contains(&task.target) { + if let Some(index) = self.run_task(&project, dep_task, None)? { + indices.push(index); + } + } + } + + // From other projects + for dependent_id in self.project_graph.dependents_of(&project)? { + let dep_project = self.project_graph.get(dependent_id)?; + + for dep_task in dep_project.tasks.values() { + if dep_task.deps.contains(&task.target) { + if let Some(index) = self.run_task(&dep_project, dep_task, None)? { + indices.push(index); + } + } + } + } + } + + Ok(indices) + } + + pub fn run_task_by_target>( + &mut self, + target: T, + touched_files: Option<&TouchedFilePaths>, + ) -> miette::Result<(FxHashSet, FxHashSet)> { + let target = target.as_ref(); + let mut inserted_targets = FxHashSet::default(); + let mut inserted_indices = FxHashSet::default(); + + match &target.scope { + // :task + TargetScope::All => { + let mut projects = vec![]; + + if let Some(all_query) = &self.all_query { + projects.extend(self.project_graph.query(all_query)?); + } else { + projects.extend(self.project_graph.get_all()?); + }; + + for project in projects { + // Don't error if the task does not exist + if let Ok(task) = project.get_task(&target.task_id) { + if let Some(index) = self.run_task(&project, task, touched_files)? { + inserted_targets.insert(task.target.clone()); + inserted_indices.insert(index); + } + } + } + } + // ^:task + TargetScope::Deps => { + return Err(TargetError::NoDepsInRunContext.into()); + } + // project:task + TargetScope::Project(project_locator) => { + let project = self.project_graph.get(project_locator)?; + let task = project.get_task(&target.task_id)?; + + if let Some(index) = self.run_task(&project, task, touched_files)? { + inserted_targets.insert(task.target.to_owned()); + inserted_indices.insert(index); + } + } + // #tag:task + TargetScope::Tag(tag) => { + let projects = self + .project_graph + .query(build_query(format!("tag={}", tag))?)?; + + for project in projects { + // Don't error if the task does not exist + if let Ok(task) = project.get_task(&target.task_id) { + if let Some(index) = self.run_task(&project, task, touched_files)? { + inserted_targets.insert(task.target.clone()); + inserted_indices.insert(index); + } + } + } + } + // ~:task + TargetScope::OwnSelf => { + return Err(TargetError::NoSelfInRunContext.into()); + } + }; + + Ok((inserted_targets, inserted_indices)) + } + + pub fn run_task_by_target_locator>( + &mut self, + target_locator: T, + touched_files: Option<&TouchedFilePaths>, + ) -> miette::Result<(FxHashSet, FxHashSet)> { + match target_locator.as_ref() { + TargetLocator::Qualified(target) => self.run_task_by_target(target, touched_files), + TargetLocator::TaskFromWorkingDir(task_id) => self.run_task_by_target( + Target::new(&self.project_graph.get_from_path(None)?.id, task_id)?, + touched_files, + ), + } + } + + pub fn setup_tool(&mut self, runtime: &Runtime) -> NodeIndex { + let node = ActionNode::SetupTool { + runtime: runtime.to_owned(), + }; + + if let Some(index) = self.get_index_from_node(&node) { + return *index; + } + + let sync_workspace_index = self.sync_workspace(); + let index = self.insert_node(node); + + self.link_requirements(index, vec![sync_workspace_index]); + + index + } + + pub fn sync_project(&mut self, project: &Project) -> miette::Result { + let node = ActionNode::SyncProject { + project: project.id.clone(), + runtime: self.get_runtime(project, None, true), + }; + + if let Some(index) = self.get_index_from_node(&node) { + return Ok(*index); + } + + // Syncing requires the language's tool to be installed + let setup_tool_index = self.setup_tool(node.get_runtime()); + let index = self.insert_node(node); + let mut reqs = vec![setup_tool_index]; + + // And we should also depend on other projects + for dep_project_id in self.project_graph.dependencies_of(project)? { + let dep_project = self.project_graph.get(dep_project_id)?; + let dep_project_index = self.sync_project(&dep_project)?; + + if index != dep_project_index { + reqs.push(dep_project_index); + } + } + + self.link_requirements(index, reqs); + + Ok(index) + } + + pub fn sync_workspace(&mut self) -> NodeIndex { + let node = ActionNode::SyncWorkspace; + + if let Some(index) = self.get_index_from_node(&node) { + return *index; + } + + self.insert_node(node) + } + + // PRIVATE + + fn link_requirements(&mut self, index: NodeIndex, reqs: Vec) { + trace!( + index = index.index(), + requires = ?reqs, + "Linking requirements for index" + ); + + for req in reqs { + self.graph.add_edge(index, req, ()); + } + } + + fn insert_node(&mut self, node: ActionNode) -> NodeIndex { + let index = self.graph.add_node(node.clone()); + + debug!( + index = index.index(), + "Adding {} to graph", + color::muted_light(node.label()) + ); + + self.indices.insert(node, index); + + index + } +} diff --git a/nextgen/action-graph/src/action_graph_error.rs b/nextgen/action-graph/src/action_graph_error.rs new file mode 100644 index 00000000000..59e37ddd11e --- /dev/null +++ b/nextgen/action-graph/src/action_graph_error.rs @@ -0,0 +1,9 @@ +use miette::Diagnostic; +use moon_common::{Style, Stylize}; +use thiserror::Error; + +#[derive(Error, Debug, Diagnostic)] +pub enum ActionGraphError { + #[error("A dependency cycle has been detected for {}.", .0.style(Style::Label))] + CycleDetected(String), +} diff --git a/nextgen/action-graph/src/action_node.rs b/nextgen/action-graph/src/action_node.rs new file mode 100644 index 00000000000..a691c58de60 --- /dev/null +++ b/nextgen/action-graph/src/action_node.rs @@ -0,0 +1,106 @@ +use moon_common::Id; +use moon_platform_runtime::Runtime; +use moon_task::Target; +use std::hash::{Hash, Hasher}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ActionNode { + /// Install tool dependencies in the workspace root. + InstallDeps { runtime: Runtime }, + + /// Install tool dependencies in the project root. + InstallProjectDeps { project: Id, runtime: Runtime }, + + /// Run a project's task. + RunTask { + interactive: bool, // Interactively with stdin + persistent: bool, // Never terminates + runtime: Runtime, + target: Target, + }, + + /// Setup a tool + version for the provided platform. + SetupTool { runtime: Runtime }, + + /// Sync a project with language specific semantics. + SyncProject { project: Id, runtime: Runtime }, + + /// Sync the entire moon workspace. + /// Install system dependencies. + SyncWorkspace, +} + +impl ActionNode { + pub fn get_runtime(&self) -> &Runtime { + match self { + Self::InstallDeps { runtime } => runtime, + Self::InstallProjectDeps { runtime, .. } => runtime, + Self::RunTask { runtime, .. } => runtime, + Self::SetupTool { runtime } => runtime, + Self::SyncProject { runtime, .. } => runtime, + Self::SyncWorkspace => unreachable!(), + } + } + + pub fn is_interactive(&self) -> bool { + match self { + Self::RunTask { interactive, .. } => *interactive, + _ => false, + } + } + + pub fn is_persistent(&self) -> bool { + match self { + Self::RunTask { persistent, .. } => *persistent, + _ => false, + } + } + + pub fn label(&self) -> String { + match self { + Self::InstallDeps { runtime } => { + format!("Install{runtime}Deps({})", runtime.requirement) + } + Self::InstallProjectDeps { runtime, project } => { + format!( + "Install{runtime}DepsInProject({}, {project})", + runtime.requirement + ) + } + Self::RunTask { + interactive, + persistent, + target, + .. + } => { + format!( + "Run{}Task({target})", + if *persistent { + "Persistent" + } else if *interactive { + "Interactive" + } else { + "" + } + ) + } + Self::SetupTool { runtime } => { + if runtime.platform.is_system() { + "SetupSystemTool".into() + } else { + format!("Setup{runtime}Tool({})", runtime.requirement) + } + } + Self::SyncProject { runtime, project } => { + format!("Sync{runtime}Project({project})") + } + Self::SyncWorkspace => "SyncWorkspace".into(), + } + } +} + +impl Hash for ActionNode { + fn hash(&self, state: &mut H) { + self.label().hash(state); + } +} diff --git a/nextgen/action-graph/src/lib.rs b/nextgen/action-graph/src/lib.rs new file mode 100644 index 00000000000..913205c5436 --- /dev/null +++ b/nextgen/action-graph/src/lib.rs @@ -0,0 +1,9 @@ +mod action_graph; +mod action_graph_builder; +mod action_graph_error; +mod action_node; + +pub use action_graph::*; +pub use action_graph_builder::*; +pub use action_graph_error::*; +pub use action_node::*; diff --git a/nextgen/action-graph/tests/__fixtures__/dep-workspace/.moon/toolchain.yml b/nextgen/action-graph/tests/__fixtures__/dep-workspace/.moon/toolchain.yml new file mode 100644 index 00000000000..88c6fbe7e3e --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/dep-workspace/.moon/toolchain.yml @@ -0,0 +1,2 @@ +node: + version: '20.0.0' diff --git a/nextgen/action-graph/tests/__fixtures__/dep-workspace/in/moon.yml b/nextgen/action-graph/tests/__fixtures__/dep-workspace/in/moon.yml new file mode 100644 index 00000000000..f1de67fbd7f --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/dep-workspace/in/moon.yml @@ -0,0 +1 @@ +language: javascript diff --git a/nextgen/action-graph/tests/__fixtures__/dep-workspace/out/moon.yml b/nextgen/action-graph/tests/__fixtures__/dep-workspace/out/moon.yml new file mode 100644 index 00000000000..30d34adef1c --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/dep-workspace/out/moon.yml @@ -0,0 +1 @@ +language: typescript diff --git a/nextgen/action-graph/tests/__fixtures__/dep-workspace/package.json b/nextgen/action-graph/tests/__fixtures__/dep-workspace/package.json new file mode 100644 index 00000000000..ab0250ab062 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/dep-workspace/package.json @@ -0,0 +1,3 @@ +{ + "workspaces": ["in"] +} diff --git a/nextgen/action-graph/tests/__fixtures__/projects/.moon/toolchain.yml b/nextgen/action-graph/tests/__fixtures__/projects/.moon/toolchain.yml new file mode 100644 index 00000000000..edf14aa87de --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/.moon/toolchain.yml @@ -0,0 +1,6 @@ +node: + version: '20.0.0' + packageManager: 'npm' + +rust: + version: '1.70.0' diff --git a/nextgen/action-graph/tests/__fixtures__/projects/bar/moon.yml b/nextgen/action-graph/tests/__fixtures__/projects/bar/moon.yml new file mode 100644 index 00000000000..f1de67fbd7f --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/bar/moon.yml @@ -0,0 +1 @@ +language: javascript diff --git a/nextgen/action-graph/tests/__fixtures__/projects/baz/moon.yml b/nextgen/action-graph/tests/__fixtures__/projects/baz/moon.yml new file mode 100644 index 00000000000..0a620fec5d0 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/baz/moon.yml @@ -0,0 +1,5 @@ +language: typescript + +toolchain: + node: + version: '18.0.0' diff --git a/nextgen/action-graph/tests/__fixtures__/projects/foo/moon.yml b/nextgen/action-graph/tests/__fixtures__/projects/foo/moon.yml new file mode 100644 index 00000000000..38118b204e1 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/foo/moon.yml @@ -0,0 +1 @@ +dependsOn: [bar] diff --git a/nextgen/action-graph/tests/__fixtures__/projects/package.json b/nextgen/action-graph/tests/__fixtures__/projects/package.json new file mode 100644 index 00000000000..b203b4c6c45 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/package.json @@ -0,0 +1,3 @@ +{ + "workspaces": ["*"] +} diff --git a/nextgen/action-graph/tests/__fixtures__/projects/qux/moon.yml b/nextgen/action-graph/tests/__fixtures__/projects/qux/moon.yml new file mode 100644 index 00000000000..22761ba7ee1 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/projects/qux/moon.yml @@ -0,0 +1 @@ +language: rust diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/base/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/base/moon.yml new file mode 100644 index 00000000000..606e4cef104 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/base/moon.yml @@ -0,0 +1,3 @@ +tasks: + build: + command: 'build' diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/client/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/client/moon.yml new file mode 100644 index 00000000000..59d5f8f9696 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/client/moon.yml @@ -0,0 +1,14 @@ +language: 'javascript' + +dependsOn: ['common', 'server'] + +tags: ['frontend'] + +tasks: + build: + command: 'build' + deps: ['^:build'] + lint: + command: 'lint' + test: + command: 'test' diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/common/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/common/moon.yml new file mode 100644 index 00000000000..6a36e76ede4 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/common/moon.yml @@ -0,0 +1,9 @@ +dependsOn: ['base'] + +tags: ['frontend'] + +tasks: + build: + command: 'build' + lint: + command: 'lint' diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/deps-external/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/deps-external/moon.yml new file mode 100644 index 00000000000..bd188ab4774 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/deps-external/moon.yml @@ -0,0 +1,3 @@ +tasks: + external: + deps: ['deps:base'] diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/deps/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/deps/moon.yml new file mode 100644 index 00000000000..0bead8f5582 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/deps/moon.yml @@ -0,0 +1,31 @@ +tasks: + base: {} + + a: + command: 'a' + b: + command: 'b' + c: + command: 'd' + + parallel: + deps: [c, a, b] + + serial: + deps: [b, c, a] + options: + runDepsInParallel: false + + chain1: + deps: ['chain2'] + chain2: + deps: ['chain3'] + chain3: {} + + internal: + deps: ['base'] + + cycle1: + deps: ['cycle2'] + cycle2: + deps: ['cycle1'] diff --git a/nextgen/action-graph/tests/__fixtures__/tasks/server/moon.yml b/nextgen/action-graph/tests/__fixtures__/tasks/server/moon.yml new file mode 100644 index 00000000000..1cdb6dfa0c2 --- /dev/null +++ b/nextgen/action-graph/tests/__fixtures__/tasks/server/moon.yml @@ -0,0 +1,10 @@ +language: 'rust' + +tasks: + build: + command: 'build' + deps: ['^:build'] + lint: + command: 'lint' + test: + command: 'test' diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs new file mode 100644 index 00000000000..de0b9e52d44 --- /dev/null +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -0,0 +1,1008 @@ +#![allow(clippy::disallowed_names)] + +mod utils; + +use moon_action_graph::*; +use moon_common::path::WorkspaceRelativePathBuf; +use moon_common::Id; +use moon_platform_runtime::*; +use moon_project_graph::ProjectGraph; +use moon_task::{Target, TargetLocator, Task}; +use moon_test_utils2::generate_project_graph; +use rustc_hash::FxHashSet; +use starbase_sandbox::{assert_snapshot, create_sandbox}; +use utils::ActionGraphContainer; + +fn create_task(id: &str, project: &str) -> Task { + Task { + id: Id::raw(id), + target: Target::new(project, id).unwrap(), + ..Task::default() + } +} + +async fn create_project_graph() -> ProjectGraph { + generate_project_graph("projects").await +} + +fn create_node_runtime() -> Runtime { + Runtime::new( + PlatformType::Node, + RuntimeReq::with_version(Version::new(20, 0, 0)), + ) +} + +fn create_rust_runtime() -> Runtime { + Runtime::new( + PlatformType::Rust, + RuntimeReq::with_version(Version::new(1, 70, 0)), + ) +} + +fn topo(mut graph: ActionGraph) -> Vec { + let mut nodes = vec![]; + + graph.reset_iterator().unwrap(); + + for node in graph { + nodes.push(node); + } + + nodes +} + +mod action_graph { + use super::*; + + #[tokio::test] + #[should_panic(expected = "A dependency cycle has been detected for RunTask(deps:cycle2).")] + async fn errors_on_cycle() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("deps").unwrap(); + + builder + .run_task(&project, project.get_task("cycle1").unwrap(), None) + .unwrap(); + builder + .run_task(&project, project.get_task("cycle2").unwrap(), None) + .unwrap(); + + builder.build().unwrap().reset_iterator().unwrap(); + } + + mod install_deps { + use super::*; + + #[tokio::test] + async fn graphs() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let bar = container.project_graph.get("bar").unwrap(); + builder.install_deps(&bar, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::InstallDeps { + runtime: create_node_runtime() + } + ] + ); + } + + #[tokio::test] + async fn ignores_dupes() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let bar = container.project_graph.get("bar").unwrap(); + builder.install_deps(&bar, None).unwrap(); + builder.install_deps(&bar, None).unwrap(); + builder.install_deps(&bar, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::InstallDeps { + runtime: create_node_runtime() + } + ] + ); + } + + #[tokio::test] + async fn installs_in_project_when_not_in_depman_workspace() { + let sandbox = create_sandbox("dep-workspace"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let inside = container.project_graph.get("in").unwrap(); + builder.install_deps(&inside, None).unwrap(); + + let outside = container.project_graph.get("out").unwrap(); + builder.install_deps(&outside, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::InstallProjectDeps { + project: Id::raw("out"), + runtime: create_node_runtime() + }, + ActionNode::InstallDeps { + runtime: create_node_runtime() + }, + ] + ); + } + } + + mod run_task { + use super::*; + use starbase_sandbox::pretty_assertions::assert_eq; + + #[tokio::test] + async fn graphs() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.platform = PlatformType::Node; + + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: create_node_runtime() + }, + ActionNode::InstallDeps { + runtime: create_node_runtime() + }, + ActionNode::RunTask { + interactive: false, + persistent: false, + runtime: create_node_runtime(), + target: task.target + } + ] + ); + } + + #[tokio::test] + async fn ignores_dupes() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.platform = PlatformType::Node; + + builder.run_task(&project, &task, None).unwrap(); + builder.run_task(&project, &task, None).unwrap(); + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: create_node_runtime() + }, + ActionNode::InstallDeps { + runtime: create_node_runtime() + }, + ActionNode::RunTask { + interactive: false, + persistent: false, + runtime: create_node_runtime(), + target: task.target + } + ] + ); + } + + #[tokio::test] + async fn doesnt_graph_if_not_affected_by_touched_files() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.platform = PlatformType::Node; + + builder + // Empty set works fine, just needs to be some + .run_task(&project, &task, Some(&FxHashSet::default())) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert!(topo(graph).is_empty()); + } + + #[tokio::test] + async fn graphs_if_affected_by_touched_files() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let file = WorkspaceRelativePathBuf::from("bar/file.js"); + + let project = container.project_graph.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.platform = PlatformType::Node; + task.input_files.insert(file.clone()); + + builder + .run_task(&project, &task, Some(&FxHashSet::from_iter([file]))) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert!(!topo(graph).is_empty()); + } + + #[tokio::test] + async fn task_can_have_a_diff_platform_from_project() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + // node + let project = container.project_graph.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.platform = PlatformType::Rust; + + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::SetupTool { + runtime: create_rust_runtime() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: create_node_runtime() + }, + ActionNode::InstallDeps { + runtime: create_rust_runtime() + }, + ActionNode::RunTask { + interactive: false, + persistent: false, + runtime: create_rust_runtime(), + target: task.target + } + ] + ); + } + + #[tokio::test] + async fn sets_interactive() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.options.interactive = true; + + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph).last().unwrap(), + &ActionNode::RunTask { + interactive: true, + persistent: false, + runtime: Runtime::system(), + target: task.target + } + ); + } + + #[tokio::test] + async fn sets_persistent() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("bar").unwrap(); + + let mut task = create_task("build", "bar"); + task.options.persistent = true; + + builder.run_task(&project, &task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph).last().unwrap(), + &ActionNode::RunTask { + interactive: false, + persistent: true, + runtime: Runtime::system(), + target: task.target + } + ); + } + } + + mod run_task_dependencies { + use super::*; + + #[tokio::test] + async fn runs_deps_in_parallel() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("deps").unwrap(); + let task = project.get_task("parallel").unwrap(); + + builder.run_task(&project, task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn runs_deps_in_serial() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("deps").unwrap(); + let task = project.get_task("serial").unwrap(); + + builder.run_task(&project, task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn can_create_a_chain() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("deps").unwrap(); + let task = project.get_task("chain1").unwrap(); + + builder.run_task(&project, task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn doesnt_include_dependents() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let project = container.project_graph.get("deps").unwrap(); + let task = project.get_task("base").unwrap(); + + builder.run_task(&project, task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn includes_dependents() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder.include_dependents = true; + + let project = container.project_graph.get("deps").unwrap(); + let task = project.get_task("base").unwrap(); + + builder.run_task(&project, task, None).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + } + + mod run_task_by_target { + use super::*; + + #[tokio::test] + #[should_panic(expected = "Dependencies scope (^:) is not supported in run contexts.")] + async fn errors_on_parent_scope() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("^:build").unwrap(), None) + .unwrap(); + } + + #[tokio::test] + #[should_panic(expected = "Self scope (~:) is not supported in run contexts.")] + async fn errors_on_self_scope() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("~:build").unwrap(), None) + .unwrap(); + } + + #[tokio::test] + async fn runs_all() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse(":build").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn runs_all_with_query() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder.set_query("language=rust").unwrap(); + + builder + .run_task_by_target(Target::parse(":build").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn runs_all_no_nodes() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse(":unknown").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert!(graph.is_empty()); + } + + #[tokio::test] + async fn runs_by_project() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("client:lint").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + #[should_panic(expected = "No project has been configured with the name or alias unknown.")] + async fn errors_for_unknown_project() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("unknown:build").unwrap(), None) + .unwrap(); + } + + #[tokio::test] + #[should_panic(expected = "Unknown task unknown for project server.")] + async fn errors_for_unknown_project_task() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("server:unknown").unwrap(), None) + .unwrap(); + } + + #[tokio::test] + async fn runs_tag() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("#frontend:lint").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn runs_tag_no_nodes() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target(Target::parse("#unknown:lint").unwrap(), None) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert!(graph.is_empty()); + } + } + + mod run_task_by_target_locator { + use super::*; + + #[tokio::test] + async fn runs_by_target() { + let sandbox = create_sandbox("tasks"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + builder + .run_task_by_target_locator( + TargetLocator::Qualified(Target::parse("server:build").unwrap()), + None, + ) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + async fn runs_by_file_path() { + let sandbox = create_sandbox("tasks"); + let mut container = ActionGraphContainer::new(sandbox.path()).await; + + container.project_graph.working_dir = sandbox.path().join("server/nested"); + + let mut builder = container.create_builder(); + + builder + .run_task_by_target_locator( + TargetLocator::TaskFromWorkingDir(Id::raw("lint")), + None, + ) + .unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + } + + #[tokio::test] + #[should_panic(expected = "No project could be located starting from path unknown/path.")] + async fn errors_if_no_project_by_path() { + let sandbox = create_sandbox("tasks"); + let mut container = ActionGraphContainer::new(sandbox.path()).await; + + container.project_graph.working_dir = sandbox.path().join("unknown/path"); + + let mut builder = container.create_builder(); + + builder + .run_task_by_target_locator( + TargetLocator::TaskFromWorkingDir(Id::raw("lint")), + None, + ) + .unwrap(); + } + } + + mod setup_tool { + use super::*; + + #[tokio::test] + async fn graphs() { + let pg = ProjectGraph::default(); + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + let system = Runtime::system(); + let node = Runtime::new( + PlatformType::Node, + RuntimeReq::with_version(Version::new(1, 2, 3)), + ); + + builder.setup_tool(&system); + builder.setup_tool(&node); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { runtime: node }, + ActionNode::SetupTool { runtime: system }, + ] + ); + } + + #[tokio::test] + async fn graphs_same_platform() { + let pg = ProjectGraph::default(); + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + + let node1 = Runtime::new( + PlatformType::Node, + RuntimeReq::with_version(Version::new(1, 2, 3)), + ); + let node2 = Runtime::new_override( + PlatformType::Node, + RuntimeReq::with_version(Version::new(4, 5, 6)), + ); + let node3 = Runtime::new(PlatformType::Node, RuntimeReq::Global); + + builder.setup_tool(&node1); + builder.setup_tool(&node2); + builder.setup_tool(&node3); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { runtime: node3 }, + ActionNode::SetupTool { runtime: node2 }, + ActionNode::SetupTool { runtime: node1 }, + ] + ); + } + + #[tokio::test] + async fn ignores_dupes() { + let pg = ProjectGraph::default(); + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + let system = Runtime::system(); + + builder.setup_tool(&system); + builder.setup_tool(&system); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { runtime: system }, + ] + ); + } + } + + mod sync_project { + use super::*; + + #[tokio::test] + async fn graphs_single() { + let pg = create_project_graph().await; + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + + let bar = pg.get("bar").unwrap(); + builder.sync_project(&bar).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: Runtime::system() + } + ] + ); + } + + #[tokio::test] + async fn graphs_single_with_dep() { + let pg = create_project_graph().await; + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + + let foo = pg.get("foo").unwrap(); + builder.sync_project(&foo).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("foo"), + runtime: Runtime::system() + } + ] + ); + } + + #[tokio::test] + async fn graphs_multiple() { + let pg = create_project_graph().await; + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + + let foo = pg.get("foo").unwrap(); + builder.sync_project(&foo).unwrap(); + + let bar = pg.get("bar").unwrap(); + builder.sync_project(&bar).unwrap(); + + let qux = pg.get("qux").unwrap(); + builder.sync_project(&qux).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("qux"), + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("foo"), + runtime: Runtime::system() + }, + ] + ); + } + + #[tokio::test] + async fn ignores_dupes() { + let pg = create_project_graph().await; + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + + let foo = pg.get("foo").unwrap(); + + builder.sync_project(&foo).unwrap(); + builder.sync_project(&foo).unwrap(); + builder.sync_project(&foo).unwrap(); + + let graph = builder.build().unwrap(); + + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: Runtime::system() + }, + ActionNode::SyncProject { + project: Id::raw("foo"), + runtime: Runtime::system() + } + ] + ); + } + + #[tokio::test] + async fn inherits_platform_tool() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let bar = container.project_graph.get("bar").unwrap(); + builder.sync_project(&bar).unwrap(); + + let qux = container.project_graph.get("qux").unwrap(); + builder.sync_project(&qux).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_rust_runtime() + }, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::SyncProject { + project: Id::raw("qux"), + runtime: create_rust_runtime() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: create_node_runtime() + } + ] + ); + } + + #[tokio::test] + async fn supports_platform_override() { + let sandbox = create_sandbox("projects"); + let container = ActionGraphContainer::new(sandbox.path()).await; + let mut builder = container.create_builder(); + + let bar = container.project_graph.get("bar").unwrap(); + builder.sync_project(&bar).unwrap(); + + let baz = container.project_graph.get("baz").unwrap(); + builder.sync_project(&baz).unwrap(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!( + topo(graph), + vec![ + ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: Runtime::new_override( + PlatformType::Node, + RuntimeReq::with_version(Version::new(18, 0, 0)) + ) + }, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::SyncProject { + project: Id::raw("baz"), + runtime: Runtime::new_override( + PlatformType::Node, + RuntimeReq::with_version(Version::new(18, 0, 0)) + ) + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: create_node_runtime() + }, + ] + ); + } + } + + mod sync_workspace { + use super::*; + + #[tokio::test] + async fn graphs() { + let pg = ProjectGraph::default(); + + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + builder.sync_workspace(); + + let graph = builder.build().unwrap(); + + assert_snapshot!(graph.to_dot()); + assert_eq!(topo(graph), vec![ActionNode::SyncWorkspace]); + } + + #[tokio::test] + async fn ignores_dupes() { + let pg = ProjectGraph::default(); + + let mut builder = ActionGraphBuilder::new(&pg).unwrap(); + builder.sync_workspace(); + builder.sync_workspace(); + builder.sync_workspace(); + + let graph = builder.build().unwrap(); + + assert_eq!(topo(graph), vec![ActionNode::SyncWorkspace]); + } + } +} diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__graphs.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__graphs.snap new file mode 100644 index 00000000000..10d4bd747e0 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__graphs.snap @@ -0,0 +1,12 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="InstallNodeDeps(20.0.0)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__installs_in_project_when_not_in_depman_workspace.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__installs_in_project_when_not_in_depman_workspace.snap new file mode 100644 index 00000000000..3e423cd537e --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__install_deps__installs_in_project_when_not_in_depman_workspace.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="InstallNodeDeps(20.0.0)" ] + 3 [ label="InstallNodeDepsInProject(20.0.0, out)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 1 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__graphs.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__graphs.snap new file mode 100644 index 00000000000..cb49bf8219b --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__graphs.snap @@ -0,0 +1,17 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="InstallNodeDeps(20.0.0)" ] + 3 [ label="SyncNodeProject(bar)" ] + 4 [ label="RunTask(bar:build)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 1 [ ] + 4 -> 2 [ ] + 4 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__task_can_have_a_diff_platform_from_project.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__task_can_have_a_diff_platform_from_project.snap new file mode 100644 index 00000000000..38f0da434df --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task__task_can_have_a_diff_platform_from_project.snap @@ -0,0 +1,19 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupRustTool(1.70.0)" ] + 2 [ label="InstallRustDeps(1.70.0)" ] + 3 [ label="SetupNodeTool(20.0.0)" ] + 4 [ label="SyncNodeProject(bar)" ] + 5 [ label="RunTask(bar:build)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 0 [ ] + 4 -> 3 [ ] + 5 -> 2 [ ] + 5 -> 4 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap new file mode 100644 index 00000000000..e03fe506210 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap @@ -0,0 +1,31 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(base)" ] + 3 [ label="RunTask(base:build)" ] + 4 [ label="SyncSystemProject(server)" ] + 5 [ label="RunTask(server:build)" ] + 6 [ label="SyncSystemProject(common)" ] + 7 [ label="RunTask(common:build)" ] + 8 [ label="SyncSystemProject(client)" ] + 9 [ label="RunTask(client:build)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] + 4 -> 1 [ ] + 5 -> 4 [ ] + 6 -> 1 [ ] + 6 -> 2 [ ] + 7 -> 6 [ ] + 8 -> 1 [ ] + 8 -> 4 [ ] + 8 -> 6 [ ] + 9 -> 8 [ ] + 9 -> 7 [ ] + 9 -> 5 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all_with_query.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all_with_query.snap new file mode 100644 index 00000000000..6dc1e1b0df3 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all_with_query.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(server)" ] + 3 [ label="RunTask(server:build)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_by_project.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_by_project.snap new file mode 100644 index 00000000000..6e78aa95cfd --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_by_project.snap @@ -0,0 +1,23 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(client)" ] + 3 [ label="SyncSystemProject(server)" ] + 4 [ label="SyncSystemProject(common)" ] + 5 [ label="SyncSystemProject(base)" ] + 6 [ label="RunTask(client:lint)" ] + 1 -> 0 [ ] + 3 -> 1 [ ] + 5 -> 1 [ ] + 4 -> 1 [ ] + 4 -> 5 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] + 2 -> 4 [ ] + 6 -> 2 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_tag.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_tag.snap new file mode 100644 index 00000000000..8943f0f12b9 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_tag.snap @@ -0,0 +1,25 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(client)" ] + 3 [ label="SyncSystemProject(server)" ] + 4 [ label="SyncSystemProject(common)" ] + 5 [ label="SyncSystemProject(base)" ] + 6 [ label="RunTask(client:lint)" ] + 7 [ label="RunTask(common:lint)" ] + 1 -> 0 [ ] + 3 -> 1 [ ] + 5 -> 1 [ ] + 4 -> 1 [ ] + 4 -> 5 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] + 2 -> 4 [ ] + 6 -> 2 [ ] + 7 -> 4 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_file_path.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_file_path.snap new file mode 100644 index 00000000000..d074f59cf86 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_file_path.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(server)" ] + 3 [ label="RunTask(server:lint)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_target.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_target.snap new file mode 100644 index 00000000000..6dc1e1b0df3 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target_locator__runs_by_target.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(server)" ] + 3 [ label="RunTask(server:build)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__can_create_a_chain.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__can_create_a_chain.snap new file mode 100644 index 00000000000..290953fed5f --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__can_create_a_chain.snap @@ -0,0 +1,20 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(deps)" ] + 3 [ label="RunTask(deps:chain1)" ] + 4 [ label="RunTask(deps:chain2)" ] + 5 [ label="RunTask(deps:chain3)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 5 -> 2 [ ] + 4 -> 2 [ ] + 4 -> 5 [ ] + 3 -> 2 [ ] + 3 -> 4 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__doesnt_include_dependents.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__doesnt_include_dependents.snap new file mode 100644 index 00000000000..7de75775a1d --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__doesnt_include_dependents.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(deps)" ] + 3 [ label="RunTask(deps:base)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__includes_dependents.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__includes_dependents.snap new file mode 100644 index 00000000000..7375595d48b --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__includes_dependents.snap @@ -0,0 +1,23 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(deps)" ] + 3 [ label="RunTask(deps:base)" ] + 4 [ label="RunTask(deps:internal)" ] + 5 [ label="SyncSystemProject(deps-external)" ] + 6 [ label="RunTask(deps-external:external)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 2 [ ] + 4 -> 2 [ ] + 4 -> 3 [ ] + 5 -> 1 [ ] + 5 -> 2 [ ] + 6 -> 5 [ ] + 6 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_parallel.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_parallel.snap new file mode 100644 index 00000000000..f5d5002c747 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_parallel.snap @@ -0,0 +1,23 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(deps)" ] + 3 [ label="RunTask(deps:parallel)" ] + 4 [ label="RunTask(deps:c)" ] + 5 [ label="RunTask(deps:a)" ] + 6 [ label="RunTask(deps:b)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 4 -> 2 [ ] + 5 -> 2 [ ] + 6 -> 2 [ ] + 3 -> 2 [ ] + 3 -> 4 [ ] + 3 -> 5 [ ] + 3 -> 6 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_serial.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_serial.snap new file mode 100644 index 00000000000..e444a36cc07 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_dependencies__runs_deps_in_serial.snap @@ -0,0 +1,23 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(deps)" ] + 3 [ label="RunTask(deps:serial)" ] + 4 [ label="RunTask(deps:b)" ] + 5 [ label="RunTask(deps:c)" ] + 6 [ label="RunTask(deps:a)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 4 -> 2 [ ] + 5 -> 2 [ ] + 5 -> 4 [ ] + 6 -> 2 [ ] + 6 -> 5 [ ] + 3 -> 2 [ ] + 3 -> 6 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs.snap new file mode 100644 index 00000000000..e8e7634758e --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs.snap @@ -0,0 +1,12 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SetupNodeTool(1.2.3)" ] + 1 -> 0 [ ] + 2 -> 0 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs_same_platform.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs_same_platform.snap new file mode 100644 index 00000000000..1c0fe1c6ada --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__setup_tool__graphs_same_platform.snap @@ -0,0 +1,14 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(1.2.3)" ] + 2 [ label="SetupNodeTool(4.5.6)" ] + 3 [ label="SetupNodeTool(global)" ] + 1 -> 0 [ ] + 2 -> 0 [ ] + 3 -> 0 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs.snap new file mode 100644 index 00000000000..46d410389a3 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs.snap @@ -0,0 +1,15 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(foo)" ] + 3 [ label="SyncSystemProject(bar)" ] + 1 -> 0 [ ] + 3 -> 1 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap new file mode 100644 index 00000000000..6f9dda954a9 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_multiple.snap @@ -0,0 +1,17 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(foo)" ] + 3 [ label="SyncSystemProject(bar)" ] + 4 [ label="SyncSystemProject(qux)" ] + 1 -> 0 [ ] + 3 -> 1 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] + 4 -> 1 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap new file mode 100644 index 00000000000..ddfdb147fe1 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single.snap @@ -0,0 +1,12 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(bar)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap new file mode 100644 index 00000000000..46d410389a3 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__graphs_single_with_dep.snap @@ -0,0 +1,15 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupSystemTool" ] + 2 [ label="SyncSystemProject(foo)" ] + 3 [ label="SyncSystemProject(bar)" ] + 1 -> 0 [ ] + 3 -> 1 [ ] + 2 -> 1 [ ] + 2 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap new file mode 100644 index 00000000000..8a93b55531b --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__inherits_platform_tool.snap @@ -0,0 +1,16 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="SyncNodeProject(bar)" ] + 3 [ label="SetupRustTool(1.70.0)" ] + 4 [ label="SyncRustProject(qux)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 0 [ ] + 4 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap new file mode 100644 index 00000000000..8f0bbd2e090 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_project__supports_platform_override.snap @@ -0,0 +1,16 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(20.0.0)" ] + 2 [ label="SyncNodeProject(bar)" ] + 3 [ label="SetupNodeTool(18.0.0)" ] + 4 [ label="SyncNodeProject(baz)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 0 [ ] + 4 -> 3 [ ] +} + diff --git a/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_workspace__graphs.snap b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_workspace__graphs.snap new file mode 100644 index 00000000000..35bf4b9bad6 --- /dev/null +++ b/nextgen/action-graph/tests/snapshots/action_graph_test__action_graph__sync_workspace__graphs.snap @@ -0,0 +1,8 @@ +--- +source: nextgen/action-graph/tests/action_graph_test.rs +expression: graph.to_dot() +--- +digraph { + 0 [ label="SyncWorkspace" ] +} + diff --git a/nextgen/action-graph/tests/utils.rs b/nextgen/action-graph/tests/utils.rs new file mode 100644 index 00000000000..0c61e18096c --- /dev/null +++ b/nextgen/action-graph/tests/utils.rs @@ -0,0 +1,27 @@ +use moon_action_graph::ActionGraphBuilder; +use moon_platform::PlatformManager; +use moon_project_graph::ProjectGraph; +use moon_test_utils2::{ + generate_platform_manager_from_sandbox, generate_project_graph_from_sandbox, +}; +use std::path::{Path, PathBuf}; + +pub struct ActionGraphContainer { + pub platform_manager: PlatformManager, + pub project_graph: ProjectGraph, + pub workspace_root: PathBuf, +} + +impl ActionGraphContainer { + pub async fn new(root: &Path) -> Self { + Self { + platform_manager: generate_platform_manager_from_sandbox(root).await, + project_graph: generate_project_graph_from_sandbox(root).await, + workspace_root: root.to_path_buf(), + } + } + + pub fn create_builder(&self) -> ActionGraphBuilder { + ActionGraphBuilder::with_platforms(&self.platform_manager, &self.project_graph).unwrap() + } +} diff --git a/nextgen/config/tests/inherited_tasks_config_test.rs b/nextgen/config/tests/inherited_tasks_config_test.rs index bdf2be65f73..b77ab251d92 100644 --- a/nextgen/config/tests/inherited_tasks_config_test.rs +++ b/nextgen/config/tests/inherited_tasks_config_test.rs @@ -198,8 +198,6 @@ fileGroups: let url = server.url("/config.yml"); - dbg!(&url); - sandbox.create_file( "tasks.yml", format!( diff --git a/nextgen/platform-runtime/src/lib.rs b/nextgen/platform-runtime/src/lib.rs index f6683fae574..b3627dc60a5 100644 --- a/nextgen/platform-runtime/src/lib.rs +++ b/nextgen/platform-runtime/src/lib.rs @@ -12,6 +12,10 @@ pub enum RuntimeReq { } impl RuntimeReq { + pub fn with_version(version: Version) -> Self { + Self::Toolchain(UnresolvedVersionSpec::Version(version)) + } + pub fn is_global(&self) -> bool { matches!(self, Self::Global) } diff --git a/nextgen/project-builder/src/project_builder.rs b/nextgen/project-builder/src/project_builder.rs index 5fbbbf469ef..51aec8e3699 100644 --- a/nextgen/project-builder/src/project_builder.rs +++ b/nextgen/project-builder/src/project_builder.rs @@ -1,12 +1,13 @@ use moon_common::path::WorkspaceRelativePath; use moon_common::{color, consts, Id}; use moon_config::{ - DependencyConfig, DependencySource, InheritedTasksManager, InheritedTasksResult, LanguageType, - PlatformType, ProjectConfig, ProjectDependsOn, TaskConfig, ToolchainConfig, + DependencyConfig, DependencyScope, DependencySource, InheritedTasksManager, + InheritedTasksResult, LanguageType, PlatformType, ProjectConfig, ProjectDependsOn, TaskConfig, + ToolchainConfig, }; use moon_file_group::FileGroup; use moon_project::Project; -use moon_task::Task; +use moon_task::{TargetScope, Task}; use moon_task_builder::{DetectPlatformEvent, TasksBuilder, TasksBuilderContext}; use rustc_hash::FxHashMap; use starbase_events::{Emitter, Event}; @@ -197,11 +198,13 @@ impl<'app> ProjectBuilder<'app> { #[tracing::instrument(name = "project", skip_all)] pub async fn build(mut self) -> miette::Result { + let tasks = self.build_tasks().await?; + let mut project = Project { alias: self.alias.map(|a| a.to_owned()), - dependencies: self.build_dependencies()?, + dependencies: self.build_dependencies(&tasks)?, file_groups: self.build_file_groups()?, - tasks: self.build_tasks().await?, + tasks, id: self.id.to_owned(), language: self.language, platform: self.platform, @@ -220,7 +223,10 @@ impl<'app> ProjectBuilder<'app> { Ok(project) } - fn build_dependencies(&self) -> miette::Result> { + fn build_dependencies( + &self, + tasks: &BTreeMap, + ) -> miette::Result> { let mut deps = FxHashMap::default(); trace!(id = self.id.as_str(), "Building project dependencies"); @@ -237,7 +243,41 @@ impl<'app> ProjectBuilder<'app> { deps.insert(dep_config.id.clone(), dep_config); } + } + + // Tasks can depend on arbitray projects, so include them also + for task_config in tasks.values() { + for task_dep in &task_config.deps { + if let TargetScope::Project(dep_id) = &task_dep.scope { + // Already a dependency, or references self + if deps.contains_key(dep_id) + || self.id == dep_id + || self.alias.as_ref().is_some_and(|a| *a == dep_id.as_str()) + { + continue; + } + + trace!( + id = self.id.as_str(), + dep = dep_id.as_str(), + task = task_config.target.as_str(), + "Marking arbitrary project as a peer dependency because of a task dependency" + ); + + deps.insert( + dep_id.to_owned(), + DependencyConfig { + id: dep_id.to_owned(), + scope: DependencyScope::Peer, + source: DependencySource::Implicit, + via: Some(task_config.target.to_string()), + }, + ); + } + } + } + if !deps.is_empty() { trace!( id = self.id.as_str(), deps = ?deps.keys().map(|k| k.as_str()).collect::>(), diff --git a/nextgen/project-graph/Cargo.toml b/nextgen/project-graph/Cargo.toml index a7f09387974..d0ad40416a8 100644 --- a/nextgen/project-graph/Cargo.toml +++ b/nextgen/project-graph/Cargo.toml @@ -32,5 +32,6 @@ thiserror = { workspace = true } tracing = { workspace = true } [dev-dependencies] +moon_test_utils2 = { path = "../test-utils" } starbase_sandbox = { workspace = true } tokio = { workspace = true } diff --git a/nextgen/project-graph/src/project_graph.rs b/nextgen/project-graph/src/project_graph.rs index c06508c90db..38b15e90294 100644 --- a/nextgen/project-graph/src/project_graph.rs +++ b/nextgen/project-graph/src/project_graph.rs @@ -34,10 +34,22 @@ pub struct ProjectNode { pub source: WorkspaceRelativePathBuf, } +impl ProjectNode { + pub fn new(index: usize) -> Self { + ProjectNode { + index: NodeIndex::new(index), + ..ProjectNode::default() + } + } +} + #[derive(Default)] pub struct ProjectGraph { pub check_boundaries: bool, + /// Cache of file path lookups, mapped by starting path to project ID (as a string). + fs_cache: OnceMap, + /// Directed-acyclic graph (DAG) of non-expanded projects and their dependencies. graph: GraphType, @@ -67,6 +79,7 @@ impl ProjectGraph { projects: Arc::new(RwLock::new(FxHashMap::default())), working_dir: workspace_root.to_owned(), workspace_root: workspace_root.to_owned(), + fs_cache: OnceMap::new(), query_cache: OnceMap::new(), check_boundaries: false, } @@ -162,36 +175,7 @@ impl ProjectGraph { current_file }; - // Find the deepest matching path in case sub-projects are being used - let mut remaining_length = 1000; // Start with a really fake number - let mut possible_id = String::new(); - - for (id, node) in &self.nodes { - if !file.starts_with(node.source.as_str()) { - continue; - } - - if let Ok(diff) = file.relative_to(node.source.as_str()) { - let diff_comps = diff.components().count(); - - // Exact match, abort - if diff_comps == 0 { - possible_id = id.as_str().to_owned(); - break; - } - - if diff_comps < remaining_length { - remaining_length = diff_comps; - possible_id = id.as_str().to_owned(); - } - } - } - - if possible_id.is_empty() { - return Err(ProjectGraphError::MissingFromPath(file.to_path_buf()).into()); - } - - self.get(&possible_id) + self.get(self.internal_search(file)?) } /// Return a list of IDs for all projects currently within the graph. @@ -312,10 +296,10 @@ impl ProjectGraph { let query = query.as_ref(); let query_input = query .input - .clone() + .as_ref() .expect("Querying the project graph requires a query input string."); - self.query_cache.try_insert(query_input.clone(), |_| { + self.query_cache.try_insert(query_input.to_owned(), |_| { debug!("Querying projects with {}", color::shell(query_input)); let mut project_ids = vec![]; @@ -345,6 +329,41 @@ impl ProjectGraph { }) } + fn internal_search(&self, search: &Path) -> miette::Result<&str> { + self.fs_cache.try_insert(search.to_path_buf(), |_| { + // Find the deepest matching path in case sub-projects are being used + let mut remaining_length = 1000; // Start with a really fake number + let mut possible_id = String::new(); + + for (id, node) in &self.nodes { + if !search.starts_with(node.source.as_str()) { + continue; + } + + if let Ok(diff) = search.relative_to(node.source.as_str()) { + let diff_comps = diff.components().count(); + + // Exact match, abort + if diff_comps == 0 { + possible_id = id.as_str().to_owned(); + break; + } + + if diff_comps < remaining_length { + remaining_length = diff_comps; + possible_id = id.as_str().to_owned(); + } + } + } + + if possible_id.is_empty() { + return Err(ProjectGraphError::MissingFromPath(search.to_path_buf()).into()); + } + + Ok(possible_id) + }) + } + fn resolve_id(&self, alias_or_id: &str) -> Id { Id::raw(if self.nodes.contains_key(alias_or_id) { alias_or_id diff --git a/nextgen/project-graph/src/project_graph_builder.rs b/nextgen/project-graph/src/project_graph_builder.rs index 8e02337e838..b3a721cc6f7 100644 --- a/nextgen/project-graph/src/project_graph_builder.rs +++ b/nextgen/project-graph/src/project_graph_builder.rs @@ -1,5 +1,4 @@ -use crate::project_events::ExtendProjectEvent; -use crate::project_events::ExtendProjectGraphEvent; +use crate::project_events::{ExtendProjectEvent, ExtendProjectGraphEvent}; use crate::project_graph::{GraphType, ProjectGraph, ProjectNode}; use crate::project_graph_cache::ProjectsState; use crate::project_graph_error::ProjectGraphError; @@ -7,17 +6,13 @@ use crate::project_graph_hash::ProjectGraphHash; use crate::projects_locator::locate_projects_with_globs; use async_recursion::async_recursion; use moon_cache::CacheEngine; -use moon_common::is_test_env; use moon_common::path::{to_virtual_string, WorkspaceRelativePath, WorkspaceRelativePathBuf}; -use moon_common::{color, consts, Id}; -use moon_config::{ - DependencyScope, InheritedTasksManager, ToolchainConfig, WorkspaceConfig, WorkspaceProjects, -}; +use moon_common::{color, consts, is_test_env, Id}; +use moon_config::{InheritedTasksManager, ToolchainConfig, WorkspaceConfig, WorkspaceProjects}; use moon_hash::HashEngine; use moon_project::Project; use moon_project_builder::{DetectLanguageEvent, ProjectBuilder, ProjectBuilderContext}; use moon_project_constraints::{enforce_project_type_relationships, enforce_tag_relationships}; -use moon_task::TargetScope; use moon_task_builder::DetectPlatformEvent; use moon_vcs::BoxedVcs; use petgraph::graph::DiGraph; @@ -256,36 +251,6 @@ impl<'app> ProjectGraphBuilder<'app> { } } - // Tasks can depend on arbitray projects, so include them also - for (task_id, task_config) in &project.tasks { - for task_dep in &task_config.deps { - if let TargetScope::Project(dep_id) = &task_dep.scope { - if - // Already a dependency - project.dependencies.contains_key(dep_id) || - // Don't reference itself - project.matches_locator(dep_id.as_str()) - { - continue; - } - - if cycle.contains(dep_id) { - warn!( - id = id.as_str(), - dependency_id = dep_id.as_str(), - task_id = task_id.as_str(), - "Encountered a dependency cycle (from task); will disconnect nodes to avoid recursion", - ); - } else { - edges.push(( - self.internal_load(dep_id, cycle).await?, - DependencyScope::Peer, - )); - } - } - } - } - // Insert into the graph and connect edges let index = self.graph.add_node(project); diff --git a/nextgen/project-graph/tests/project_graph_test.rs b/nextgen/project-graph/tests/project_graph_test.rs index 380722c6b97..96bd361e5d4 100644 --- a/nextgen/project-graph/tests/project_graph_test.rs +++ b/nextgen/project-graph/tests/project_graph_test.rs @@ -1,124 +1,26 @@ use moon_common::{path::WorkspaceRelativePathBuf, Id}; -use moon_config::PartialTaskConfig; use moon_config::{ - DependencyConfig, DependencyScope, DependencySource, InheritedTasksEntry, - InheritedTasksManager, NodeConfig, PartialInheritedTasksConfig, ToolchainConfig, - WorkspaceConfig, WorkspaceProjects, WorkspaceProjectsConfig, + DependencyConfig, DependencyScope, DependencySource, InheritedTasksManager, WorkspaceProjects, + WorkspaceProjectsConfig, }; use moon_project::{FileGroup, Project}; -use moon_project_builder::DetectLanguageEvent; use moon_project_graph::{ ExtendProjectData, ExtendProjectEvent, ExtendProjectGraphData, ExtendProjectGraphEvent, - ProjectGraph, ProjectGraphBuilder, ProjectGraphBuilderContext, + ProjectGraph, ProjectGraphBuilder, }; use moon_query::build_query; use moon_task::Target; -use moon_task_builder::DetectPlatformEvent; -use moon_vcs::{BoxedVcs, Git}; +use moon_test_utils2::*; use rustc_hash::{FxHashMap, FxHashSet}; -use starbase_events::{Emitter, EventState}; +use starbase_events::EventState; use starbase_sandbox::{assert_snapshot, create_sandbox, Sandbox}; -use starbase_utils::string_vec; -use starbase_utils::{fs, json}; -use std::collections::BTreeMap; +use starbase_utils::{fs, json, string_vec}; use std::fs::OpenOptions; use std::io::Write; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::Arc; use tokio::sync::RwLock; -#[derive(Default)] -struct GraphContainer { - pub inherited_tasks: InheritedTasksManager, - pub toolchain_config: ToolchainConfig, - pub workspace_config: WorkspaceConfig, - pub workspace_root: PathBuf, - pub vcs: Option, -} - -impl GraphContainer { - pub fn new(root: &Path) -> Self { - let mut graph = Self { - workspace_root: root.to_path_buf(), - ..Default::default() - }; - - // Add a global task to all projects - graph.inherited_tasks.configs.insert( - "*".into(), - InheritedTasksEntry { - input: ".moon/tasks.yml".into(), - config: PartialInheritedTasksConfig { - tasks: Some(BTreeMap::from_iter([( - "global".into(), - PartialTaskConfig::default(), - )])), - ..PartialInheritedTasksConfig::default() - }, - }, - ); - - // Always use the node platform - graph.toolchain_config.node = Some(NodeConfig::default()); - - // Use folders as project names - graph.workspace_config.projects = WorkspaceProjects::Globs(string_vec!["*"]); - - graph - } - - pub fn new_with_vcs(root: &Path) -> Self { - let mut container = Self::new(root); - container.vcs = Some(Box::new(Git::load(root, "master", &[]).unwrap())); - container - } - - pub fn create_context(&self) -> ProjectGraphBuilderContext { - ProjectGraphBuilderContext { - extend_project: Emitter::::new(), - extend_project_graph: Emitter::::new(), - detect_language: Emitter::::new(), - detect_platform: Emitter::::new(), - inherited_tasks: &self.inherited_tasks, - toolchain_config: &self.toolchain_config, - vcs: self.vcs.as_ref(), - working_dir: &self.workspace_root, - workspace_config: &self.workspace_config, - workspace_root: &self.workspace_root, - } - } - - pub async fn build_graph<'l>(&self, context: ProjectGraphBuilderContext<'l>) -> ProjectGraph { - let mut builder = ProjectGraphBuilder::new(context).await.unwrap(); - builder.load_all().await.unwrap(); - - let mut graph = builder.build().await.unwrap(); - graph.check_boundaries = true; - graph.get_all().unwrap(); - graph - } - - pub async fn build_graph_for<'l>( - &self, - context: ProjectGraphBuilderContext<'l>, - ids: &[&str], - ) -> ProjectGraph { - let mut builder = ProjectGraphBuilder::new(context).await.unwrap(); - - for id in ids { - builder.load(id).await.unwrap(); - } - - let graph = builder.build().await.unwrap(); - - for id in ids { - graph.get(id).unwrap(); - } - - graph - } -} - pub fn append_file>(path: P, data: &str) { let mut file = OpenOptions::new() .write(true) @@ -145,19 +47,6 @@ fn get_ids_from_projects(projects: Vec>) -> Vec { mod project_graph { use super::*; - async fn generate_project_graph(fixture: &str) -> ProjectGraph { - let sandbox = create_sandbox(fixture); - - generate_project_graph_from_sandbox(sandbox.path()).await - } - - async fn generate_project_graph_from_sandbox(path: &Path) -> ProjectGraph { - let container = GraphContainer::new(path); - let context = container.create_context(); - - container.build_graph(context).await - } - #[tokio::test] async fn gets_by_id() { let graph = generate_project_graph("dependencies").await; @@ -219,7 +108,7 @@ mod project_graph { // Move files so that we can infer a compatible root project name fs::copy_dir_all(sandbox.path(), sandbox.path(), &root).unwrap(); - let mut container = GraphContainer::new(&root); + let mut container = ProjectGraphContainer::new(&root); container.workspace_config.projects = WorkspaceProjects::Globs(string_vec!["*", "."]); @@ -235,7 +124,7 @@ mod project_graph { #[tokio::test] async fn paths() { let sandbox = create_sandbox("dependencies"); - let mut container = GraphContainer::new(sandbox.path()); + let mut container = ProjectGraphContainer::new(sandbox.path()); container.workspace_config.projects = WorkspaceProjects::Sources(FxHashMap::from_iter([ @@ -252,7 +141,7 @@ mod project_graph { #[tokio::test] async fn paths_and_globs() { let sandbox = create_sandbox("dependencies"); - let mut container = GraphContainer::new(sandbox.path()); + let mut container = ProjectGraphContainer::new(sandbox.path()); container.workspace_config.projects = WorkspaceProjects::Both(WorkspaceProjectsConfig { @@ -308,7 +197,7 @@ mod project_graph { sandbox.enable_git(); sandbox.create_file(".gitignore", "*-other"); - let container = GraphContainer::new_with_vcs(sandbox.path()); + let container = ProjectGraphContainer::with_vcs(sandbox.path()); let context = container.create_context(); let graph = container.build_graph(context).await; @@ -348,7 +237,7 @@ mod project_graph { async fn do_generate(root: &Path) -> ProjectGraph { let cache_engine = CacheEngine::new(root).unwrap(); let hash_engine = HashEngine::new(&cache_engine.cache_dir).unwrap(); - let container = GraphContainer::new_with_vcs(root); + let container = ProjectGraphContainer::with_vcs(root); let context = container.create_context(); let mut builder = ProjectGraphBuilder::generate(context, &cache_engine, &hash_engine) @@ -538,13 +427,12 @@ mod project_graph { async fn generate_inheritance_project_graph(fixture: &str) -> ProjectGraph { let sandbox = create_sandbox(fixture); - let mut container = GraphContainer::new(sandbox.path()); - container.inherited_tasks = - InheritedTasksManager::load(sandbox.path(), sandbox.path().join(".moon")).unwrap(); - - let context = container.create_context(); - - container.build_graph(context).await + generate_project_graph_with_changes(sandbox.path(), |container| { + container.inherited_tasks = + InheritedTasksManager::load(sandbox.path(), sandbox.path().join(".moon")) + .unwrap(); + }) + .await } #[tokio::test] @@ -802,7 +690,7 @@ mod project_graph { #[tokio::test] async fn no_depends_on() { let sandbox = create_sandbox("dependency-types"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); let graph = container.build_graph_for(context, &["no-depends-on"]).await; @@ -812,7 +700,7 @@ mod project_graph { #[tokio::test] async fn some_depends_on() { let sandbox = create_sandbox("dependency-types"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); let graph = container .build_graph_for(context, &["some-depends-on"]) @@ -824,7 +712,7 @@ mod project_graph { #[tokio::test] async fn from_task_deps() { let sandbox = create_sandbox("dependency-types"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); let graph = container .build_graph_for(context, &["from-task-deps"]) @@ -836,7 +724,7 @@ mod project_graph { #[tokio::test] async fn self_task_deps() { let sandbox = create_sandbox("dependency-types"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); let graph = container .build_graph_for(context, &["self-task-deps"]) @@ -852,7 +740,7 @@ mod project_graph { async fn generate_aliases_project_graph() -> ProjectGraph { let sandbox = create_sandbox("aliases"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); // Set aliases for projects @@ -1036,7 +924,7 @@ mod project_graph { func(&sandbox); - let mut container = GraphContainer::new(sandbox.path()); + let mut container = ProjectGraphContainer::new(sandbox.path()); container .workspace_config @@ -1178,7 +1066,7 @@ mod project_graph { func(&sandbox); - let mut container = GraphContainer::new(sandbox.path()); + let mut container = ProjectGraphContainer::new(sandbox.path()); container .workspace_config @@ -1464,7 +1352,7 @@ mod project_graph { #[tokio::test] async fn renders_partial() { let sandbox = create_sandbox("dependencies"); - let container = GraphContainer::new(sandbox.path()); + let container = ProjectGraphContainer::new(sandbox.path()); let context = container.create_context(); let graph = container.build_graph_for(context, &["b"]).await; diff --git a/nextgen/task/src/lib.rs b/nextgen/task/src/lib.rs index 6ad258d2350..7b3bb75e447 100644 --- a/nextgen/task/src/lib.rs +++ b/nextgen/task/src/lib.rs @@ -2,6 +2,6 @@ mod task; mod task_options; pub use moon_config::{TaskConfig, TaskOptionsConfig, TaskType}; -pub use moon_target::{Target, TargetScope}; +pub use moon_target::*; pub use task::*; pub use task_options::*; diff --git a/nextgen/test-utils/Cargo.toml b/nextgen/test-utils/Cargo.toml new file mode 100644 index 00000000000..f290514b133 --- /dev/null +++ b/nextgen/test-utils/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "moon_test_utils2" +version = "0.1.0" +edition = "2021" +license = "MIT" +description = "Testing utilities." +homepage = "https://moonrepo.dev/moon" +repository = "https://github.com/moonrepo/moon" + +[dependencies] +moon_config = { path = "../config" } +moon_project_graph = { path = "../project-graph" } +moon_vcs = { path = "../vcs" } +miette = { workspace = true } +proto_core = { workspace = true } +starbase_events = { workspace = true } +starbase_sandbox = { workspace = true } + +# TODO +moon_platform = { path = "../../crates/core/platform" } +moon_node_platform = { path = "../../crates/node/platform" } +moon_rust_platform = { path = "../../crates/rust/platform" } +moon_system_platform = { path = "../../crates/system/platform" } diff --git a/nextgen/test-utils/src/lib.rs b/nextgen/test-utils/src/lib.rs new file mode 100644 index 00000000000..5b73bce286f --- /dev/null +++ b/nextgen/test-utils/src/lib.rs @@ -0,0 +1,5 @@ +mod platform_manager; +mod project_graph; + +pub use platform_manager::*; +pub use project_graph::*; diff --git a/nextgen/test-utils/src/platform_manager.rs b/nextgen/test-utils/src/platform_manager.rs new file mode 100644 index 00000000000..50fffc38380 --- /dev/null +++ b/nextgen/test-utils/src/platform_manager.rs @@ -0,0 +1,32 @@ +use moon_config::{PlatformType, ToolchainConfig, ToolsConfig}; +use moon_node_platform::NodePlatform; +use moon_platform::PlatformManager; +use moon_rust_platform::RustPlatform; +use moon_system_platform::SystemPlatform; +use proto_core::ProtoEnvironment; +use std::path::Path; +use std::sync::Arc; + +pub async fn generate_platform_manager_from_sandbox(root: &Path) -> PlatformManager { + let proto = Arc::new(ProtoEnvironment::new_testing(root)); + let config = ToolchainConfig::load_from(root, &ToolsConfig::default()).unwrap(); + let mut manager = PlatformManager::default(); + + if let Some(node_config) = &config.node { + manager.register( + PlatformType::Node, + Box::new(NodePlatform::new(node_config, &None, root, proto.clone())), + ); + } + + if let Some(rust_config) = &config.rust { + manager.register( + PlatformType::Rust, + Box::new(RustPlatform::new(rust_config, root, proto.clone())), + ); + } + + manager.register(PlatformType::System, Box::::default()); + + manager +} diff --git a/nextgen/test-utils/src/project_graph.rs b/nextgen/test-utils/src/project_graph.rs new file mode 100644 index 00000000000..5478968b90d --- /dev/null +++ b/nextgen/test-utils/src/project_graph.rs @@ -0,0 +1,132 @@ +use moon_config::{ + InheritedTasksEntry, InheritedTasksManager, NodeConfig, PartialInheritedTasksConfig, + PartialTaskConfig, ToolchainConfig, ToolsConfig, WorkspaceConfig, WorkspaceProjects, +}; +use moon_project_graph::{ + DetectLanguageEvent, DetectPlatformEvent, ExtendProjectEvent, ExtendProjectGraphEvent, + ProjectGraph, ProjectGraphBuilder, ProjectGraphBuilderContext, +}; +use moon_vcs::{BoxedVcs, Git}; +use starbase_events::Emitter; +use starbase_sandbox::create_sandbox; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +#[derive(Default)] +pub struct ProjectGraphContainer { + pub inherited_tasks: InheritedTasksManager, + pub toolchain_config: ToolchainConfig, + pub workspace_config: WorkspaceConfig, + pub workspace_root: PathBuf, + pub vcs: Option, +} + +impl ProjectGraphContainer { + pub fn new(root: &Path) -> Self { + let proto = ToolsConfig::default(); + let mut graph = Self { + inherited_tasks: InheritedTasksManager::load_from(root).unwrap(), + toolchain_config: ToolchainConfig::load_from(root, &proto).unwrap(), + workspace_root: root.to_path_buf(), + ..Default::default() + }; + + // Add a global task to all projects + graph.inherited_tasks.configs.insert( + "*".into(), + InheritedTasksEntry { + input: ".moon/tasks.yml".into(), + config: PartialInheritedTasksConfig { + tasks: Some(BTreeMap::from_iter([( + "global".into(), + PartialTaskConfig::default(), + )])), + ..PartialInheritedTasksConfig::default() + }, + }, + ); + + // Always use the node platform + if graph.toolchain_config.node.is_none() { + graph.toolchain_config.node = Some(NodeConfig::default()); + } + + // Use folders as project names + graph.workspace_config.projects = WorkspaceProjects::Globs(vec!["*".into()]); + + graph + } + + pub fn with_vcs(root: &Path) -> Self { + let mut container = Self::new(root); + container.vcs = Some(Box::new(Git::load(root, "master", &[]).unwrap())); + container + } + + pub fn create_context(&self) -> ProjectGraphBuilderContext { + ProjectGraphBuilderContext { + extend_project: Emitter::::new(), + extend_project_graph: Emitter::::new(), + detect_language: Emitter::::new(), + detect_platform: Emitter::::new(), + inherited_tasks: &self.inherited_tasks, + toolchain_config: &self.toolchain_config, + vcs: self.vcs.as_ref(), + working_dir: &self.workspace_root, + workspace_config: &self.workspace_config, + workspace_root: &self.workspace_root, + } + } + + pub async fn build_graph<'l>(&self, context: ProjectGraphBuilderContext<'l>) -> ProjectGraph { + let mut builder = ProjectGraphBuilder::new(context).await.unwrap(); + builder.load_all().await.unwrap(); + + let mut graph = builder.build().await.unwrap(); + graph.check_boundaries = true; + graph.get_all().unwrap(); + graph + } + + pub async fn build_graph_for<'l>( + &self, + context: ProjectGraphBuilderContext<'l>, + ids: &[&str], + ) -> ProjectGraph { + let mut builder = ProjectGraphBuilder::new(context).await.unwrap(); + + for id in ids { + builder.load(id).await.unwrap(); + } + + let mut graph = builder.build().await.unwrap(); + graph.check_boundaries = true; + + for id in ids { + graph.get(id).unwrap(); + } + + graph + } +} + +pub async fn generate_project_graph(fixture: &str) -> ProjectGraph { + generate_project_graph_from_sandbox(create_sandbox(fixture).path()).await +} + +pub async fn generate_project_graph_from_sandbox(root: &Path) -> ProjectGraph { + generate_project_graph_with_changes(root, |_| {}).await +} + +pub async fn generate_project_graph_with_changes(root: &Path, mut op: F) -> ProjectGraph +where + F: FnMut(&mut ProjectGraphContainer), +{ + let mut container = ProjectGraphContainer::new(root); + + op(&mut container); + + let context = container.create_context(); + + container.build_graph(context).await +} diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index c966243c309..6cdc6a341b4 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -10,6 +10,63 @@ - More accurately monitors signals (ctrl+c) and shutdowns. - Tasks can now be configured with a timeout. +## Unreleased + +#### 💥 Breaking + +- Tasks that depend (via `deps`) on other tasks from arbitrary projects (the parent project doesn't + implicitly or explicitly depend on the other project) will now automatically mark that other + project as a "peer" dependency. For example, "b" becomes a peer dependency for "a". + + ```yaml + tasks: + build: + deps: ['b:build'] + + # Now internally becomes: + dependsOn: + - id: 'b' + scope: 'peer' + + tasks: + build: + deps: ['b:build'] + ``` + + We're marking this as a breaking change as this could subtly introduce cycles in the project graph + that weren't present before, and for Node.js projects, this may inject `peerDependencies`. + +#### 🎉 Release + +- Rewrote the dependency graph from the ground-up: + - Now known as the action graph. + - All actions now depend on the `SyncWorkspace` action, instead of this action running + arbitrarily. + - Cleaned up dependency chains between actions, greatly reducing the number of nodes in the graph. + - Renamed `RunTarget` to `RunTask`, including interactive and persistent variants. +- Updated the action graph to iterate using a topological queue, which executes ready-to-run actions + in the thread pool. Previously, we would sort topologically _into batches_, which worked, but + resulted in many threads uselessly waiting for an action to run, which was blocked waiting for the + current batch to complete. + - For large graphs, this should result in a significant performance improvement, upwards of 10x. + - Persistent tasks will still be ran as a batch, but since it's the last operation, it's fine. +- Released a new GitHub action, + [`moonrepo/setup-toolchain`](https://github.com/marketplace/actions/setup-proto-and-moon-toolchains), + that replaces both `setup-moon-action` and `setup-proto`. + +#### 🚀 Updates + +- Added a `moon action-graph` command and deprecated `moon dep-graph`. + +#### 🐞 Fixes + +- Fixed an issue where task dependents (via `moon ci` or `moon run --dependents`) wouldn't always + locate all downstream tasks. + +#### ⚙️ Internal + +- Added in-memory caching to project graph file system lookup operations. + ## 1.14.5 #### 🐞 Fixes diff --git a/packages/report/tests/action.test.ts b/packages/report/tests/action.test.ts index c1fa88e3a07..b20ce300f99 100644 --- a/packages/report/tests/action.test.ts +++ b/packages/report/tests/action.test.ts @@ -10,7 +10,7 @@ const action: Action = { }, error: null, flaky: false, - label: 'RunTarget(app:build)', + label: 'RunTask(app:build)', nodeIndex: 8, status: 'passed', finishedAt: '2022-09-12T22:50:12.932311Z', diff --git a/packages/report/tests/report.test.ts b/packages/report/tests/report.test.ts index 023d48c7b9a..789f54cc2c6 100644 --- a/packages/report/tests/report.test.ts +++ b/packages/report/tests/report.test.ts @@ -13,7 +13,7 @@ function mockReport(): RunReport { }, error: null, flaky: false, - label: 'RunTarget(types:build)', + label: 'RunTask(types:build)', nodeIndex: 5, status: 'cached', finishedAt: '2022-09-12T22:50:12.932311Z', @@ -28,7 +28,7 @@ function mockReport(): RunReport { }, error: null, flaky: true, - label: 'RunTarget(runtime:typecheck)', + label: 'RunTask(runtime:typecheck)', nodeIndex: 4, status: 'passed', finishedAt: '2022-09-12T22:50:12.932311Z', @@ -43,7 +43,7 @@ function mockReport(): RunReport { }, error: null, flaky: false, - label: 'RunTarget(types:typecheck)', + label: 'RunTask(types:typecheck)', nodeIndex: 6, status: 'passed', finishedAt: '2022-09-12T22:50:12.932311Z', @@ -58,7 +58,7 @@ function mockReport(): RunReport { }, error: null, flaky: false, - label: 'RunTarget(website:typecheck)', + label: 'RunTask(website:typecheck)', nodeIndex: 8, status: 'passed', finishedAt: '2022-09-12T22:50:12.932311Z', @@ -102,10 +102,10 @@ describe('sortReport()', () => { sortReport(report, 'time', 'asc'); expect(report.actions.map((a) => a.label)).toEqual([ - 'RunTarget(types:build)', - 'RunTarget(website:typecheck)', - 'RunTarget(types:typecheck)', - 'RunTarget(runtime:typecheck)', + 'RunTask(types:build)', + 'RunTask(website:typecheck)', + 'RunTask(types:typecheck)', + 'RunTask(runtime:typecheck)', ]); }); @@ -114,10 +114,10 @@ describe('sortReport()', () => { sortReport(report, 'time', 'desc'); expect(report.actions.map((a) => a.label)).toEqual([ - 'RunTarget(runtime:typecheck)', - 'RunTarget(types:typecheck)', - 'RunTarget(website:typecheck)', - 'RunTarget(types:build)', + 'RunTask(runtime:typecheck)', + 'RunTask(types:typecheck)', + 'RunTask(website:typecheck)', + 'RunTask(types:build)', ]); }); @@ -126,10 +126,10 @@ describe('sortReport()', () => { sortReport(report, 'label', 'asc'); expect(report.actions.map((a) => a.label)).toEqual([ - 'RunTarget(runtime:typecheck)', - 'RunTarget(types:build)', - 'RunTarget(types:typecheck)', - 'RunTarget(website:typecheck)', + 'RunTask(runtime:typecheck)', + 'RunTask(types:build)', + 'RunTask(types:typecheck)', + 'RunTask(website:typecheck)', ]); }); @@ -138,10 +138,10 @@ describe('sortReport()', () => { sortReport(report, 'label', 'desc'); expect(report.actions.map((a) => a.label)).toEqual([ - 'RunTarget(website:typecheck)', - 'RunTarget(types:typecheck)', - 'RunTarget(types:build)', - 'RunTarget(runtime:typecheck)', + 'RunTask(website:typecheck)', + 'RunTask(types:typecheck)', + 'RunTask(types:build)', + 'RunTask(runtime:typecheck)', ]); }); }); @@ -156,7 +156,7 @@ describe('prepareReportActions()', () => { secs: 0, }, icon: '🟪', - label: 'RunTarget(types:build)', + label: 'RunTask(types:build)', status: 'cached', time: '0s', }, @@ -167,7 +167,7 @@ describe('prepareReportActions()', () => { secs: 1922, }, icon: '🟩', - label: 'RunTarget(runtime:typecheck)', + label: 'RunTask(runtime:typecheck)', status: 'passed', time: '32m 2s', }, @@ -178,7 +178,7 @@ describe('prepareReportActions()', () => { secs: 64, }, icon: '🟩', - label: 'RunTarget(types:typecheck)', + label: 'RunTask(types:typecheck)', status: 'passed', time: '1m 4s', }, @@ -189,7 +189,7 @@ describe('prepareReportActions()', () => { secs: 34, }, icon: '🟩', - label: 'RunTarget(website:typecheck)', + label: 'RunTask(website:typecheck)', status: 'passed', time: '34.4s', }, diff --git a/packages/visualizer/src/helpers/render.ts b/packages/visualizer/src/helpers/render.ts index f755c987a17..f57f42dd8f8 100644 --- a/packages/visualizer/src/helpers/render.ts +++ b/packages/visualizer/src/helpers/render.ts @@ -9,8 +9,8 @@ function getActionType(label: string) { return 'sync-workspace'; } - if (label.startsWith('RunTarget') || label.startsWith('RunPersistentTarget')) { - return 'run-target'; + if (label.startsWith('Run') && (label.includes('Target') || label.includes('Task'))) { + return 'run-task'; } if (label.startsWith('Sync') && label.includes('Project')) { @@ -105,7 +105,7 @@ export function render(element: HTMLElement, data: GraphInfo) { }, }, { - selector: 'node[type="run-target"]', + selector: 'node[type="run-task"]', style: { // @ts-expect-error Types incorrect 'background-gradient-stop-colors': '#6e58d1 #4a2ec6 #3b259e', diff --git a/website/docs/commands/dep-graph.mdx b/website/docs/commands/dep-graph.mdx index f916862a1df..7ed5439c1eb 100644 --- a/website/docs/commands/dep-graph.mdx +++ b/website/docs/commands/dep-graph.mdx @@ -34,7 +34,7 @@ digraph { 0 [ label="SetupNodeTool" style=filled, shape=oval, fillcolor=black, fontcolor=white] 1 [ label="InstallNodeDeps" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 2 [ label="SyncNodeProject(node)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="RunTarget(node:standard)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] + 3 [ label="RunTask(node:standard)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] 1 -> 0 [ arrowhead=box, arrowtail=box] 2 -> 0 [ arrowhead=box, arrowtail=box] 3 -> 1 [ arrowhead=box, arrowtail=box] diff --git a/website/docs/concepts/project.mdx b/website/docs/concepts/project.mdx index f49c31d5423..6dccd3206de 100644 --- a/website/docs/concepts/project.mdx +++ b/website/docs/concepts/project.mdx @@ -36,7 +36,7 @@ pattern that makes the most sense for your company or team! ## Dependencies Projects can depend on other projects within the [workspace](./workspace) to build a -[project graph](../how-it-works/dep-graph), and in turn, a dependency graph for executing +[project graph](../how-it-works/action-graph), and in turn, an action graph for executing [tasks](./task). Project dependencies are divided into 2 categories: - **Explicit dependencies** - These are dependencies that are explicitly defined in a project's diff --git a/website/docs/concepts/task.mdx b/website/docs/concepts/task.mdx index 74e4637fb7d..94449e36b45 100644 --- a/website/docs/concepts/task.mdx +++ b/website/docs/concepts/task.mdx @@ -30,7 +30,7 @@ Tasks are grouped into 1 of the following types based on their configured parame ## Modes Alongside types, tasks can also grouped into a special mode that provides unique handling within the -dependency graph and pipelines. +action graph and pipelines. ### Local only @@ -52,8 +52,8 @@ tasks: Tasks that need to interact with the user via terminal prompts are known as interactive tasks. Because interactive tasks require stdin, and it's not possible to have multiple parallel running -tasks interact with stdin, we isolate interactive tasks from other tasks in the dependency graph. -This ensures that only 1 interactive task is ran at a time. +tasks interact with stdin, we isolate interactive tasks from other tasks in the action graph. This +ensures that only 1 interactive task is ran at a time. To mark a task as interactive, enable the [`options.interactive`](../config/project#interactive) setting. @@ -72,8 +72,8 @@ Tasks that never complete, like servers and watchers, are known as persistent ta tasks are typically problematic when it comes to dependency graphs, because if they run in the middle of the graph, subsequent tasks will never run because the persistent task never completes! -However in moon, this is a non-issue, as we collect all persistent tasks within the dependency graph -and run them _last as a batch_. This is perfect for a few reasons: +However in moon, this is a non-issue, as we collect all persistent tasks within the action graph and +run them _last as a batch_. This is perfect for a few reasons: - All persistent tasks are ran in parallel, so they don't block each other. - Running both the backend API and frontend webapp in parallel is a breeze. diff --git a/website/docs/config/project.mdx b/website/docs/config/project.mdx index c34f5b84f06..3b55865fa2f 100644 --- a/website/docs/config/project.mdx +++ b/website/docs/config/project.mdx @@ -830,8 +830,8 @@ tasks: Marks the task as persistent (continuously running). [Persistent tasks](../concepts/task#persistent) -are handled differently than non-persistent tasks in the dependency graph. When running a target, -all persistent tasks are _ran last_ and _in parallel_, after all their dependencies have completed. +are handled differently than non-persistent tasks in the action graph. When running a target, all +persistent tasks are _ran last_ and _in parallel_, after all their dependencies have completed. This is extremely useful for running a server (or a watcher) in the background while other tasks are running. diff --git a/website/docs/editors/vscode.mdx b/website/docs/editors/vscode.mdx index 94cbcb22e9e..80acb0987d4 100644 --- a/website/docs/editors/vscode.mdx +++ b/website/docs/editors/vscode.mdx @@ -59,7 +59,7 @@ Information about the last ran target will be displayed in a beautiful table wit Only tasks ran from the [projects view](#projects) or on the command line will be displayed here. This table displays all actions that were ran alongside the running primary target(s). They are -ordered topologically via the dependency graph. +ordered topologically via the action graph. diff --git a/website/docs/how-it-works/dep-graph.mdx b/website/docs/how-it-works/action-graph.mdx similarity index 68% rename from website/docs/how-it-works/dep-graph.mdx rename to website/docs/how-it-works/action-graph.mdx index a7803c5ccb5..3c9cb38d2ea 100644 --- a/website/docs/how-it-works/dep-graph.mdx +++ b/website/docs/how-it-works/action-graph.mdx @@ -1,18 +1,17 @@ --- -title: Dependency graph +title: Action graph --- -import DepGraph from '@site/src/components/Docs/DepGraph'; +import ActionGraph from '@site/src/components/Docs/ActionGraph'; -When you run a [task](../config/project#tasks-1) on the command line, we generate a dependency graph -to ensure [dependencies](../config/project#deps) of tasks have ran before running run the primary -task. +When you run a [task](../config/project#tasks-1) on the command line, we generate an action graph to +ensure [dependencies](../config/project#deps) of tasks have ran before running run the primary task. -The dependency graph is a representation of all [tasks](../concepts/task), derived from the +The action graph is a representation of all [tasks](../concepts/task), derived from the [project graph](./project-graph), and is also represented internally as a directed acyclic graph (DAG). - + ## Actions @@ -21,7 +20,7 @@ represent each node in the graph as an action to perform. This allows us to be m efficient with how we run tasks, and allows us to provide more functionality and automation than other runners. -The following actions compose our dependency graph: +The following actions compose our action graph: ### Sync workspace @@ -36,12 +35,12 @@ tier 3 language into the toolchain. For other tiers, this is basically a no-oper - When the tool has already been installed, this action will be skipped. - Actions will be scoped by language and version, also known as a runtime. For example, `SetupNodeTool(18.1.0)` or `SetupDenoTool(1.31.0)`. -- Tools that require a global language binary will display the version as "global". For example, - `SetupNodeTool(global)`. +- Tools that require a global binary (found on `PATH`) will display the version as "global". For + example, `SetupNodeTool(global)`. ### Install dependencies -Before we run a target, we ensure that all language dependencies (`node_modules` for example) have +Before we run a task, we ensure that all language dependencies (`node_modules` for example) have been installed, by automatically installing them if we detect changes since the last run. We achieve this by comparing lockfile modified timestamps, parsing manifest files, and hashing resolved dependency versions. @@ -58,7 +57,7 @@ dependency versions. ### Sync project To ensure a consistently healthy project and repository, we run a process known as syncing -_everytime_ a target is ran. Actions will be scoped by language, for example, +_everytime_ a task is ran. Actions will be scoped by language, for example, `SyncNodeProject(example)`. What is synced or considered healthcare is dependent on the language and its ecosystem. @@ -74,26 +73,27 @@ What is synced or considered healthcare is dependent on the language and its eco > This action depends on the setup toolchain action, in case it requires binaries or functionality > that the toolchain provides. -### Run target +### Run task -The primary action in the graph is the run [target](../concepts/target) action, which runs a -project's task as a child process. Tasks can depend on other tasks, and they'll be effectively -orchestrated and executed by running in topological order _and_ in batches via a worker pool. +The primary action in the graph is the run [task](../concepts/task) action, which runs a project's +task as a child process, derived from a [target](../concepts/target). Tasks can depend on other +tasks, and they'll be effectively orchestrated and executed by running in topological order using a +thread pool. > This action depends on the previous actions, as the toolchain is used for running the task's > command, and the outcome of the task is best when the project state is healthy and deterministic. -### Run interactive target +### Run interactive task -Like the base run target, but runs the [task interactively](../concepts/task#interactive) with stdin +Like the base run task, but runs the [task interactively](../concepts/task#interactive) with stdin capabilities. All interactive tasks are run in isolation in the graph. -### Run persistent target +### Run persistent task -Like the base run target, but runs the [task in a persistent process](../concepts/task#persistent) +Like the base run task, but runs the [task in a persistent process](../concepts/task#persistent) that never exits. All persistent tasks are run in parallel as the last batch in the graph. ## What is the graph used for? -Without the dependency graph, tasks would not efficiently, or possibly at all! The graph helps to -run tasks in parallel, in the correct order, and to ensure a reliable outcome. +Without the action graph, tasks would not efficiently, or possibly at all! The graph helps to run +tasks in parallel, in the correct order, and to ensure a reliable outcome. diff --git a/website/docs/how-it-works/project-graph.mdx b/website/docs/how-it-works/project-graph.mdx index fddb4ca2814..1db3ddef951 100644 --- a/website/docs/how-it-works/project-graph.mdx +++ b/website/docs/how-it-works/project-graph.mdx @@ -82,7 +82,7 @@ integration. Great question, the project graph is used throughout the codebase to accomplish a variety of functions, but mainly: -- Is fed into the [dependency graph](./dep-graph) to determine relationships of tasks between other +- Is fed into the [action graph](./action-graph) to determine relationships of tasks between other tasks, and across projects. - Powers our [Docker](../guides/docker) layer caching and scaffolding implementations. - Utilized for [project syncing](../commands/sync) to ensure a healthy repository state. diff --git a/website/docs/run-task.mdx b/website/docs/run-task.mdx index df4cbd5b2fb..48f72e2e5ef 100644 --- a/website/docs/run-task.mdx +++ b/website/docs/run-task.mdx @@ -23,7 +23,7 @@ $ moon app:build When this command is ran, it will do the following: -- Generate a directed acyclic graph, known as the dependency graph. +- Generate a directed acyclic graph, known as the action (dependency) graph. - Insert [`deps`](./config/project#deps) as targets into the graph. - Insert the primary target into the graph. - Run all tasks in the graph in parallel and in topological order (the dependency chain). diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 7eb8d9187b4..aa683167fb3 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -211,6 +211,10 @@ const config = { '@docusaurus/plugin-client-redirects', { redirects: [ + { + from: '/docs/how-it-works/dep-graph', + to: '/docs/how-it-works/action-graph', + }, { from: '/docs/config/global-project', to: '/docs/config/tasks', diff --git a/website/sidebars.js b/website/sidebars.js index 3ead0dd75f5..d9cfca902b6 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -10,7 +10,7 @@ const sidebars = { label: 'How it works', collapsed: true, collapsible: true, - items: ['how-it-works/languages', 'how-it-works/project-graph', 'how-it-works/dep-graph'], + items: ['how-it-works/languages', 'how-it-works/project-graph', 'how-it-works/action-graph'], link: { type: 'generated-index', title: 'How it works', diff --git a/website/src/components/Docs/DepGraph.tsx b/website/src/components/Docs/ActionGraph.tsx similarity index 87% rename from website/src/components/Docs/DepGraph.tsx rename to website/src/components/Docs/ActionGraph.tsx index 84b701602d8..a8a1e52764b 100644 --- a/website/src/components/Docs/DepGraph.tsx +++ b/website/src/components/Docs/ActionGraph.tsx @@ -1,13 +1,25 @@ import React, { useEffect, useRef } from 'react'; import { renderGraph } from '../../utils/renderGraph'; -export default function DepGraph() { +export default function ActionGraph() { const graphRef = useRef(null); useEffect(() => { if (graphRef.current) { renderGraph(graphRef.current, { edges: [ + { + data: { + source: 'sync-workspace', + target: 'node-tool', + }, + }, + { + data: { + source: 'sync-workspace', + target: 'system-tool', + }, + }, { data: { source: 'node-tool', @@ -125,21 +137,21 @@ export default function DepGraph() { { data: { id: 'target-clean', - label: 'RunTarget(example:clean)', + label: 'RunTask(example:clean)', type: 'sm', }, }, { data: { id: 'target-build', - label: 'RunTarget(example:build)', + label: 'RunTask(example:build)', type: 'sm', }, }, { data: { id: 'target-package', - label: 'RunTarget(example:package)', + label: 'RunTask(example:package)', type: 'sm', }, },