From 03a2c7339ef3dd9e21d4a909ed262c748a71a9a2 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Sat, 7 Oct 2023 22:19:14 -0700 Subject: [PATCH] new: Migrate pipeline to new action graph. (#1104) * Update moon. * Update impl. * Update pipeline. * Update commands. * Remove dep graph. * Get running. * Redo graph iterator. * Get pipeline working. * Get tests working. * Fix persistent. * Renable test. * Renable test. * Change method. * Use a channel. * Fix dep chain. * Remove log. --- Cargo.lock | 35 +- crates/cli/Cargo.toml | 2 +- crates/cli/src/commands/ci.rs | 33 +- crates/cli/src/commands/docker/setup.rs | 10 +- crates/cli/src/commands/graph/action.rs | 19 +- crates/cli/src/commands/graph/utils.rs | 6 +- crates/cli/src/commands/run.rs | 51 +- crates/cli/src/commands/syncs/projects.rs | 10 +- crates/cli/tests/action_graph_test.rs | 11 +- ...ph__aliases__can_focus_using_an_alias.snap | 28 +- ...liases__resolves_aliases_in_task_deps.snap | 47 +- ...test__action_graph__focused_by_target.snap | 19 +- ..._action_graph__focused_by_task_in_cwd.snap | 19 +- ...h__includes_dependencies_when_focused.snap | 27 +- ...h__includes_dependents_when_focused-2.snap | 83 +++ ...aph__includes_dependents_when_focused.snap | 36 +- ...raph_test__action_graph__outputs_json.snap | 2 +- ...n_test__errors_for_cycle_in_task_deps.snap | 2 +- crates/core/action-pipeline/Cargo.toml | 9 +- .../benches/pipeline_benchmark.rs | 63 -- crates/core/action-pipeline/src/estimator.rs | 8 +- crates/core/action-pipeline/src/pipeline.rs | 336 +++++---- crates/core/action-pipeline/src/processor.rs | 25 +- .../src/subscribers/moonbase.rs | 10 +- .../action-pipeline/tests/estimator_test.rs | 69 +- crates/core/action/Cargo.toml | 1 + crates/core/action/src/action.rs | 6 +- crates/core/action/src/lib.rs | 3 +- crates/core/action/src/node.rs | 83 --- crates/core/dep-graph/Cargo.toml | 37 - .../dep-graph/benches/dep_graph_benchmark.rs | 49 -- crates/core/dep-graph/src/dep_builder.rs | 474 ------------ crates/core/dep-graph/src/dep_graph.rs | 215 ------ crates/core/dep-graph/src/errors.rs | 12 - crates/core/dep-graph/src/lib.rs | 7 - crates/core/dep-graph/tests/dep_graph_test.rs | 679 ------------------ ..._deps__tool_is_based_on_task_platform.snap | 22 - ...test__run_target__avoids_dupe_targets.snap | 16 - ...h_test__run_target__deps_chain_target.snap | 44 -- ...un_target__isolates_interactive_tasks.snap | 45 -- ...n_target__moves_persistent_tasks_last.snap | 54 -- ...uns_all_projects_for_target_all_scope.snap | 38 - ...raph_test__run_target__single_targets.snap | 19 - ...f_touched__skips_if_untouched_project.snap | 16 - ...t_if_touched__skips_if_untouched_task.snap | 21 - ...t__sync_project__avoids_dupe_projects.snap | 11 - ...test__sync_project__isolated_projects.snap | 19 - ...st__sync_project__projects_with_tasks.snap | 14 - crates/core/moon/Cargo.toml | 2 +- crates/core/moon/src/lib.rs | 6 +- nextgen/action-graph/Cargo.toml | 3 +- nextgen/action-graph/src/action_graph.rs | 165 +++-- .../action-graph/src/action_graph_builder.rs | 35 +- nextgen/action-graph/src/action_node.rs | 3 +- .../action-graph/tests/action_graph_test.rs | 83 +-- packages/cli/CHANGELOG.md | 11 +- website/docs/guides/webhooks.mdx | 3 +- 57 files changed, 722 insertions(+), 2434 deletions(-) create mode 100644 crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependents_when_focused-2.snap delete mode 100644 crates/core/action-pipeline/benches/pipeline_benchmark.rs delete mode 100644 crates/core/action/src/node.rs delete mode 100644 crates/core/dep-graph/Cargo.toml delete mode 100644 crates/core/dep-graph/benches/dep_graph_benchmark.rs delete mode 100644 crates/core/dep-graph/src/dep_builder.rs delete mode 100644 crates/core/dep-graph/src/dep_graph.rs delete mode 100644 crates/core/dep-graph/src/errors.rs delete mode 100644 crates/core/dep-graph/src/lib.rs delete mode 100644 crates/core/dep-graph/tests/dep_graph_test.rs delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__installs_deps__tool_is_based_on_task_platform.snap delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__avoids_dupe_targets.snap delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__deps_chain_target.snap delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__isolates_interactive_tasks.snap delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__runs_all_projects_for_target_all_scope.snap delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__single_targets.snap delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target_if_touched__skips_if_untouched_project.snap delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target_if_touched__skips_if_untouched_task.snap delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__avoids_dupe_projects.snap delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__isolated_projects.snap delete mode 100644 crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__projects_with_tasks.snap diff --git a/Cargo.lock b/Cargo.lock index fb827287c7f..c155cc9b881 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2987,9 +2987,9 @@ name = "moon" version = "0.1.0" dependencies = [ "miette", + "moon_action_graph", "moon_config", "moon_deno_platform", - "moon_dep_graph", "moon_node_platform", "moon_platform", "moon_platform_detector", @@ -3008,6 +3008,7 @@ name = "moon_action" version = "0.1.0" dependencies = [ "miette", + "moon_action_graph", "moon_common", "moon_platform_runtime", "moon_target", @@ -3042,6 +3043,7 @@ dependencies = [ "moon_test_utils2", "petgraph", "rustc-hash", + "serde", "starbase_sandbox", "thiserror", "tokio", @@ -3059,10 +3061,10 @@ dependencies = [ "moon", "moon_action", "moon_action_context", + "moon_action_graph", "moon_actions", "moon_api", "moon_cache_item", - "moon_dep_graph", "moon_emitter", "moon_hash", "moon_logger", @@ -3216,6 +3218,7 @@ dependencies = [ "mimalloc", "moon", "moon_action_context", + "moon_action_graph", "moon_action_pipeline", "moon_actions", "moon_api", @@ -3223,7 +3226,6 @@ dependencies = [ "moon_codegen", "moon_common", "moon_config", - "moon_dep_graph", "moon_lang", "moon_node_lang", "moon_node_platform", @@ -3389,33 +3391,6 @@ dependencies = [ "rustc-hash", ] -[[package]] -name = "moon_dep_graph" -version = "0.1.0" -dependencies = [ - "criterion", - "miette", - "moon", - "moon_action", - "moon_common", - "moon_config", - "moon_logger", - "moon_platform", - "moon_project", - "moon_project_graph", - "moon_query", - "moon_target", - "moon_task", - "moon_test_utils", - "moon_utils", - "moon_workspace", - "petgraph", - "rustc-hash", - "starbase_styles", - "thiserror", - "tokio", -] - [[package]] name = "moon_emitter" version = "0.1.0" diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 9faaea65203..a3f77f1fbe6 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -16,6 +16,7 @@ crate-type = ["rlib"] [dependencies] moon = { path = "../core/moon" } moon_action_context = { path = "../core/action-context" } +moon_action_graph = { path = "../../nextgen/action-graph" } moon_action_pipeline = { path = "../core/action-pipeline" } moon_actions = { path = "../core/actions" } moon_api = { path = "../../nextgen/api" } @@ -23,7 +24,6 @@ moon_cache = { path = "../../nextgen/cache" } moon_codegen = { path = "../../nextgen/codegen" } moon_common = { path = "../../nextgen/common" } moon_config = { path = "../../nextgen/config" } -moon_dep_graph = { path = "../core/dep-graph" } moon_lang = { path = "../core/lang" } moon_node_lang = { path = "../node/lang" } moon_node_platform = { path = "../node/platform" } diff --git a/crates/cli/src/commands/ci.rs b/crates/cli/src/commands/ci.rs index 05ddd85a683..b9d6de486f9 100644 --- a/crates/cli/src/commands/ci.rs +++ b/crates/cli/src/commands/ci.rs @@ -3,11 +3,11 @@ use crate::queries::touched_files::{query_touched_files, QueryTouchedFilesOption use ci_env::CiOutput; use clap::Args; use itertools::Itertools; -use moon::{build_dep_graph, generate_project_graph}; +use moon::{build_action_graph, generate_project_graph}; use moon_action_context::ActionContext; +use moon_action_graph::ActionGraph; use moon_action_pipeline::Pipeline; use moon_common::path::WorkspaceRelativePathBuf; -use moon_dep_graph::DepGraph; use moon_project_graph::ProjectGraph; use moon_target::Target; use moon_terminal::safe_exit; @@ -162,31 +162,31 @@ fn distribute_targets_across_jobs( } /// Generate a dependency graph with the runnable targets. -fn generate_dep_graph( +fn generate_action_graph( provider: &CiOutput, project_graph: &ProjectGraph, targets: &TargetList, touched_files: &FxHashSet, -) -> AppResult { - print_header(provider, "Generating dependency graph"); +) -> AppResult { + print_header(provider, "Generating action graph"); - let mut dep_builder = build_dep_graph(project_graph); + let mut action_graph_builder = build_action_graph(project_graph)?; + + // Run dependents to ensure consumers still work correctly + action_graph_builder.include_dependents(); for target in targets { // Run the target and its dependencies - dep_builder.run_target(target, Some(touched_files))?; - - // And also run its dependents to ensure consumers still work correctly - dep_builder.run_dependents_for_target(target)?; + action_graph_builder.run_task_by_target(target, Some(touched_files))?; } - let dep_graph = dep_builder.build(); + let action_graph = action_graph_builder.build()?; println!("Target count: {}", targets.len()); - println!("Action count: {}", dep_graph.get_node_count()); + println!("Action count: {}", action_graph.get_node_count()); print_footer(provider); - Ok(dep_graph) + Ok(action_graph) } #[system] @@ -210,9 +210,10 @@ pub async fn ci( } let targets = distribute_targets_across_jobs(&ci_provider, args, targets); - let dep_graph = generate_dep_graph(&ci_provider, &project_graph, &targets, &touched_files)?; + let action_graph = + generate_action_graph(&ci_provider, &project_graph, &targets, &touched_files)?; - if dep_graph.is_empty() { + if action_graph.is_empty() { println!( "{}", color::invalid("No targets to run based on touched files") @@ -239,7 +240,7 @@ pub async fn ci( let results = pipeline .generate_report("ciReport.json") - .run(dep_graph, Some(context)) + .run(action_graph, Some(context)) .await?; print_footer(&ci_provider); diff --git a/crates/cli/src/commands/docker/setup.rs b/crates/cli/src/commands/docker/setup.rs index f9bf319ff12..c938e66e5f3 100644 --- a/crates/cli/src/commands/docker/setup.rs +++ b/crates/cli/src/commands/docker/setup.rs @@ -1,6 +1,6 @@ use super::MANIFEST_NAME; use crate::commands::docker::scaffold::DockerManifest; -use moon::{build_dep_graph, generate_project_graph}; +use moon::{build_action_graph, generate_project_graph}; use moon_action_pipeline::Pipeline; use moon_terminal::safe_exit; use moon_workspace::Workspace; @@ -18,17 +18,17 @@ pub async fn setup(workspace: ResourceMut) { let manifest: DockerManifest = json::read_file(manifest_path)?; let project_graph = generate_project_graph(workspace).await?; - let mut dep_builder = build_dep_graph(&project_graph); + let mut action_graph_builder = build_action_graph(&project_graph)?; for project_id in &manifest.focused_projects { let project = project_graph.get(project_id)?; - dep_builder.install_deps(&project, None)?; + action_graph_builder.install_deps(&project, None)?; } - let dep_graph = dep_builder.build(); + let action_graph = action_graph_builder.build()?; Pipeline::new(workspace.to_owned(), project_graph) - .run(dep_graph, None) + .run(action_graph, None) .await?; } diff --git a/crates/cli/src/commands/graph/action.rs b/crates/cli/src/commands/graph/action.rs index 7fe2778b092..9ece3882321 100644 --- a/crates/cli/src/commands/graph/action.rs +++ b/crates/cli/src/commands/graph/action.rs @@ -1,7 +1,7 @@ use crate::commands::graph::utils::{action_graph_repr, respond_to_request, setup_server}; use clap::Args; use miette::IntoDiagnostic; -use moon::{build_dep_graph, generate_project_graph}; +use moon::{build_action_graph, generate_project_graph}; use moon_target::TargetLocator; use moon_workspace::Workspace; use starbase::{system, SystemResult}; @@ -11,6 +11,9 @@ pub struct ActionGraphArgs { #[arg(help = "Target to *only* graph")] target: Option, + #[arg(long, help = "Include dependents of the focused target")] + dependents: bool, + #[arg(long, help = "Print the graph in DOT format")] dot: bool, @@ -23,24 +26,26 @@ pub async fn internal_action_graph( workspace: &mut Workspace, ) -> SystemResult { let project_graph = generate_project_graph(workspace).await?; - let mut action_graph_builder = build_dep_graph(&project_graph); + let mut action_graph_builder = build_action_graph(&project_graph)?; // Focus a target and its dependencies/dependents if let Some(locator) = args.target.clone() { - for target in action_graph_builder.run_targets_by_locator(&[locator], None)? { - action_graph_builder.run_dependents_for_target(&target)?; + if args.dependents { + action_graph_builder.include_dependents(); } + action_graph_builder.run_task_by_target_locator(locator, None)?; + // Show all targets and actions } else { - for project in project_graph.get_all_unexpanded() { + for project in project_graph.get_all()? { for task in project.tasks.values() { - action_graph_builder.run_target(&task.target, None)?; + action_graph_builder.run_task(&project, task, None)?; } } } - let action_graph = action_graph_builder.build(); + let action_graph = action_graph_builder.build()?; if args.dot { println!("{}", action_graph.to_dot()); diff --git a/crates/cli/src/commands/graph/utils.rs b/crates/cli/src/commands/graph/utils.rs index ae858905abd..c74bbf2e9cb 100644 --- a/crates/cli/src/commands/graph/utils.rs +++ b/crates/cli/src/commands/graph/utils.rs @@ -1,6 +1,6 @@ use super::dto::{GraphEdgeDto, GraphInfoDto, GraphNodeDto}; use miette::IntoDiagnostic; -use moon_dep_graph::DepGraph; +use moon_action_graph::ActionGraph; use moon_project_graph::ProjectGraph; use petgraph::{graph::NodeIndex, Graph}; use rustc_hash::FxHashMap; @@ -84,8 +84,8 @@ pub async fn project_graph_repr(project_graph: &ProjectGraph) -> GraphInfoDto { } /// Get a serialized representation of the dependency graph. -pub async fn action_graph_repr(dep_graph: &DepGraph) -> GraphInfoDto { - let labeled_graph = dep_graph.labeled_graph(); +pub async fn action_graph_repr(action_graph: &ActionGraph) -> GraphInfoDto { + let labeled_graph = action_graph.labeled_graph(); extract_nodes_and_edges_from_graph(&labeled_graph, false) } diff --git a/crates/cli/src/commands/run.rs b/crates/cli/src/commands/run.rs index 17e344a8429..380a42ffccb 100644 --- a/crates/cli/src/commands/run.rs +++ b/crates/cli/src/commands/run.rs @@ -4,7 +4,7 @@ use crate::helpers::map_list; use crate::queries::touched_files::{query_touched_files, QueryTouchedFilesOptions}; use clap::Args; use miette::miette; -use moon::{build_dep_graph, generate_project_graph}; +use moon::{build_action_graph, generate_project_graph}; use moon_action_context::{ActionContext, ProfileType}; use moon_action_pipeline::Pipeline; use moon_common::is_test_env; @@ -26,10 +26,7 @@ pub struct RunArgs { #[arg(required = true, help = "List of targets to run")] pub targets: Vec, - #[arg( - long, - help = "Run dependents of the primary targets, as well as dependencies" - )] + #[arg(long, help = "Run dependents of the primary targets")] pub dependents: bool, #[arg( @@ -136,21 +133,34 @@ pub async fn run_target( }; // Generate a dependency graph for all the targets that need to be ran - let mut dep_builder = build_dep_graph(&project_graph); + let mut action_graph_builder = build_action_graph(&project_graph)?; + + // Run dependents for all primary targets + if args.dependents { + action_graph_builder.include_dependents(); + } if let Some(query_input) = &args.query { - dep_builder.set_query(query_input)?; + action_graph_builder.set_query(query_input)?; } // Run targets, optionally based on affected files - let primary_targets = dep_builder.run_targets_by_locator( - target_locators, - if should_run_affected { - Some(&touched_files) - } else { - None - }, - )?; + let mut primary_targets = vec![]; + + for locator in target_locators { + primary_targets.extend( + action_graph_builder + .run_task_by_target_locator( + locator, + if should_run_affected { + Some(&touched_files) + } else { + None + }, + )? + .0, + ); + } if primary_targets.is_empty() { let targets_list = map_list(target_locators, |id| color::label(id)); @@ -183,13 +193,6 @@ pub async fn run_target( )); } - // Run dependents for all primary targets - if args.dependents { - for target in &primary_targets { - dep_builder.run_dependents_for_target(target)?; - } - } - // Process all tasks in the graph let context = ActionContext { affected_only: should_run_affected, @@ -203,7 +206,7 @@ pub async fn run_target( ..ActionContext::default() }; - let dep_graph = dep_builder.build(); + let action_graph = action_graph_builder.build()?; let mut pipeline = Pipeline::new(workspace.to_owned(), project_graph); if let Some(concurrency) = concurrency { @@ -213,7 +216,7 @@ pub async fn run_target( let results = pipeline .bail_on_error() .generate_report("runReport.json") - .run(dep_graph, Some(context)) + .run(action_graph, Some(context)) .await?; pipeline.render_stats(&results, true)?; diff --git a/crates/cli/src/commands/syncs/projects.rs b/crates/cli/src/commands/syncs/projects.rs index a091e2952d8..910f7bd8339 100644 --- a/crates/cli/src/commands/syncs/projects.rs +++ b/crates/cli/src/commands/syncs/projects.rs @@ -1,5 +1,5 @@ use crate::helpers::create_progress_bar; -use moon::{build_dep_graph, generate_project_graph}; +use moon::{build_action_graph, generate_project_graph}; use moon_action_pipeline::Pipeline; use moon_workspace::Workspace; use starbase::{system, SystemResult}; @@ -10,17 +10,17 @@ pub async fn internal_sync(workspace: &mut Workspace) -> SystemResult { let project_graph = generate_project_graph(workspace).await?; let mut project_count = 0; - let mut dep_builder = build_dep_graph(&project_graph); + let mut action_graph_builder = build_action_graph(&project_graph)?; for project in project_graph.get_all_unexpanded() { - dep_builder.sync_project(project)?; + action_graph_builder.sync_project(project)?; project_count += 1; } - let dep_graph = dep_builder.build(); + let action_graph = action_graph_builder.build()?; let mut pipeline = Pipeline::new(workspace.to_owned(), project_graph); - let results = pipeline.run(dep_graph, None).await?; + let results = pipeline.run(action_graph, None).await?; done( format!("Successfully synced {project_count} projects"), diff --git a/crates/cli/tests/action_graph_test.rs b/crates/cli/tests/action_graph_test.rs index 897032ae97d..8e643c07735 100644 --- a/crates/cli/tests/action_graph_test.rs +++ b/crates/cli/tests/action_graph_test.rs @@ -24,7 +24,7 @@ mod action_graph { let dot = assert.output(); // Snapshot is not deterministic - assert_eq!(dot.split('\n').count(), 568); + assert_eq!(dot.split('\n').count(), 450); } #[test] @@ -98,6 +98,15 @@ mod action_graph { }); assert_snapshot!(assert.output()); + + let assert = sandbox.run_moon(|cmd| { + cmd.arg("action-graph") + .arg("--dot") + .arg("--dependents") + .arg("basic:build"); + }); + + assert_snapshot!(assert.output()); } #[test] diff --git a/crates/cli/tests/snapshots/action_graph_test__action_graph__aliases__can_focus_using_an_alias.snap b/crates/cli/tests/snapshots/action_graph_test__action_graph__aliases__can_focus_using_an_alias.snap index ca6462de586..c29a6676117 100644 --- a/crates/cli/tests/snapshots/action_graph_test__action_graph__aliases__can_focus_using_an_alias.snap +++ b/crates/cli/tests/snapshots/action_graph_test__action_graph__aliases__can_focus_using_an_alias.snap @@ -3,24 +3,16 @@ source: crates/cli/tests/action_graph_test.rs expression: assert.output() --- digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(nodeNameScope)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunTarget(nodeNameScope:test)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="SyncNodeProject(node)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 6 [ label="SyncNodeProject(nodeNameOnly)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 7 [ label="RunTarget(node:test)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] - 5 -> 1 [ arrowhead=box, arrowtail=box] - 5 -> 3 [ arrowhead=box, arrowtail=box] - 6 -> 1 [ arrowhead=box, arrowtail=box] - 5 -> 6 [ arrowhead=box, arrowtail=box] - 7 -> 2 [ arrowhead=box, arrowtail=box] - 7 -> 5 [ arrowhead=box, arrowtail=box] + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(18.0.0)" ] + 2 [ label="InstallNodeDeps(18.0.0)" ] + 3 [ label="SyncNodeProject(nodeNameScope)" ] + 4 [ label="RunTask(nodeNameScope:test)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 1 [ ] + 4 -> 2 [ ] + 4 -> 3 [ ] } diff --git a/crates/cli/tests/snapshots/action_graph_test__action_graph__aliases__resolves_aliases_in_task_deps.snap b/crates/cli/tests/snapshots/action_graph_test__action_graph__aliases__resolves_aliases_in_task_deps.snap index 48d5317a03a..4c87d471b7b 100644 --- a/crates/cli/tests/snapshots/action_graph_test__action_graph__aliases__resolves_aliases_in_task_deps.snap +++ b/crates/cli/tests/snapshots/action_graph_test__action_graph__aliases__resolves_aliases_in_task_deps.snap @@ -3,29 +3,30 @@ source: crates/cli/tests/action_graph_test.rs expression: assert.output() --- digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(node)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="SyncNodeProject(nodeNameScope)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="SyncNodeProject(nodeNameOnly)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 6 [ label="RunTarget(node:aliasDeps)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 7 [ label="RunTarget(nodeNameScope:standard)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 8 [ label="RunTarget(nodeNameOnly:standard)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 4 [ arrowhead=box, arrowtail=box] - 5 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 5 [ arrowhead=box, arrowtail=box] - 6 -> 2 [ arrowhead=box, arrowtail=box] - 6 -> 3 [ arrowhead=box, arrowtail=box] - 7 -> 2 [ arrowhead=box, arrowtail=box] - 7 -> 4 [ arrowhead=box, arrowtail=box] - 8 -> 2 [ arrowhead=box, arrowtail=box] - 8 -> 5 [ arrowhead=box, arrowtail=box] - 6 -> 7 [ arrowhead=box, arrowtail=box] - 6 -> 8 [ arrowhead=box, arrowtail=box] + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(18.0.0)" ] + 2 [ label="InstallNodeDeps(18.0.0)" ] + 3 [ label="SyncNodeProject(node)" ] + 4 [ label="SyncNodeProject(nodeNameScope)" ] + 5 [ label="SyncNodeProject(nodeNameOnly)" ] + 6 [ label="RunTask(node:aliasDeps)" ] + 7 [ label="RunTask(nodeNameScope:standard)" ] + 8 [ label="RunTask(nodeNameOnly:standard)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 4 -> 1 [ ] + 5 -> 1 [ ] + 3 -> 1 [ ] + 3 -> 4 [ ] + 3 -> 5 [ ] + 7 -> 2 [ ] + 7 -> 4 [ ] + 8 -> 2 [ ] + 8 -> 5 [ ] + 6 -> 2 [ ] + 6 -> 3 [ ] + 6 -> 7 [ ] + 6 -> 8 [ ] } diff --git a/crates/cli/tests/snapshots/action_graph_test__action_graph__focused_by_target.snap b/crates/cli/tests/snapshots/action_graph_test__action_graph__focused_by_target.snap index 83966a2ce42..d3ee89ce3a8 100644 --- a/crates/cli/tests/snapshots/action_graph_test__action_graph__focused_by_target.snap +++ b/crates/cli/tests/snapshots/action_graph_test__action_graph__focused_by_target.snap @@ -3,15 +3,16 @@ source: crates/cli/tests/action_graph_test.rs expression: assert.output() --- digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(basic)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunTarget(basic:lint)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(18.0.0)" ] + 2 [ label="InstallNodeDeps(18.0.0)" ] + 3 [ label="SyncNodeProject(basic)" ] + 4 [ label="RunTask(basic:lint)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 1 [ ] + 4 -> 2 [ ] + 4 -> 3 [ ] } diff --git a/crates/cli/tests/snapshots/action_graph_test__action_graph__focused_by_task_in_cwd.snap b/crates/cli/tests/snapshots/action_graph_test__action_graph__focused_by_task_in_cwd.snap index 83966a2ce42..d3ee89ce3a8 100644 --- a/crates/cli/tests/snapshots/action_graph_test__action_graph__focused_by_task_in_cwd.snap +++ b/crates/cli/tests/snapshots/action_graph_test__action_graph__focused_by_task_in_cwd.snap @@ -3,15 +3,16 @@ source: crates/cli/tests/action_graph_test.rs expression: assert.output() --- digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(basic)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunTarget(basic:lint)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(18.0.0)" ] + 2 [ label="InstallNodeDeps(18.0.0)" ] + 3 [ label="SyncNodeProject(basic)" ] + 4 [ label="RunTask(basic:lint)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 1 [ ] + 4 -> 2 [ ] + 4 -> 3 [ ] } diff --git a/crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependencies_when_focused.snap b/crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependencies_when_focused.snap index 98dcdfe1ac2..cd1c38ff8b7 100644 --- a/crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependencies_when_focused.snap +++ b/crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependencies_when_focused.snap @@ -3,19 +3,20 @@ source: crates/cli/tests/action_graph_test.rs expression: assert.output() --- digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(chain)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunTarget(chain:e)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="RunTarget(chain:f)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] - 5 -> 2 [ arrowhead=box, arrowtail=box] - 5 -> 3 [ arrowhead=box, arrowtail=box] - 4 -> 5 [ arrowhead=box, arrowtail=box] + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(18.0.0)" ] + 2 [ label="InstallNodeDeps(18.0.0)" ] + 3 [ label="SyncNodeProject(chain)" ] + 4 [ label="RunTask(chain:e)" ] + 5 [ label="RunTask(chain:f)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 1 [ ] + 5 -> 2 [ ] + 5 -> 3 [ ] + 4 -> 2 [ ] + 4 -> 3 [ ] + 4 -> 5 [ ] } diff --git a/crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependents_when_focused-2.snap b/crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependents_when_focused-2.snap new file mode 100644 index 00000000000..476525f957f --- /dev/null +++ b/crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependents_when_focused-2.snap @@ -0,0 +1,83 @@ +--- +source: crates/cli/tests/action_graph_test.rs +expression: assert.output() +--- +digraph { + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(18.0.0)" ] + 2 [ label="InstallNodeDeps(18.0.0)" ] + 3 [ label="SyncNodeProject(basic)" ] + 4 [ label="RunTask(basic:build)" ] + 5 [ label="SyncNodeProject(buildA)" ] + 6 [ label="SyncNodeProject(buildC)" ] + 7 [ label="SyncNodeProject(noTasks)" ] + 8 [ label="RunTask(buildA:build)" ] + 9 [ label="RunTask(buildC:build)" ] + 10 [ label="SetupSystemTool" ] + 11 [ label="SyncSystemProject(scopeDeps)" ] + 12 [ label="SyncNodeProject(buildB)" ] + 13 [ label="RunTask(scopeDeps:build)" ] + 14 [ label="RunTask(buildB:build)" ] + 15 [ label="RunTask(scopeDeps:buildNoDupes)" ] + 16 [ label="SyncSystemProject(mergeAllStrategies)" ] + 17 [ label="RunTask(mergeAllStrategies:standard)" ] + 18 [ label="SyncSystemProject(mergePrepend)" ] + 19 [ label="RunTask(mergePrepend:standard)" ] + 20 [ label="SyncSystemProject(mergeReplace)" ] + 21 [ label="RunTask(mergeReplace:standard)" ] + 22 [ label="SyncSystemProject(mergeAppend)" ] + 23 [ label="RunTask(mergeAppend:standard)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 1 [ ] + 4 -> 2 [ ] + 4 -> 3 [ ] + 6 -> 1 [ ] + 7 -> 1 [ ] + 5 -> 1 [ ] + 5 -> 6 [ ] + 5 -> 3 [ ] + 5 -> 7 [ ] + 9 -> 2 [ ] + 9 -> 6 [ ] + 10 -> 0 [ ] + 12 -> 1 [ ] + 11 -> 10 [ ] + 11 -> 7 [ ] + 11 -> 5 [ ] + 11 -> 12 [ ] + 11 -> 6 [ ] + 14 -> 2 [ ] + 14 -> 12 [ ] + 15 -> 11 [ ] + 15 -> 8 [ ] + 15 -> 14 [ ] + 15 -> 9 [ ] + 16 -> 10 [ ] + 16 -> 12 [ ] + 17 -> 16 [ ] + 17 -> 14 [ ] + 18 -> 10 [ ] + 18 -> 12 [ ] + 19 -> 18 [ ] + 19 -> 14 [ ] + 20 -> 10 [ ] + 20 -> 12 [ ] + 21 -> 20 [ ] + 21 -> 14 [ ] + 22 -> 10 [ ] + 22 -> 12 [ ] + 23 -> 22 [ ] + 23 -> 14 [ ] + 13 -> 11 [ ] + 13 -> 8 [ ] + 13 -> 14 [ ] + 13 -> 9 [ ] + 8 -> 2 [ ] + 8 -> 5 [ ] + 8 -> 4 [ ] + 8 -> 9 [ ] +} + + + diff --git a/crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependents_when_focused.snap b/crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependents_when_focused.snap index 12f095418da..d9e4c9d82eb 100644 --- a/crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependents_when_focused.snap +++ b/crates/cli/tests/snapshots/action_graph_test__action_graph__includes_dependents_when_focused.snap @@ -3,32 +3,16 @@ source: crates/cli/tests/action_graph_test.rs expression: assert.output() --- digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(18.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(basic)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunTarget(basic:build)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="SyncNodeProject(buildA)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 6 [ label="SyncNodeProject(buildC)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 7 [ label="SyncNodeProject(noTasks)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 8 [ label="RunTarget(buildA:build)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 9 [ label="RunTarget(buildC:build)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] - 5 -> 1 [ arrowhead=box, arrowtail=box] - 6 -> 1 [ arrowhead=box, arrowtail=box] - 5 -> 6 [ arrowhead=box, arrowtail=box] - 5 -> 3 [ arrowhead=box, arrowtail=box] - 7 -> 1 [ arrowhead=box, arrowtail=box] - 5 -> 7 [ arrowhead=box, arrowtail=box] - 8 -> 2 [ arrowhead=box, arrowtail=box] - 8 -> 5 [ arrowhead=box, arrowtail=box] - 9 -> 2 [ arrowhead=box, arrowtail=box] - 9 -> 6 [ arrowhead=box, arrowtail=box] - 8 -> 4 [ arrowhead=box, arrowtail=box] - 8 -> 9 [ arrowhead=box, arrowtail=box] + 0 [ label="SyncWorkspace" ] + 1 [ label="SetupNodeTool(18.0.0)" ] + 2 [ label="InstallNodeDeps(18.0.0)" ] + 3 [ label="SyncNodeProject(basic)" ] + 4 [ label="RunTask(basic:build)" ] + 1 -> 0 [ ] + 2 -> 1 [ ] + 3 -> 1 [ ] + 4 -> 2 [ ] + 4 -> 3 [ ] } diff --git a/crates/cli/tests/snapshots/action_graph_test__action_graph__outputs_json.snap b/crates/cli/tests/snapshots/action_graph_test__action_graph__outputs_json.snap index 534f5387238..e8b1e3d5858 100644 --- a/crates/cli/tests/snapshots/action_graph_test__action_graph__outputs_json.snap +++ b/crates/cli/tests/snapshots/action_graph_test__action_graph__outputs_json.snap @@ -2,6 +2,6 @@ source: crates/cli/tests/action_graph_test.rs expression: assert.output() --- -{"nodes":[{"id":2,"label":"InstallNodeDeps(18.0.0)"},{"id":4,"label":"RunTarget(basic:lint)"},{"id":1,"label":"SetupNodeTool(18.0.0)"},{"id":3,"label":"SyncNodeProject(basic)"}],"edges":[{"id":"2 -> 1","label":"","source":2,"target":1},{"id":"3 -> 1","label":"","source":3,"target":1},{"id":"4 -> 2","label":"","source":4,"target":2},{"id":"4 -> 3","label":"","source":4,"target":3}]} +{"nodes":[{"id":0,"label":"SyncWorkspace"},{"id":2,"label":"InstallNodeDeps(18.0.0)"},{"id":4,"label":"RunTask(basic:lint)"},{"id":1,"label":"SetupNodeTool(18.0.0)"},{"id":3,"label":"SyncNodeProject(basic)"}],"edges":[{"id":"1 -> 0","label":"","source":1,"target":0},{"id":"2 -> 1","label":"","source":2,"target":1},{"id":"3 -> 1","label":"","source":3,"target":1},{"id":"4 -> 2","label":"","source":4,"target":2},{"id":"4 -> 3","label":"","source":4,"target":3}]} diff --git a/crates/cli/tests/snapshots/run_test__errors_for_cycle_in_task_deps.snap b/crates/cli/tests/snapshots/run_test__errors_for_cycle_in_task_deps.snap index bbac63af8b6..c021624872c 100644 --- a/crates/cli/tests/snapshots/run_test__errors_for_cycle_in_task_deps.snap +++ b/crates/cli/tests/snapshots/run_test__errors_for_cycle_in_task_deps.snap @@ -2,7 +2,7 @@ source: crates/cli/tests/run_test.rs expression: assert.output() --- -Error: × A dependency cycle has been detected for RunTarget(depsC:taskCycle). +Error: × A dependency cycle has been detected for RunTask(depsC:taskCycle). diff --git a/crates/core/action-pipeline/Cargo.toml b/crates/core/action-pipeline/Cargo.toml index aa1e4647436..dc6691a25e9 100644 --- a/crates/core/action-pipeline/Cargo.toml +++ b/crates/core/action-pipeline/Cargo.toml @@ -4,20 +4,13 @@ version = "0.1.0" edition = "2021" publish = false -[lib] -bench = false - -[[bench]] -name = "pipeline_benchmark" -harness = false - [dependencies] moon_action = { path = "../action" } moon_action_context = { path = "../action-context" } +moon_action_graph = { path = "../../../nextgen/action-graph" } moon_actions = { path = "../actions" } moon_api = { path = "../../../nextgen/api" } moon_cache_item = { path = "../../../nextgen/cache-item" } -moon_dep_graph = { path = "../dep-graph" } moon_emitter = { path = "../emitter" } moon_hash = { path = "../../../nextgen/hash" } moon_logger = { path = "../logger" } diff --git a/crates/core/action-pipeline/benches/pipeline_benchmark.rs b/crates/core/action-pipeline/benches/pipeline_benchmark.rs deleted file mode 100644 index 9025569d393..00000000000 --- a/crates/core/action-pipeline/benches/pipeline_benchmark.rs +++ /dev/null @@ -1,63 +0,0 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use moon::{build_dep_graph, generate_project_graph, load_workspace_from}; -use moon_action_pipeline::Pipeline; -use moon_dep_graph::DepGraph; -use moon_project_graph::ProjectGraph; -use moon_target::Target; -use moon_test_utils::{create_sandbox_with_config, get_cases_fixture_configs}; -use moon_workspace::Workspace; - -fn generate_dep_graph(_workspace: &Workspace, project_graph: &ProjectGraph) -> DepGraph { - let mut dep_graph = build_dep_graph(project_graph); - - dep_graph - .run_target(Target::parse("base:standard").unwrap(), None) - .unwrap(); - - dep_graph - .run_target(Target::parse("depsA:dependencyOrder").unwrap(), None) - .unwrap(); - - dep_graph - .run_target(Target::parse("outputs:withDeps").unwrap(), None) - .unwrap(); - - dep_graph - .run_target(Target::parse("passthroughArgs:c").unwrap(), None) - .unwrap(); - - dep_graph - .run_target(Target::parse("targetScopeB:self").unwrap(), None) - .unwrap(); - - dep_graph.build() -} - -pub fn pipeline_benchmark(c: &mut Criterion) { - let (workspace_config, toolchain_config, tasks_config) = get_cases_fixture_configs(); - - let sandbox = create_sandbox_with_config( - "cases", - Some(workspace_config), - Some(toolchain_config), - Some(tasks_config), - ); - - c.bench_function("pipeline", |b| { - b.iter(|| async { - let mut workspace = load_workspace_from(sandbox.path()).await.unwrap(); - let project_graph = generate_project_graph(&mut workspace).await.unwrap(); - let dep_graph = generate_dep_graph(&workspace, &project_graph); - - black_box( - Pipeline::new(workspace, project_graph) - .run(dep_graph, None) - .await - .unwrap(), - ); - }) - }); -} - -criterion_group!(pipeline, pipeline_benchmark); -criterion_main!(pipeline); diff --git a/crates/core/action-pipeline/src/estimator.rs b/crates/core/action-pipeline/src/estimator.rs index dc823417a19..86daf2561ec 100644 --- a/crates/core/action-pipeline/src/estimator.rs +++ b/crates/core/action-pipeline/src/estimator.rs @@ -67,12 +67,12 @@ impl Estimator { } match node { - ActionNode::SetupTool(_) - | ActionNode::InstallDeps(_) - | ActionNode::InstallProjectDeps(_, _) => { + ActionNode::SetupTool { .. } + | ActionNode::InstallDeps { .. } + | ActionNode::InstallProjectDeps { .. } => { install_duration += task_duration; } - ActionNode::RunTarget(_, target) => { + ActionNode::RunTask { target, .. } => { let task_id = target.task_id.to_string(); if let Some(task) = tasks.get_mut(&task_id) { diff --git a/crates/core/action-pipeline/src/pipeline.rs b/crates/core/action-pipeline/src/pipeline.rs index a98765d0f2d..c5816ba92fb 100644 --- a/crates/core/action-pipeline/src/pipeline.rs +++ b/crates/core/action-pipeline/src/pipeline.rs @@ -7,18 +7,20 @@ use crate::subscribers::moonbase::MoonbaseSubscriber; use console::Term; use moon_action::{Action, ActionNode, ActionStatus}; use moon_action_context::ActionContext; -use moon_dep_graph::DepGraph; +use moon_action_graph::ActionGraph; use moon_emitter::{Emitter, Event}; -use moon_logger::{debug, error, trace}; +use moon_logger::{debug, error, trace, warn}; use moon_notifier::WebhooksSubscriber; use moon_project_graph::ProjectGraph; use moon_terminal::{label_checkpoint, label_to_the_moon, Checkpoint, ExtendedTerm}; use moon_utils::{is_ci, is_test_env, time}; use moon_workspace::Workspace; use starbase_styles::color; +use std::mem; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::{RwLock, Semaphore}; +use tokio::sync::{RwLock, RwLockReadGuard, Semaphore}; +use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; const LOG_TARGET: &str = "moon:action-pipeline"; @@ -68,7 +70,7 @@ impl Pipeline { pub async fn run( &mut self, - dep_graph: DepGraph, + action_graph: ActionGraph, context: Option, ) -> miette::Result { let start = Instant::now(); @@ -78,21 +80,14 @@ impl Pipeline { )); let workspace = Arc::clone(&self.workspace); let project_graph = Arc::clone(&self.project_graph); - let mut results: ActionResults = vec![]; - let mut passed_count = 0; - let mut cached_count = 0; - let mut failed_count = 0; - // Queue actions in topological order that need to be processed, - // grouped into batches based on dependency requirements - let total_actions_count = dep_graph.get_node_count(); - let batches = dep_graph.sort_batched_topological()?; - let batches_count = batches.len(); + // Queue actions in topological order that need to be processed + let total_actions_count = action_graph.get_node_count(); let local_emitter = emitter.read().await; debug!( target: LOG_TARGET, - "Running {} actions across {} batches", total_actions_count, batches_count + "Running {} actions", total_actions_count ); local_emitter @@ -117,121 +112,173 @@ impl Pipeline { self.concurrency.unwrap_or_else(num_cpus::get), )); - for (b, batch) in batches.into_iter().enumerate() { - let batch_index = b + 1; - let batch_target_name = format!("{LOG_TARGET}:batch:{batch_index}"); - let actions_count = batch.len(); - let mut action_handles = vec![]; + let mut results: ActionResults = vec![]; + let mut action_handles = vec![]; + let mut persistent_nodes = vec![]; + let mut action_graph_iter = action_graph.try_iter()?; + + action_graph_iter.monitor_completed(); - trace!( - target: &batch_target_name, - "Running {} actions in batch {}", - actions_count, - batch_index - ); + while action_graph_iter.has_pending() { + let Some(node_index) = action_graph_iter.next() else { + // Nothing new to run since they're waiting on currently + // running actions, so exhaust the current list + self.run_handles(mem::take(&mut action_handles), &mut results, &local_emitter) + .await?; - for (i, node_index) in batch.into_iter().enumerate() { - let action_index = i + 1; - - if let Some(node) = dep_graph.get_node_from_index(&node_index) { - let context_clone = Arc::clone(&context); - let emitter_clone = Arc::clone(&emitter); - let workspace_clone = Arc::clone(&workspace); - let project_graph_clone = Arc::clone(&project_graph); - let cancel_token_clone = cancel_token.clone(); - - let mut action = Action::new(node.to_owned()); - action.log_target = format!("{batch_target_name}:{action_index}"); - - let Ok(permit) = semaphore.clone().acquire_owned().await else { - break; // Should error? - }; - - action_handles.push(tokio::spawn(async move { - let result = tokio::select! { - biased; - - _ = cancel_token_clone.cancelled() => { - Err(PipelineError::Aborted("Received ctrl + c, shutting down".into()).into()) - } - res = process_action( - action, - context_clone, - emitter_clone, - workspace_clone, - project_graph_clone, - ) => res - }; - - drop(permit); - - result - })); - } else { - return Err(PipelineError::UnknownActionNode.into()); - } + continue; + }; + + let Some(node) = action_graph.get_node_from_index(&node_index) else { + warn!( + target: LOG_TARGET, + "Received a graph index {} with no associated node, unable to process", + node_index.index() + ); + + continue; + }; + + // Run these later as a parallel batch + if node.is_persistent() { + trace!( + target: LOG_TARGET, + // index = node_index.index(), + "Marking action {} as persistent, will defer run", + node_index.index() + ); + + // Must mark as completed otherwise the loop hangs + action_graph_iter.mark_completed(node_index); + + persistent_nodes.push(node_index); + + continue; } - // Wait for all actions in this batch to complete - let mut abort_error: Option = None; - let mut show_abort_log = false; + let context_clone = Arc::clone(&context); + let emitter_clone = Arc::clone(&emitter); + let workspace_clone = Arc::clone(&workspace); + let project_graph_clone = Arc::clone(&project_graph); + let cancel_token_clone = cancel_token.clone(); + let sender = action_graph_iter.sender.clone(); + let mut action = Action::new(node.to_owned()); + action.node_index = node_index.index(); + + let Ok(permit) = semaphore.clone().acquire_owned().await else { + continue; // Should error? + }; - for handle in action_handles { - if abort_error.is_some() { - if !handle.is_finished() { - handle.abort(); + action_handles.push(tokio::spawn(async move { + let result = tokio::select! { + biased; + + _ = cancel_token_clone.cancelled() => { + Err(PipelineError::Aborted("Received ctrl + c, shutting down".into()).into()) } - } else { - match handle.await { - Ok(Ok(mut result)) => { - if result.has_failed() { - failed_count += 1; - } else if result.was_cached() { - cached_count += 1; - } else { - passed_count += 1; - } - - show_abort_log = result.should_abort(); - - if self.bail && result.should_bail() || result.should_abort() { - abort_error = Some(result.get_error()); - } else { - results.push(result); - } - } - Ok(Err(error)) => { - abort_error = Some(error); - } - _ => { - abort_error = - Some(PipelineError::Aborted("Unknown error!".into()).into()); - } - }; + res = process_action( + action, + context_clone, + emitter_clone, + workspace_clone, + project_graph_clone, + ) => res + }; + + if let Ok(action) = &result { + let _ = sender.send(action.node_index); } + + drop(permit); + + result + })); + + // Run this in isolation by exhausting the current list of handles + if node.is_interactive() || semaphore.available_permits() == 0 { + self.run_handles(mem::take(&mut action_handles), &mut results, &local_emitter) + .await?; } + } - if let Some(abort_error) = abort_error { - if show_abort_log { - error!( - target: &batch_target_name, - "Encountered a critical error, aborting the action pipeline" - ); + if !persistent_nodes.is_empty() { + trace!( + target: LOG_TARGET, + "Running {} persistent actions", + persistent_nodes.len(), + ); + } + + for node_index in persistent_nodes { + let Some(node) = action_graph.get_node_from_index(&node_index) else { + warn!( + target: LOG_TARGET, + "Received a graph index {} with no associated node, unable to process", + node_index.index() + ); + + continue; + }; + + let context_clone = Arc::clone(&context); + let emitter_clone = Arc::clone(&emitter); + let workspace_clone = Arc::clone(&workspace); + let project_graph_clone = Arc::clone(&project_graph); + let cancel_token_clone = cancel_token.clone(); + let sender = action_graph_iter.sender.clone(); + let mut action = Action::new(node.to_owned()); + action.node_index = node_index.index(); + + let Ok(permit) = semaphore.clone().acquire_owned().await else { + continue; // Should error? + }; + + action_handles.push(tokio::spawn(async move { + let result = tokio::select! { + biased; + + _ = cancel_token_clone.cancelled() => { + Err(PipelineError::Aborted("Received ctrl + c, shutting down".into()).into()) + } + res = process_action( + action, + context_clone, + emitter_clone, + workspace_clone, + project_graph_clone, + ) => res + }; + + if let Ok(action) = &result { + let _ = sender.send(action.node_index); } - local_emitter - .emit(Event::PipelineAborted { - error: abort_error.to_string(), - }) - .await?; + drop(permit); - return Err(abort_error); - } + result + })); } + // Run any remaining actions + self.run_handles(action_handles, &mut results, &local_emitter) + .await?; + let duration = start.elapsed(); let estimate = Estimator::calculate(&results, duration); let context = Arc::into_inner(context).unwrap().into_inner(); + let mut passed_count = 0; + let mut cached_count = 0; + let mut failed_count = 0; + + for result in &results { + if result.has_failed() { + failed_count += 1; + } else if result.was_cached() { + cached_count += 1; + } else { + passed_count += 1; + } + } debug!( target: LOG_TARGET, @@ -256,6 +303,58 @@ impl Pipeline { Ok(results) } + async fn run_handles( + &self, + handles: Vec>>, + results: &mut ActionResults, + emitter: &RwLockReadGuard<'_, Emitter>, + ) -> miette::Result<()> { + let mut abort_error: Option = None; + let mut show_abort_log = false; + + for handle in handles { + if abort_error.is_some() { + if !handle.is_finished() { + handle.abort(); + } + } else { + match handle.await { + Ok(Ok(mut result)) => { + show_abort_log = result.should_abort(); + + if self.bail && result.should_bail() || result.should_abort() { + abort_error = Some(result.get_error()); + } else { + results.push(result); + } + } + Ok(Err(error)) => { + abort_error = Some(error); + } + _ => { + abort_error = Some(PipelineError::Aborted("Unknown error!".into()).into()); + } + }; + } + } + + if let Some(abort_error) = abort_error { + if show_abort_log { + error!("Encountered a critical error, aborting the action pipeline"); + } + + emitter + .emit(Event::PipelineAborted { + error: abort_error.to_string(), + }) + .await?; + + return Err(abort_error); + } + + Ok(()) + } + pub fn render_summary(&self, results: &ActionResults) -> miette::Result<()> { let term = Term::buffered_stdout(); term.line("")?; @@ -269,9 +368,7 @@ impl Pipeline { term.line(label_checkpoint( match &result.node { - Some(ActionNode::RunTarget(_, target)) => target.as_str(), - Some(ActionNode::RunInteractiveTarget(_, target)) => target.as_str(), - Some(ActionNode::RunPersistentTarget(_, target)) => target.as_str(), + Some(ActionNode::RunTask { target, .. }) => target.as_str(), _ => &result.label, }, Checkpoint::RunFailed, @@ -373,14 +470,7 @@ impl Pipeline { let mut skipped_count = 0; for result in results { - if compact - && !matches!( - result.node.as_ref().unwrap(), - ActionNode::RunTarget(_, _) - | ActionNode::RunInteractiveTarget(_, _) - | ActionNode::RunPersistentTarget(_, _) - ) - { + if compact && !matches!(result.node.as_ref().unwrap(), ActionNode::RunTask { .. }) { continue; } diff --git a/crates/core/action-pipeline/src/processor.rs b/crates/core/action-pipeline/src/processor.rs index 4a099f08047..cc939b1002e 100644 --- a/crates/core/action-pipeline/src/processor.rs +++ b/crates/core/action-pipeline/src/processor.rs @@ -53,7 +53,7 @@ pub async fn process_action( let result = match &node { // Setup and install the specific tool - ActionNode::SetupTool(runtime) => { + ActionNode::SetupTool { runtime } => { local_emitter .emit(Event::ToolInstalling { runtime }) .await?; @@ -71,7 +71,7 @@ pub async fn process_action( } // Install dependencies in the workspace root - ActionNode::InstallDeps(runtime) => { + ActionNode::InstallDeps { runtime } => { local_emitter .emit(Event::DependenciesInstalling { project: None, @@ -93,7 +93,10 @@ pub async fn process_action( } // Install dependencies in the project root - ActionNode::InstallProjectDeps(runtime, project_id) => { + ActionNode::InstallProjectDeps { + runtime, + project: project_id, + } => { let project = local_project_graph.get(project_id)?; local_emitter @@ -118,7 +121,10 @@ pub async fn process_action( } // Sync a project within the graph - ActionNode::SyncProject(runtime, project_id) => { + ActionNode::SyncProject { + runtime, + project: project_id, + } => { let project = local_project_graph.get(project_id)?; local_emitter @@ -165,9 +171,9 @@ pub async fn process_action( } // Run a task within a project - ActionNode::RunTarget(runtime, target) - | ActionNode::RunInteractiveTarget(runtime, target) - | ActionNode::RunPersistentTarget(runtime, target) => { + ActionNode::RunTask { + runtime, target, .. + } => { let project = local_project_graph.get(target.get_project_id().unwrap())?; local_emitter.emit(Event::TargetRunning { target }).await?; @@ -207,7 +213,10 @@ pub async fn process_action( if action.has_failed() { // If these fail, we should abort instead of trying to continue - if matches!(node, ActionNode::SetupTool(_)) || matches!(node, ActionNode::InstallDeps(_)) { + if matches!( + node, + ActionNode::SetupTool { .. } | ActionNode::InstallDeps { .. } + ) { action.abort(); } } diff --git a/crates/core/action-pipeline/src/subscribers/moonbase.rs b/crates/core/action-pipeline/src/subscribers/moonbase.rs index 18e63479b4d..20b900df961 100644 --- a/crates/core/action-pipeline/src/subscribers/moonbase.rs +++ b/crates/core/action-pipeline/src/subscribers/moonbase.rs @@ -459,9 +459,13 @@ impl Subscriber for MoonbaseSubscriber { let archive_path = archive_path.to_owned(); // Create a fake action label so that we can check the CI cache - let action_label = - ActionNode::RunTarget(Runtime::system(), (*target).to_owned()) - .label(); + let action_label = ActionNode::RunTask { + interactive: false, + persistent: false, + runtime: Runtime::system(), + target: (*target).to_owned(), + } + .label(); let job_id = self.job_ids.get(&action_label).cloned(); // Run this in the background so we don't slow down the pipeline diff --git a/crates/core/action-pipeline/tests/estimator_test.rs b/crates/core/action-pipeline/tests/estimator_test.rs index 13bea75f702..80ab2ba684a 100644 --- a/crates/core/action-pipeline/tests/estimator_test.rs +++ b/crates/core/action-pipeline/tests/estimator_test.rs @@ -7,6 +7,15 @@ use std::time::Duration; const NANOS_PER_MILLI: u32 = 1_000_000; const HALF_SECOND: u32 = NANOS_PER_MILLI * 500; +fn create_run_task_action(runtime: Runtime, target: &str) -> ActionNode { + ActionNode::RunTask { + interactive: false, + persistent: false, + runtime, + target: target.into(), + } +} + mod estimator { use super::*; @@ -31,7 +40,7 @@ mod estimator { let est = Estimator::calculate( &[Action { duration: Some(Duration::new(10, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "proj:task".into())), + node: Some(create_run_task_action(Runtime::system(), "proj:task")), ..Action::default() }], Duration::new(5, 0), @@ -58,27 +67,27 @@ mod estimator { &[ Action { duration: Some(Duration::new(10, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "a:build".into())), + node: Some(create_run_task_action(Runtime::system(), "a:build")), ..Action::default() }, Action { duration: Some(Duration::new(5, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "a:lint".into())), + node: Some(create_run_task_action(Runtime::system(), "a:lint")), ..Action::default() }, Action { duration: Some(Duration::new(15, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "b:build".into())), + node: Some(create_run_task_action(Runtime::system(), "b:build")), ..Action::default() }, Action { duration: Some(Duration::new(8, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "c:test".into())), + node: Some(create_run_task_action(Runtime::system(), "c:test")), ..Action::default() }, Action { duration: Some(Duration::new(12, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "d:lint".into())), + node: Some(create_run_task_action(Runtime::system(), "d:lint")), ..Action::default() }, ], @@ -113,17 +122,21 @@ mod estimator { &[ Action { duration: Some(Duration::new(10, 0)), - node: Some(ActionNode::SetupTool(Runtime::system())), + node: Some(ActionNode::SetupTool { + runtime: Runtime::system(), + }), ..Action::default() }, Action { duration: Some(Duration::new(25, 0)), - node: Some(ActionNode::InstallDeps(Runtime::system())), + node: Some(ActionNode::InstallDeps { + runtime: Runtime::system(), + }), ..Action::default() }, Action { duration: Some(Duration::new(10, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "proj:task".into())), + node: Some(create_run_task_action(Runtime::system(), "proj:task")), ..Action::default() }, ], @@ -153,7 +166,7 @@ mod estimator { let est = Estimator::calculate( &[Action { duration: Some(Duration::new(3, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "proj:task".into())), + node: Some(create_run_task_action(Runtime::system(), "proj:task")), status: ActionStatus::Cached, ..Action::default() }], @@ -181,37 +194,41 @@ mod estimator { &[ Action { duration: Some(Duration::new(10, 0)), - node: Some(ActionNode::SetupTool(Runtime::system())), + node: Some(ActionNode::SetupTool { + runtime: Runtime::system(), + }), ..Action::default() }, Action { duration: Some(Duration::new(25, 0)), - node: Some(ActionNode::InstallDeps(Runtime::system())), + node: Some(ActionNode::InstallDeps { + runtime: Runtime::system(), + }), ..Action::default() }, Action { duration: Some(Duration::new(10, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "a:build".into())), + node: Some(create_run_task_action(Runtime::system(), "a:build")), ..Action::default() }, Action { duration: Some(Duration::new(5, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "a:lint".into())), + node: Some(create_run_task_action(Runtime::system(), "a:lint")), ..Action::default() }, Action { duration: Some(Duration::new(15, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "b:build".into())), + node: Some(create_run_task_action(Runtime::system(), "b:build")), ..Action::default() }, Action { duration: Some(Duration::new(8, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "c:test".into())), + node: Some(create_run_task_action(Runtime::system(), "c:test")), ..Action::default() }, Action { duration: Some(Duration::new(12, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "d:lint".into())), + node: Some(create_run_task_action(Runtime::system(), "d:lint")), ..Action::default() }, ], @@ -250,37 +267,41 @@ mod estimator { &[ Action { duration: Some(Duration::new(10, 0)), - node: Some(ActionNode::SetupTool(Runtime::system())), + node: Some(ActionNode::SetupTool { + runtime: Runtime::system(), + }), ..Action::default() }, Action { duration: Some(Duration::new(25, 0)), - node: Some(ActionNode::InstallDeps(Runtime::system())), + node: Some(ActionNode::InstallDeps { + runtime: Runtime::system(), + }), ..Action::default() }, Action { duration: Some(Duration::new(10, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "a:build".into())), + node: Some(create_run_task_action(Runtime::system(), "a:build")), ..Action::default() }, Action { duration: Some(Duration::new(5, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "a:lint".into())), + node: Some(create_run_task_action(Runtime::system(), "a:lint")), ..Action::default() }, Action { duration: Some(Duration::new(15, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "b:build".into())), + node: Some(create_run_task_action(Runtime::system(), "b:build")), ..Action::default() }, Action { duration: Some(Duration::new(8, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "c:test".into())), + node: Some(create_run_task_action(Runtime::system(), "c:test")), ..Action::default() }, Action { duration: Some(Duration::new(12, 0)), - node: Some(ActionNode::RunTarget(Runtime::system(), "d:lint".into())), + node: Some(create_run_task_action(Runtime::system(), "d:lint")), ..Action::default() }, ], diff --git a/crates/core/action/Cargo.toml b/crates/core/action/Cargo.toml index 7d8a6a0046b..5c9a8f1bf49 100644 --- a/crates/core/action/Cargo.toml +++ b/crates/core/action/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" publish = false [dependencies] +moon_action_graph = { path = "../../../nextgen/action-graph" } moon_common = { path = "../../../nextgen/common" } moon_platform_runtime = { path = "../../../nextgen/platform-runtime" } moon_target = { path = "../../../nextgen/target" } diff --git a/crates/core/action/src/action.rs b/crates/core/action/src/action.rs index a36a3226909..deedebcf05d 100644 --- a/crates/core/action/src/action.rs +++ b/crates/core/action/src/action.rs @@ -1,4 +1,4 @@ -use crate::node::ActionNode; +use moon_action_graph::ActionNode; use moon_common::color; use moon_utils::time::{chrono::prelude::*, now_timestamp}; use serde::{Deserialize, Serialize}; @@ -102,6 +102,9 @@ pub struct Action { #[serde(skip)] pub node: Option, + #[serde(skip)] + pub node_index: usize, + pub started_at: Option, #[serde(skip)] @@ -124,6 +127,7 @@ impl Action { label: node.label(), log_target: String::new(), node: Some(node), + node_index: 0, started_at: None, start_time: None, status: ActionStatus::Running, diff --git a/crates/core/action/src/lib.rs b/crates/core/action/src/lib.rs index b195cc47f6e..4a5e97f1cc3 100644 --- a/crates/core/action/src/lib.rs +++ b/crates/core/action/src/lib.rs @@ -1,5 +1,4 @@ mod action; -mod node; pub use action::*; -pub use node::*; +pub use moon_action_graph::ActionNode; diff --git a/crates/core/action/src/node.rs b/crates/core/action/src/node.rs deleted file mode 100644 index f79e2a63f54..00000000000 --- a/crates/core/action/src/node.rs +++ /dev/null @@ -1,83 +0,0 @@ -use moon_common::Id; -use moon_platform_runtime::Runtime; -use moon_target::Target; -use serde::Serialize; -use std::hash::{Hash, Hasher}; - -#[derive(Clone, Debug, Eq, Serialize)] -#[serde(tag = "action", content = "params")] -pub enum ActionNode { - /// Install tool dependencies in the workspace root. - InstallDeps(Runtime), - - /// Install tool dependencies in the project root. - InstallProjectDeps(Runtime, Id), - - /// Run a target (project task). - RunTarget(Runtime, Target), - - /// Run a target (project task) interactively with stdin. - RunInteractiveTarget(Runtime, Target), - - /// Run a target (project task) that never terminates. - RunPersistentTarget(Runtime, Target), - - /// Setup a tool + version for the provided platform. - SetupTool(Runtime), - - /// Sync a project with language specific semantics. - SyncProject(Runtime, Id), - - /// Sync the entire moon workspace. - SyncWorkspace, -} - -impl ActionNode { - pub fn label(&self) -> String { - match self { - ActionNode::InstallDeps(runtime) => { - if runtime.requirement.is_global() { - format!("Install{}Deps", runtime) - } else { - format!("Install{}Deps({})", runtime, runtime.requirement) - } - } - ActionNode::InstallProjectDeps(runtime, id) => { - if runtime.requirement.is_global() { - format!("Install{}DepsInProject({id})", runtime) - } else { - format!( - "Install{}DepsInProject({}, {id})", - runtime, runtime.requirement - ) - } - } - ActionNode::RunTarget(_, id) => format!("RunTarget({id})"), - ActionNode::RunInteractiveTarget(_, id) => format!("RunInteractiveTarget({id})"), - ActionNode::RunPersistentTarget(_, id) => format!("RunPersistentTarget({id})"), - ActionNode::SetupTool(runtime) => { - if runtime.requirement.is_global() { - format!("Setup{}Tool", runtime) - } else { - format!("Setup{}Tool({})", runtime, runtime.requirement) - } - } - ActionNode::SyncProject(runtime, id) => { - format!("Sync{}Project({id})", runtime) - } - ActionNode::SyncWorkspace => "SyncWorkspace".into(), - } - } -} - -impl PartialEq for ActionNode { - fn eq(&self, other: &Self) -> bool { - self.label() == other.label() - } -} - -impl Hash for ActionNode { - fn hash(&self, state: &mut H) { - self.label().hash(state); - } -} diff --git a/crates/core/dep-graph/Cargo.toml b/crates/core/dep-graph/Cargo.toml deleted file mode 100644 index 29484bc5c05..00000000000 --- a/crates/core/dep-graph/Cargo.toml +++ /dev/null @@ -1,37 +0,0 @@ -[package] -name = "moon_dep_graph" -version = "0.1.0" -edition = "2021" -publish = false - -[lib] -bench = false - -[[bench]] -name = "dep_graph_benchmark" -harness = false - -[dependencies] -moon_action = { path = "../action" } -moon_common = { path = "../../../nextgen/common" } -moon_logger = { path = "../logger" } -moon_platform = { path = "../platform" } -moon_project = { path = "../../../nextgen/project" } -moon_project_graph = { path = "../../../nextgen/project-graph" } -moon_query = { path = "../../../nextgen/query" } -moon_target = { path = "../../../nextgen/target" } -moon_task = { path = "../../../nextgen/task" } -miette = { workspace = true } -petgraph = { workspace = true } -rustc-hash = { workspace = true } -starbase_styles = { workspace = true } -thiserror = { workspace = true } - -[dev-dependencies] -moon = { path = "../moon" } -moon_config = { path = "../../../nextgen/config" } -moon_test_utils = { path = "../test-utils" } -moon_utils = { path = "../utils" } -moon_workspace = { path = "../../../nextgen/workspace" } -criterion = { workspace = true } -tokio = { workspace = true } diff --git a/crates/core/dep-graph/benches/dep_graph_benchmark.rs b/crates/core/dep-graph/benches/dep_graph_benchmark.rs deleted file mode 100644 index 6d33f8bc7a6..00000000000 --- a/crates/core/dep-graph/benches/dep_graph_benchmark.rs +++ /dev/null @@ -1,49 +0,0 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use moon::{build_dep_graph, generate_project_graph, load_workspace_from}; -use moon_target::Target; -use moon_test_utils::{create_sandbox_with_config, get_cases_fixture_configs}; - -pub fn build_benchmark(c: &mut Criterion) { - let (workspace_config, toolchain_config, tasks_config) = get_cases_fixture_configs(); - - let sandbox = create_sandbox_with_config( - "cases", - Some(workspace_config), - Some(toolchain_config), - Some(tasks_config), - ); - - c.bench_function("dep_graph_build", |b| { - b.to_async(tokio::runtime::Runtime::new().unwrap()) - .iter(|| async { - let mut workspace = load_workspace_from(sandbox.path()).await.unwrap(); - let project_graph = generate_project_graph(&mut workspace).await.unwrap(); - let mut dep_graph = build_dep_graph(&project_graph); - - dep_graph - .run_target(Target::parse("base:standard").unwrap(), None) - .unwrap(); - - dep_graph - .run_target(Target::parse("depsA:dependencyOrder").unwrap(), None) - .unwrap(); - - dep_graph - .run_target(Target::parse("outputs:withDeps").unwrap(), None) - .unwrap(); - - dep_graph - .run_target(Target::parse("passthroughArgs:c").unwrap(), None) - .unwrap(); - - dep_graph - .run_target(Target::parse("targetScopeB:self").unwrap(), None) - .unwrap(); - - black_box(dep_graph.build()); - }) - }); -} - -criterion_group!(dep_graph, build_benchmark); -criterion_main!(dep_graph); diff --git a/crates/core/dep-graph/src/dep_builder.rs b/crates/core/dep-graph/src/dep_builder.rs deleted file mode 100644 index fa199c1162e..00000000000 --- a/crates/core/dep-graph/src/dep_builder.rs +++ /dev/null @@ -1,474 +0,0 @@ -use crate::dep_graph::{DepGraph, DepGraphType, IndicesType}; -use moon_action::ActionNode; -use moon_common::path::WorkspaceRelativePathBuf; -use moon_common::Id; -use moon_logger::{debug, map_list, trace}; -use moon_platform::{PlatformManager, Runtime}; -use moon_project::Project; -use moon_project_graph::ProjectGraph; -use moon_query::{build_query, Criteria}; -use moon_target::{Target, TargetError, TargetLocator, TargetScope}; -use moon_task::Task; -use petgraph::graph::NodeIndex; -use petgraph::Graph; -use rustc_hash::{FxHashMap, FxHashSet}; -use starbase_styles::color; -use std::mem; - -const LOG_TARGET: &str = "moon:dep-graph"; - -type RuntimePair = (Runtime, Runtime); -type TouchedFilePaths = FxHashSet; - -/// A directed acyclic graph (DAG) for the work that needs to be processed, based on a -/// project or task's dependency chain. This is also known as a "task graph" (not to -/// be confused with our tasks) or a "dependency graph". -pub struct DepGraphBuilder<'ws> { - all_query: Option, - graph: DepGraphType, - indices: IndicesType, - project_graph: &'ws ProjectGraph, - runtimes: FxHashMap, -} - -impl<'ws> DepGraphBuilder<'ws> { - pub fn new(project_graph: &'ws ProjectGraph) -> Self { - debug!(target: LOG_TARGET, "Creating dependency graph"); - - let mut graph = Graph::new(); - let mut indices = FxHashMap::default(); - - // Always sync the workspace - let node = ActionNode::SyncWorkspace; - - trace!( - target: LOG_TARGET, - "Adding {} to graph", - color::muted_light(node.label()) - ); - - indices.insert(node.to_owned(), graph.add_node(node)); - - DepGraphBuilder { - all_query: None, - graph, - indices, - project_graph, - runtimes: FxHashMap::default(), - } - } - - pub fn build(&mut self) -> DepGraph { - DepGraph::new(mem::take(&mut self.graph), mem::take(&mut self.indices)) - } - - pub fn set_query(&mut self, input: &str) -> miette::Result<()> { - self.all_query = Some(build_query(input)?); - - Ok(()) - } - - pub fn get_index_from_node(&self, node: &ActionNode) -> Option<&NodeIndex> { - self.indices.get(node) - } - - // Projects support overriding the the version of their language (tool), - // so we need to account for this via the runtime. However, some actions require - // the workspace version of the language, so we must extract 2 runtimes here. - pub fn get_runtimes_from_project( - &mut self, - project: &Project, - task: Option<&Task>, - ) -> (Runtime, Runtime) { - let key = match task { - Some(task) => task.target.id.clone(), - None => project.id.to_string(), - }; - - if let Some(pair) = self.runtimes.get(&key) { - return pair.clone(); - } - - let mut project_runtime = Runtime::system(); - let mut workspace_runtime = Runtime::system(); - - if let Some(platform) = PlatformManager::read().find(|p| match task { - Some(task) => p.matches(&task.platform, None), - None => p.matches(&project.language.clone().into(), None), - }) { - project_runtime = platform.get_runtime_from_config(Some(&project.config)); - workspace_runtime = platform.get_runtime_from_config(None); - } - - let pair = (project_runtime, workspace_runtime); - - self.runtimes.insert(key, pair.clone()); - - pair - } - - pub fn install_deps( - &mut self, - project: &Project, - task: Option<&Task>, - ) -> miette::Result { - let (project_runtime, workspace_runtime) = self.get_runtimes_from_project(project, task); - let mut installs_in_project = false; - - // If project is NOT in the package manager workspace, then we should - // install dependencies in the project, not the workspace root. - if let Ok(platform) = PlatformManager::read().get(project.language.clone()) { - if !platform.is_project_in_dependency_workspace(project.source.as_str())? { - installs_in_project = true; - - debug!( - target: LOG_TARGET, - "Project {} not within dependency workspaces, will be handled externally", - color::id(&project.id), - ); - } - } - - // When installing dependencies in the project, we will use the - // overridden version if it is available. Otherwise when installing - // in the root, we should *always* use the workspace version. - Ok(if installs_in_project { - self.install_project_deps(&project_runtime, &project.id) - } else { - self.install_workspace_deps(&workspace_runtime) - }) - } - - pub fn install_project_deps(&mut self, runtime: &Runtime, project_id: &Id) -> NodeIndex { - let node = ActionNode::InstallProjectDeps(runtime.clone(), project_id.to_owned()); - - if let Some(index) = self.get_index_from_node(&node) { - return *index; - } - - trace!( - target: LOG_TARGET, - "Adding {} to graph", - color::muted_light(node.label()) - ); - - // Before we install deps, we must ensure the language has been installed - let setup_tool_index = self.setup_tool(runtime); - let index = self.insert_node(&node); - - self.graph.add_edge(index, setup_tool_index, ()); - - index - } - - pub fn install_workspace_deps(&mut self, runtime: &Runtime) -> NodeIndex { - let node = ActionNode::InstallDeps(runtime.clone()); - - if let Some(index) = self.get_index_from_node(&node) { - return *index; - } - - trace!( - target: LOG_TARGET, - "Adding {} to graph", - color::muted_light(node.label()) - ); - - // Before we install deps, we must ensure the language has been installed - let setup_tool_index = self.setup_tool(runtime); - let index = self.insert_node(&node); - - self.graph.add_edge(index, setup_tool_index, ()); - - index - } - - pub fn run_dependents_for_target>(&mut self, target: T) -> miette::Result<()> { - let target = target.as_ref(); - - trace!( - target: LOG_TARGET, - "Adding dependents to run for target {}", - color::label(target), - ); - - if let TargetScope::Project(project_locator) = &target.scope { - let project = self.project_graph.get(project_locator)?; - let dependents = self.project_graph.dependents_of(&project)?; - - for dependent_id in dependents { - let dep_project = self.project_graph.get(dependent_id)?; - - if dep_project.tasks.contains_key(&target.task_id) { - self.run_target(&dep_project.get_task(&target.task_id)?.target, None)?; - } - } - } - - Ok(()) - } - - pub fn run_target>( - &mut self, - target: T, - touched_files: Option<&TouchedFilePaths>, - ) -> miette::Result<(FxHashSet, FxHashSet)> { - let target = target.as_ref(); - let mut inserted_targets = FxHashSet::default(); - let mut inserted_indexes = FxHashSet::default(); - - match &target.scope { - // :task - TargetScope::All => { - let mut projects = vec![]; - - if let Some(all_query) = &self.all_query { - projects.extend(self.project_graph.query(all_query)?); - } else { - projects.extend(self.project_graph.get_all()?); - }; - - for project in projects { - if project.tasks.contains_key(&target.task_id) { - let all_target = Target::new(&project.id, &target.task_id)?; - - if let Some(index) = - self.run_target_by_project(&all_target, &project, touched_files)? - { - inserted_targets.insert(all_target); - inserted_indexes.insert(index); - } - } - } - } - // ^:task - TargetScope::Deps => { - return Err(TargetError::NoDepsInRunContext.into()); - } - // project:task - TargetScope::Project(project_locator) => { - let project = self.project_graph.get(project_locator)?; - let task = project.get_task(&target.task_id)?; - - if let Some(index) = - self.run_target_by_project(&task.target, &project, touched_files)? - { - inserted_targets.insert(task.target.to_owned()); - inserted_indexes.insert(index); - } - } - // #tag:task - TargetScope::Tag(tag) => { - let projects = self - .project_graph - .query(build_query(format!("tag={}", tag))?)?; - - for project in projects { - if project.tasks.contains_key(&target.task_id) { - let tag_target = Target::new(&project.id, &target.task_id)?; - - if let Some(index) = - self.run_target_by_project(&tag_target, &project, touched_files)? - { - inserted_targets.insert(tag_target); - inserted_indexes.insert(index); - } - } - } - } - // ~:task - TargetScope::OwnSelf => { - return Err(TargetError::NoSelfInRunContext.into()); - } - }; - - Ok((inserted_targets, inserted_indexes)) - } - - pub fn run_target_by_project>( - &mut self, - target: T, - project: &Project, - touched_files: Option<&TouchedFilePaths>, - ) -> miette::Result> { - let target = target.as_ref(); - let task = project.get_task(&target.task_id)?; - let (runtime, _) = self.get_runtimes_from_project(project, Some(task)); - - let node = if task.is_persistent() { - ActionNode::RunPersistentTarget(runtime, target.clone()) - } else if task.is_interactive() { - ActionNode::RunInteractiveTarget(runtime, target.clone()) - } else { - ActionNode::RunTarget(runtime, target.clone()) - }; - - if let Some(index) = self.get_index_from_node(&node) { - return Ok(Some(*index)); - } - - // Compare against touched files if provided - if let Some(touched) = touched_files { - if !task.is_affected(touched)? { - trace!( - target: LOG_TARGET, - "Target {} not affected based on touched files, skipping", - color::label(target), - ); - - return Ok(None); - } - } - - trace!( - target: LOG_TARGET, - "Adding {} to graph", - color::muted_light(node.label()) - ); - - // We should install deps & sync projects *before* running targets - let install_deps_index = self.install_deps(project, Some(task))?; - let sync_project_index = self.sync_project(project)?; - let index = self.insert_node(&node); - - self.graph.add_edge(index, install_deps_index, ()); - self.graph.add_edge(index, sync_project_index, ()); - - // And we also need to wait on all dependent targets - if !task.deps.is_empty() { - trace!( - target: LOG_TARGET, - "Adding dependencies {} for target {}", - map_list(&task.deps, |f| color::symbol(f)), - color::label(target), - ); - - // We don't pass touched files to dependencies, because if the parent - // task is affected/going to run, then so should all of these! - for dep_index in self.run_target_task_dependencies(task, None)? { - self.graph.add_edge(index, dep_index, ()); - } - } - - Ok(Some(index)) - } - - pub fn run_target_task_dependencies( - &mut self, - task: &Task, - touched_files: Option<&TouchedFilePaths>, - ) -> miette::Result> { - let parallel = task.options.run_deps_in_parallel; - let mut indexes = vec![]; - let mut previous_target_index = None; - - for dep_target in &task.deps { - let (_, dep_indexes) = self.run_target(dep_target, touched_files)?; - - for dep_index in dep_indexes { - // When parallel, parent depends on child - if parallel { - indexes.push(dep_index); - - // When serial, next child depends on previous child - } else if let Some(prev) = previous_target_index { - self.graph.add_edge(dep_index, prev, ()); - } - - previous_target_index = Some(dep_index); - } - } - - if !parallel { - indexes.push(previous_target_index.unwrap()); - } - - Ok(indexes) - } - - pub fn run_targets_by_locator( - &mut self, - target_locators: &[TargetLocator], - touched_files: Option<&TouchedFilePaths>, - ) -> miette::Result> { - let mut qualified_targets = vec![]; - let mut project = None; - - for locator in target_locators { - let result = match locator { - TargetLocator::Qualified(target) => self.run_target(target, touched_files)?, - TargetLocator::TaskFromWorkingDir(task_id) => { - if project.is_none() { - project = Some(self.project_graph.get_from_path(None)?); - } - - self.run_target( - Target::new(&project.as_ref().unwrap().id, task_id)?, - touched_files, - )? - } - }; - - qualified_targets.extend(result.0); - } - - Ok(qualified_targets) - } - - pub fn setup_tool(&mut self, runtime: &Runtime) -> NodeIndex { - let node = ActionNode::SetupTool(runtime.clone()); - - if let Some(index) = self.get_index_from_node(&node) { - return *index; - } - - trace!( - target: LOG_TARGET, - "Adding {} to graph", - color::muted_light(node.label()) - ); - - self.insert_node(&node) - } - - pub fn sync_project(&mut self, project: &Project) -> miette::Result { - let (runtime, _) = self.get_runtimes_from_project(project, None); - let node = ActionNode::SyncProject(runtime.clone(), project.id.clone()); - - if let Some(index) = self.get_index_from_node(&node) { - return Ok(*index); - } - - trace!( - target: LOG_TARGET, - "Adding {} to graph", - color::muted_light(node.label()) - ); - - // Syncing depends on the language's tool to be installed - let setup_tool_index = self.setup_tool(&runtime); - let index = self.insert_node(&node); - - self.graph.add_edge(index, setup_tool_index, ()); - - // And we should also depend on other projects - for dep_project_id in self.project_graph.dependencies_of(project)? { - let dep_project = self.project_graph.get(dep_project_id)?; - let dep_index = self.sync_project(&dep_project)?; - - if index != dep_index { - self.graph.add_edge(index, dep_index, ()); - } - } - - Ok(index) - } - - // PRIVATE - - fn insert_node(&mut self, node: &ActionNode) -> NodeIndex { - let index = self.graph.add_node(node.to_owned()); - self.indices.insert(node.to_owned(), index); - index - } -} diff --git a/crates/core/dep-graph/src/dep_graph.rs b/crates/core/dep-graph/src/dep_graph.rs deleted file mode 100644 index 2d55a71c113..00000000000 --- a/crates/core/dep-graph/src/dep_graph.rs +++ /dev/null @@ -1,215 +0,0 @@ -use crate::errors::DepGraphError; -use moon_action::ActionNode; -use petgraph::algo::toposort; -use petgraph::dot::{Config, Dot}; -use petgraph::graph::DiGraph; -use petgraph::graph::NodeIndex; -use petgraph::visit::EdgeRef; -use rustc_hash::{FxHashMap, FxHashSet}; - -pub type DepGraphType = DiGraph; -pub type IndicesType = FxHashMap; -pub type BatchedTopoSort = Vec>; - -/// A directed acyclic graph (DAG) for the work that needs to be processed, based on a -/// project or task's dependency chain. This is also known as a "task graph" (not to -/// be confused with our tasks) or a "dependency graph". -pub struct DepGraph { - graph: DepGraphType, - - indices: IndicesType, -} - -impl DepGraph { - pub fn new(graph: DepGraphType, indices: IndicesType) -> Self { - DepGraph { graph, indices } - } - - pub fn is_empty(&self) -> bool { - self.get_node_count() == 0 - } - - pub fn get_index_from_node(&self, node: &ActionNode) -> Option<&NodeIndex> { - self.indices.get(node) - } - - pub fn get_node_count(&self) -> usize { - self.graph.node_count() - } - - pub fn get_node_from_index(&self, index: &NodeIndex) -> Option<&ActionNode> { - self.graph.node_weight(*index) - } - - pub fn sort_topological(&self) -> miette::Result> { - let list = match toposort(&self.graph, None) { - Ok(nodes) => nodes, - Err(error) => { - return Err(DepGraphError::CycleDetected( - self.get_node_from_index(&error.node_id()).unwrap().label(), - ) - .into()); - } - }; - - Ok(list.into_iter().rev().collect()) - } - - pub fn sort_batched_topological(&self) -> miette::Result { - let mut batches: BatchedTopoSort = vec![]; - - // Count how many times an index is referenced across nodes and edges - let mut node_counts = FxHashMap::::default(); - - for ix in self.graph.node_indices() { - node_counts.entry(ix).and_modify(|e| *e += 1).or_insert(0); - - for dep_ix in self.graph.neighbors(ix) { - node_counts - .entry(dep_ix) - .and_modify(|e| *e += 1) - .or_insert(0); - } - } - - // Gather root nodes (count of 0) - let mut root_nodes = FxHashSet::::default(); - - for (ix, count) in &node_counts { - if *count == 0 { - root_nodes.insert(*ix); - } - } - - // If no root nodes are found, but nodes exist, then we have a cycle - let has_sync_workspace = root_nodes.contains(&NodeIndex::new(0)); - - if (!has_sync_workspace && root_nodes.is_empty() - || has_sync_workspace && root_nodes.len() == 1) - && !node_counts.is_empty() - { - self.detect_cycle()?; - } - - while !root_nodes.is_empty() { - let mut next_root_nodes = FxHashSet::::default(); - - // Decrement dependencies of the current batch nodes - for ix in &root_nodes { - for dep_ix in self.graph.neighbors(*ix) { - let count = node_counts - .entry(dep_ix) - .and_modify(|e| *e -= 1) - .or_insert(0); - - // And create a new batch if the count is 0 - if *count == 0 { - next_root_nodes.insert(dep_ix); - } - } - } - - // Push the previous batch onto the list - batches.push(root_nodes.into_iter().collect()); - - // And reset the current nodes - root_nodes = next_root_nodes; - } - - // Move persistent targets to the end - let mut sorted_batches: BatchedTopoSort = vec![]; - let mut persistent: Vec = vec![]; - - for mut batch in batches.into_iter().rev() { - batch.retain(|ix| match self.graph.node_weight(*ix).unwrap() { - // Interactive tasks must be the only item in their batch - // for stdin to work correctly - ActionNode::RunInteractiveTarget(_, _) => { - sorted_batches.push(vec![*ix]); - false - } - // Persistent tasks always run last as a combined batch, - // as their processes may never end - ActionNode::RunPersistentTarget(_, _) => { - persistent.push(*ix); - false - } - _ => true, - }); - - if !batch.is_empty() { - sorted_batches.push(batch); - } - } - - if !persistent.is_empty() { - sorted_batches.push(persistent); - } - - Ok(sorted_batches) - } - - /// Get a labelled representation of the dep graph (which can be serialized easily). - pub fn labeled_graph(&self) -> DiGraph { - let graph = self.graph.clone(); - graph.map(|_, n| n.label(), |_, _| String::new()) - } - - pub fn to_dot(&self) -> String { - let graph = self.graph.map(|_, n| n.label(), |_, e| e); - - let dot = Dot::with_attr_getters( - &graph, - &[Config::EdgeNoLabel, Config::NodeNoLabel], - &|_, e| { - if e.source().index() == 0 { - String::from("arrowhead=none") - } else { - String::from("arrowhead=box, arrowtail=box") - } - }, - &|_, n| { - let id = n.1; - - format!("label=\"{id}\" style=filled, shape=oval, fillcolor=gray, fontcolor=black") - }, - ); - - format!("{dot:?}") - } - - #[track_caller] - fn detect_cycle(&self) -> miette::Result<()> { - // use petgraph::algo::kosaraju_scc; - - // let scc = kosaraju_scc(&self.graph); - - // // Remove the sync workspace node - // let scc = scc - // .into_iter() - // .filter(|list| !(list.len() == 1 && list[0].index() == 0)) - // .collect::>>(); - - // // The cycle is always the last sequence in the list - // let Some(cycle) = scc.last() else { - // return Err(DepGraphError::CycleDetected("(unknown)".into()).into()); - // }; - - // let path = cycle - // .iter() - // .filter_map(|i| self.get_node_from_index(i).map(|n| n.label())) - // .collect::>() - // .join(" → "); - - // Err(DepGraphError::CycleDetected(path).into()) - - if let Err(error) = toposort(&self.graph, None) { - return Err(DepGraphError::CycleDetected( - self.get_node_from_index(&error.node_id()).unwrap().label(), - ) - .into()); - } - - Ok(()) - } -} diff --git a/crates/core/dep-graph/src/errors.rs b/crates/core/dep-graph/src/errors.rs deleted file mode 100644 index 7d6b1e150ec..00000000000 --- a/crates/core/dep-graph/src/errors.rs +++ /dev/null @@ -1,12 +0,0 @@ -use miette::Diagnostic; -use starbase_styles::{Style, Stylize}; -use thiserror::Error; - -#[derive(Error, Debug, Diagnostic)] -pub enum DepGraphError { - #[error("A dependency cycle has been detected for {}.", .0.style(Style::File))] - CycleDetected(String), - - #[error("Unknown node {0} found in dependency graph. How did this get here?")] - UnknownNode(usize), -} diff --git a/crates/core/dep-graph/src/lib.rs b/crates/core/dep-graph/src/lib.rs deleted file mode 100644 index 66f77643c72..00000000000 --- a/crates/core/dep-graph/src/lib.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod dep_builder; -mod dep_graph; -mod errors; - -pub use dep_builder::DepGraphBuilder; -pub use dep_graph::*; -pub use errors::DepGraphError; diff --git a/crates/core/dep-graph/tests/dep_graph_test.rs b/crates/core/dep-graph/tests/dep_graph_test.rs deleted file mode 100644 index 309dcaa57ba..00000000000 --- a/crates/core/dep-graph/tests/dep_graph_test.rs +++ /dev/null @@ -1,679 +0,0 @@ -use moon::{build_dep_graph, generate_project_graph, load_workspace_from}; -use moon_common::path::WorkspaceRelativePathBuf; -use moon_config::{ - PartialInheritedTasksConfig, PartialNodeConfig, PartialToolchainConfig, PartialWorkspaceConfig, - PartialWorkspaceProjects, UnresolvedVersionSpec, -}; -use moon_dep_graph::BatchedTopoSort; -use moon_project_graph::ProjectGraph; -use moon_target::Target; -use moon_test_utils::{assert_snapshot, create_input_paths, create_sandbox_with_config, Sandbox}; -use moon_workspace::Workspace; -use petgraph::graph::NodeIndex; -use rustc_hash::{FxHashMap, FxHashSet}; - -async fn create_project_graph() -> (Workspace, ProjectGraph, Sandbox) { - let workspace_config = PartialWorkspaceConfig { - projects: Some(PartialWorkspaceProjects::Sources(FxHashMap::from_iter([ - ("advanced".into(), "advanced".to_owned()), - ("basic".into(), "basic".to_owned()), - ("emptyConfig".into(), "empty-config".to_owned()), - ("noConfig".into(), "no-config".to_owned()), - // Deps - ("foo".into(), "deps/foo".to_owned()), - ("bar".into(), "deps/bar".to_owned()), - ("baz".into(), "deps/baz".to_owned()), - // Tasks - ("tasks".into(), "tasks".to_owned()), - ("platforms".into(), "platforms".to_owned()), - ]))), - ..PartialWorkspaceConfig::default() - }; - let toolchain_config = PartialToolchainConfig { - node: Some(PartialNodeConfig { - version: Some(UnresolvedVersionSpec::parse("16.0.0").unwrap()), - dedupe_on_lockfile_change: Some(false), - ..PartialNodeConfig::default() - }), - ..PartialToolchainConfig::default() - }; - let tasks_config = PartialInheritedTasksConfig { - file_groups: Some(FxHashMap::from_iter([ - ( - "sources".into(), - create_input_paths(["src/**/*", "types/**/*"]), - ), - ("tests".into(), create_input_paths(["tests/**/*"])), - ])), - ..PartialInheritedTasksConfig::default() - }; - - let sandbox = create_sandbox_with_config( - "projects", - Some(workspace_config), - Some(toolchain_config), - Some(tasks_config), - ); - - let mut workspace = load_workspace_from(sandbox.path()).await.unwrap(); - let project_graph = generate_project_graph(&mut workspace).await.unwrap(); - - (workspace, project_graph, sandbox) -} - -async fn create_tasks_project_graph() -> (Workspace, ProjectGraph, Sandbox) { - let workspace_config = PartialWorkspaceConfig { - projects: Some(PartialWorkspaceProjects::Sources(FxHashMap::from_iter([ - ("basic".into(), "basic".to_owned()), - ("buildA".into(), "build-a".to_owned()), - ("buildB".into(), "build-b".to_owned()), - ("buildC".into(), "build-c".to_owned()), - ("chain".into(), "chain".to_owned()), - ("cycle".into(), "cycle".to_owned()), - ("inputA".into(), "input-a".to_owned()), - ("inputB".into(), "input-b".to_owned()), - ("inputC".into(), "input-c".to_owned()), - ( - "mergeAllStrategies".into(), - "merge-all-strategies".to_owned(), - ), - ("mergeAppend".into(), "merge-append".to_owned()), - ("mergePrepend".into(), "merge-prepend".to_owned()), - ("mergeReplace".into(), "merge-replace".to_owned()), - ("noTasks".into(), "no-tasks".to_owned()), - ("persistent".into(), "persistent".to_owned()), - ("interactive".into(), "interactive".to_owned()), - ]))), - ..PartialWorkspaceConfig::default() - }; - let toolchain_config = PartialToolchainConfig { - node: Some(PartialNodeConfig { - version: Some(UnresolvedVersionSpec::parse("16.0.0").unwrap()), - ..PartialNodeConfig::default() - }), - ..PartialToolchainConfig::default() - }; - let tasks_config = PartialInheritedTasksConfig { - file_groups: Some(FxHashMap::from_iter([( - "sources".into(), - create_input_paths(["src/**/*"]), - )])), - ..PartialInheritedTasksConfig::default() - }; - - let sandbox = create_sandbox_with_config( - "tasks", - Some(workspace_config), - Some(toolchain_config), - Some(tasks_config), - ); - - let mut workspace = load_workspace_from(sandbox.path()).await.unwrap(); - let project_graph = generate_project_graph(&mut workspace).await.unwrap(); - - (workspace, project_graph, sandbox) -} - -fn sort_batches(batches: BatchedTopoSort) -> BatchedTopoSort { - let mut list: BatchedTopoSort = vec![]; - - for batch in batches { - let mut new_batch = batch.clone(); - new_batch.sort(); - list.push(new_batch); - } - - list -} - -#[tokio::test] -// #[should_panic(expected = "A dependency cycle has been detected for RunTarget(cycle:a)")] -#[should_panic(expected = "A dependency cycle has been detected for RunTarget(cycle:c)")] -async fn detects_cycles() { - let (_workspace, projects, _sandbox) = create_tasks_project_graph().await; - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::new("cycle", "a").unwrap(), None) - .unwrap(); - graph - .run_target(&Target::new("cycle", "b").unwrap(), None) - .unwrap(); - graph - .run_target(&Target::new("cycle", "c").unwrap(), None) - .unwrap(); - let graph = graph.build(); - - assert_eq!( - sort_batches(graph.sort_batched_topological().unwrap()), - vec![vec![NodeIndex::new(0)], vec![NodeIndex::new(1)]] - ); -} - -mod run_target { - use super::*; - - #[tokio::test] - async fn single_targets() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::new("tasks", "test").unwrap(), None) - .unwrap(); - graph - .run_target(&Target::new("tasks", "lint").unwrap(), None) - .unwrap(); - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - - assert_eq!( - graph.sort_topological().unwrap(), - vec![ - NodeIndex::new(0), - NodeIndex::new(1), - NodeIndex::new(2), // sync project - NodeIndex::new(3), // test - NodeIndex::new(4), // lint - NodeIndex::new(5), - ] - ); - assert_eq!( - sort_batches(graph.sort_batched_topological().unwrap()), - vec![ - vec![NodeIndex::new(1)], - vec![NodeIndex::new(2), NodeIndex::new(3)], - vec![NodeIndex::new(0), NodeIndex::new(4), NodeIndex::new(5)] - ] - ); - } - - #[tokio::test] - async fn deps_chain_target() { - let (_workspace, projects, _sandbox) = create_tasks_project_graph().await; - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::new("basic", "test").unwrap(), None) - .unwrap(); - graph - .run_target(&Target::new("basic", "lint").unwrap(), None) - .unwrap(); - graph - .run_target(&Target::new("chain", "a").unwrap(), None) - .unwrap(); - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - assert_eq!( - graph.sort_topological().unwrap(), - vec![ - NodeIndex::new(0), - NodeIndex::new(1), - NodeIndex::new(2), // sync project - NodeIndex::new(3), // test - NodeIndex::new(4), // lint - NodeIndex::new(5), // sync project - NodeIndex::new(6), - NodeIndex::new(12), // f - NodeIndex::new(11), // e - NodeIndex::new(10), // d - NodeIndex::new(9), // c - NodeIndex::new(8), // b - NodeIndex::new(7), // a - ] - ); - assert_eq!( - sort_batches(graph.sort_batched_topological().unwrap()), - vec![ - vec![NodeIndex::new(1)], - vec![NodeIndex::new(2), NodeIndex::new(6)], - vec![NodeIndex::new(12)], - vec![NodeIndex::new(11)], - vec![NodeIndex::new(10)], - vec![NodeIndex::new(9)], - vec![NodeIndex::new(3), NodeIndex::new(8)], - vec![ - NodeIndex::new(0), - NodeIndex::new(4), - NodeIndex::new(5), - NodeIndex::new(7) - ] - ] - ); - } - - #[tokio::test] - async fn moves_persistent_tasks_last() { - let (_workspace, projects, _sandbox) = create_tasks_project_graph().await; - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::new("persistent", "dev").unwrap(), None) - .unwrap(); - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - - assert_eq!( - sort_batches(graph.sort_batched_topological().unwrap()), - vec![ - vec![NodeIndex::new(1)], - vec![ - NodeIndex::new(2), - NodeIndex::new(4), - NodeIndex::new(6), - NodeIndex::new(7) - ], - vec![NodeIndex::new(5), NodeIndex::new(12), NodeIndex::new(13)], - vec![NodeIndex::new(3), NodeIndex::new(11)], - vec![NodeIndex::new(0)], - vec![ - NodeIndex::new(8), - NodeIndex::new(9), - NodeIndex::new(10), - NodeIndex::new(14) - ], - ] - ); - } - - #[tokio::test] - async fn isolates_interactive_tasks() { - let (_workspace, projects, _sandbox) = create_tasks_project_graph().await; - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::new("interactive", "one").unwrap(), None) - .unwrap(); - graph - .run_target(&Target::new("chain", "c").unwrap(), None) - .unwrap(); - graph - .run_target(&Target::new("interactive", "two").unwrap(), None) - .unwrap(); - graph - .run_target(&Target::new("basic", "lint").unwrap(), None) - .unwrap(); - graph - .run_target(&Target::new("interactive", "depOnOne").unwrap(), None) - .unwrap(); - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - - assert_eq!( - sort_batches(graph.sort_batched_topological().unwrap()), - vec![ - vec![NodeIndex::new(1)], - vec![NodeIndex::new(2), NodeIndex::new(5)], - vec![NodeIndex::new(9)], - vec![NodeIndex::new(3), NodeIndex::new(8)], - vec![NodeIndex::new(4)], // interactive - vec![NodeIndex::new(7), NodeIndex::new(11)], - vec![NodeIndex::new(13)], // interactive - vec![NodeIndex::new(10)], // interactive - vec![NodeIndex::new(0), NodeIndex::new(6), NodeIndex::new(12)], - ] - ); - } - - #[tokio::test] - async fn avoids_dupe_targets() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::new("tasks", "lint").unwrap(), None) - .unwrap(); - graph - .run_target(&Target::new("tasks", "lint").unwrap(), None) - .unwrap(); - graph - .run_target(&Target::new("tasks", "lint").unwrap(), None) - .unwrap(); - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - - assert_eq!( - graph.sort_topological().unwrap(), - vec![ - NodeIndex::new(0), - NodeIndex::new(1), - NodeIndex::new(2), // sync project - NodeIndex::new(3), // lint - NodeIndex::new(4), - ] - ); - assert_eq!( - sort_batches(graph.sort_batched_topological().unwrap()), - vec![ - vec![NodeIndex::new(1)], - vec![NodeIndex::new(2), NodeIndex::new(3)], - vec![NodeIndex::new(0), NodeIndex::new(4)] - ] - ); - } - - #[tokio::test] - async fn runs_all_projects_for_target_all_scope() { - let (_workspace, projects, _sandbox) = create_tasks_project_graph().await; - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::parse(":build").unwrap(), None) - .unwrap(); - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - - assert_eq!( - graph.sort_topological().unwrap(), - vec![ - NodeIndex::new(0), - NodeIndex::new(1), - NodeIndex::new(2), // sync project: basic - NodeIndex::new(4), // basic:build - NodeIndex::new(5), // sync project: build-c - NodeIndex::new(6), // sync project: build-a - NodeIndex::new(3), // build-c:build - NodeIndex::new(8), // build-a:build - NodeIndex::new(9), // sync project: build-b - NodeIndex::new(7), // build-b:build - NodeIndex::new(10), // notasks - NodeIndex::new(11) - ] - ); - assert_eq!( - sort_batches(graph.sort_batched_topological().unwrap()), - vec![ - vec![NodeIndex::new(1)], - vec![ - NodeIndex::new(2), - NodeIndex::new(4), - NodeIndex::new(5), - NodeIndex::new(6) - ], - vec![ - NodeIndex::new(3), - NodeIndex::new(8), - NodeIndex::new(9), - NodeIndex::new(10) - ], - vec![NodeIndex::new(0), NodeIndex::new(7), NodeIndex::new(11)], - ] - ); - } - - #[tokio::test] - #[should_panic(expected = "Dependencies scope (^:) is not supported in run contexts.")] - async fn errors_for_target_deps_scope() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::parse("^:lint").unwrap(), None) - .unwrap(); - } - - #[tokio::test] - #[should_panic(expected = "Self scope (~:) is not supported in run contexts.")] - async fn errors_for_target_self_scope() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::parse("~:lint").unwrap(), None) - .unwrap(); - } - - #[tokio::test] - #[should_panic(expected = "No project has been configured with the name or alias unknown")] - async fn errors_for_unknown_project() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::new("unknown", "test").unwrap(), None) - .unwrap(); - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - } - - #[tokio::test] - #[should_panic(expected = "Unknown task build for project tasks.")] - async fn errors_for_unknown_task() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::new("tasks", "build").unwrap(), None) - .unwrap(); - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - } -} - -mod run_target_if_touched { - use super::*; - - #[tokio::test] - async fn skips_if_untouched_project() { - let (_workspace, projects, _sandbox) = create_tasks_project_graph().await; - - let mut touched_files = FxHashSet::default(); - touched_files.insert(WorkspaceRelativePathBuf::from("input-a/a.ts")); - touched_files.insert(WorkspaceRelativePathBuf::from("input-c/c.ts")); - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::new("inputA", "a").unwrap(), Some(&touched_files)) - .unwrap(); - graph - .run_target(&Target::new("inputB", "b").unwrap(), Some(&touched_files)) - .unwrap(); - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - } - - #[tokio::test] - async fn skips_if_untouched_task() { - let (_workspace, projects, _sandbox) = create_tasks_project_graph().await; - - let mut touched_files = FxHashSet::default(); - touched_files.insert(WorkspaceRelativePathBuf::from("input-a/a2.ts")); - touched_files.insert(WorkspaceRelativePathBuf::from("input-b/b2.ts")); - touched_files.insert(WorkspaceRelativePathBuf::from("input-c/any.ts")); - - let mut graph = build_dep_graph(&projects); - graph - .run_target(&Target::new("inputA", "a").unwrap(), Some(&touched_files)) - .unwrap(); - graph - .run_target(&Target::new("inputB", "b2").unwrap(), Some(&touched_files)) - .unwrap(); - graph - .run_target(&Target::new("inputC", "c").unwrap(), Some(&touched_files)) - .unwrap(); - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - } -} - -mod sync_project { - use super::*; - use moon_dep_graph::DepGraphBuilder; - - fn sync_projects(graph: &mut DepGraphBuilder, projects: &ProjectGraph, ids: &[&str]) { - for id in ids { - let project = projects.get(id).unwrap(); - - graph.sync_project(&project).unwrap(); - } - } - - #[tokio::test] - async fn isolated_projects() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - let mut graph = build_dep_graph(&projects); - - sync_projects( - &mut graph, - &projects, - &["advanced", "basic", "emptyConfig", "noConfig"], - ); - - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - - assert_eq!( - graph.sort_topological().unwrap(), - vec![ - NodeIndex::new(0), - NodeIndex::new(1), // advanced - NodeIndex::new(2), // noConfig - NodeIndex::new(4), - NodeIndex::new(5), // basic - NodeIndex::new(3), // emptyConfig - NodeIndex::new(6), - ] - ); - assert_eq!( - sort_batches(graph.sort_batched_topological().unwrap()), - vec![ - vec![NodeIndex::new(4)], - vec![NodeIndex::new(1), NodeIndex::new(5)], - vec![ - NodeIndex::new(0), - NodeIndex::new(2), - NodeIndex::new(3), - NodeIndex::new(6) - ] - ] - ); - } - - #[tokio::test] - async fn projects_with_deps() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - let mut graph = build_dep_graph(&projects); - - sync_projects(&mut graph, &projects, &["foo", "bar", "baz", "basic"]); - - let graph = graph.build(); - - // Not deterministic! - // assert_snapshot!(graph.to_dot()); - - assert_eq!( - graph.sort_topological().unwrap(), - vec![ - NodeIndex::new(0), - NodeIndex::new(1), // baz - NodeIndex::new(3), // bar - NodeIndex::new(4), - NodeIndex::new(5), // foo - NodeIndex::new(2), // noConfig - NodeIndex::new(7), // basic - NodeIndex::new(6), - ] - ); - assert_eq!( - sort_batches(graph.sort_batched_topological().unwrap()), - vec![ - vec![NodeIndex::new(3)], - vec![ - NodeIndex::new(1), - NodeIndex::new(4), - NodeIndex::new(5), - NodeIndex::new(7) - ], - vec![NodeIndex::new(0), NodeIndex::new(2), NodeIndex::new(6)] - ] - ); - } - - #[tokio::test] - async fn projects_with_tasks() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - let mut graph = build_dep_graph(&projects); - - sync_projects(&mut graph, &projects, &["noConfig", "tasks"]); - - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - - assert_eq!( - graph.sort_topological().unwrap(), - vec![ - NodeIndex::new(0), - NodeIndex::new(1), // noConfig - NodeIndex::new(2), - NodeIndex::new(3), // tasks - NodeIndex::new(4), - ] - ); - assert_eq!( - sort_batches(graph.sort_batched_topological().unwrap()), - vec![ - vec![NodeIndex::new(1), NodeIndex::new(3)], - vec![NodeIndex::new(0), NodeIndex::new(2), NodeIndex::new(4)] - ] - ); - } - - #[tokio::test] - async fn avoids_dupe_projects() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - let mut graph = build_dep_graph(&projects); - - sync_projects(&mut graph, &projects, &["advanced", "advanced", "advanced"]); - - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - } - - #[tokio::test] - #[should_panic(expected = "No project has been configured with the name or alias unknown")] - async fn errors_for_unknown_project() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - let mut graph = build_dep_graph(&projects); - - sync_projects(&mut graph, &projects, &["unknown"]); - - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - } -} - -mod installs_deps { - use super::*; - - #[tokio::test] - async fn tool_is_based_on_task_platform() { - let (_workspace, projects, _sandbox) = create_project_graph().await; - let mut graph = build_dep_graph(&projects); - - graph - .run_target(&Target::new("platforms", "system").unwrap(), None) - .unwrap(); - - graph - .run_target(&Target::new("platforms", "node").unwrap(), None) - .unwrap(); - - let graph = graph.build(); - - assert_snapshot!(graph.to_dot()); - } -} diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__installs_deps__tool_is_based_on_task_platform.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__installs_deps__tool_is_based_on_task_platform.snap deleted file mode 100644 index 89abacad753..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__installs_deps__tool_is_based_on_task_platform.snap +++ /dev/null @@ -1,22 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupSystemTool" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallSystemDeps" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncSystemProject(platforms)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunTarget(platforms:system)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 6 [ label="InstallNodeDeps(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 7 [ label="RunTarget(platforms:node)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] - 6 -> 5 [ arrowhead=box, arrowtail=box] - 7 -> 6 [ arrowhead=box, arrowtail=box] - 7 -> 3 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__avoids_dupe_targets.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__avoids_dupe_targets.snap deleted file mode 100644 index ff64ce9d402..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__avoids_dupe_targets.snap +++ /dev/null @@ -1,16 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(tasks)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunTarget(tasks:lint)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__deps_chain_target.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__deps_chain_target.snap deleted file mode 100644 index d58328cca47..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__deps_chain_target.snap +++ /dev/null @@ -1,44 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(basic)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunTarget(basic:test)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="RunTarget(basic:lint)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 6 [ label="SyncNodeProject(chain)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 7 [ label="RunTarget(chain:a)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 8 [ label="RunTarget(chain:b)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 9 [ label="RunTarget(chain:c)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 10 [ label="RunTarget(chain:d)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 11 [ label="RunTarget(chain:e)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 12 [ label="RunTarget(chain:f)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] - 5 -> 2 [ arrowhead=box, arrowtail=box] - 5 -> 3 [ arrowhead=box, arrowtail=box] - 6 -> 1 [ arrowhead=box, arrowtail=box] - 7 -> 2 [ arrowhead=box, arrowtail=box] - 7 -> 6 [ arrowhead=box, arrowtail=box] - 8 -> 2 [ arrowhead=box, arrowtail=box] - 8 -> 6 [ arrowhead=box, arrowtail=box] - 9 -> 2 [ arrowhead=box, arrowtail=box] - 9 -> 6 [ arrowhead=box, arrowtail=box] - 10 -> 2 [ arrowhead=box, arrowtail=box] - 10 -> 6 [ arrowhead=box, arrowtail=box] - 11 -> 2 [ arrowhead=box, arrowtail=box] - 11 -> 6 [ arrowhead=box, arrowtail=box] - 12 -> 2 [ arrowhead=box, arrowtail=box] - 12 -> 6 [ arrowhead=box, arrowtail=box] - 11 -> 12 [ arrowhead=box, arrowtail=box] - 10 -> 11 [ arrowhead=box, arrowtail=box] - 9 -> 10 [ arrowhead=box, arrowtail=box] - 8 -> 9 [ arrowhead=box, arrowtail=box] - 7 -> 8 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__isolates_interactive_tasks.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__isolates_interactive_tasks.snap deleted file mode 100644 index 8fcb6cfd772..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__isolates_interactive_tasks.snap +++ /dev/null @@ -1,45 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(interactive)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunInteractiveTarget(interactive:one)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="SyncNodeProject(chain)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 6 [ label="RunTarget(chain:c)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 7 [ label="RunTarget(chain:d)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 8 [ label="RunTarget(chain:e)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 9 [ label="RunTarget(chain:f)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 10 [ label="RunInteractiveTarget(interactive:two)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 11 [ label="SyncNodeProject(basic)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 12 [ label="RunTarget(basic:lint)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 13 [ label="RunInteractiveTarget(interactive:depOnOne)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] - 5 -> 1 [ arrowhead=box, arrowtail=box] - 6 -> 2 [ arrowhead=box, arrowtail=box] - 6 -> 5 [ arrowhead=box, arrowtail=box] - 7 -> 2 [ arrowhead=box, arrowtail=box] - 7 -> 5 [ arrowhead=box, arrowtail=box] - 8 -> 2 [ arrowhead=box, arrowtail=box] - 8 -> 5 [ arrowhead=box, arrowtail=box] - 9 -> 2 [ arrowhead=box, arrowtail=box] - 9 -> 5 [ arrowhead=box, arrowtail=box] - 8 -> 9 [ arrowhead=box, arrowtail=box] - 7 -> 8 [ arrowhead=box, arrowtail=box] - 6 -> 7 [ arrowhead=box, arrowtail=box] - 10 -> 2 [ arrowhead=box, arrowtail=box] - 10 -> 3 [ arrowhead=box, arrowtail=box] - 11 -> 1 [ arrowhead=box, arrowtail=box] - 12 -> 2 [ arrowhead=box, arrowtail=box] - 12 -> 11 [ arrowhead=box, arrowtail=box] - 13 -> 2 [ arrowhead=box, arrowtail=box] - 13 -> 3 [ arrowhead=box, arrowtail=box] - 13 -> 4 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap deleted file mode 100644 index 2bf3c186a08..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__moves_persistent_tasks_last.snap +++ /dev/null @@ -1,54 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(persistent)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="SyncNodeProject(buildC)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="SyncNodeProject(buildA)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 6 [ label="SyncNodeProject(basic)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 7 [ label="SyncNodeProject(noTasks)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 8 [ label="RunPersistentTarget(persistent:dev)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 9 [ label="RunPersistentTarget(persistent:devMiddleman)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 10 [ label="RunPersistentTarget(persistent:local)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 11 [ label="RunTarget(buildA:build)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 12 [ label="RunTarget(basic:build)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 13 [ label="RunTarget(buildC:build)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 14 [ label="RunPersistentTarget(persistent:persistent)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 4 [ arrowhead=box, arrowtail=box] - 5 -> 1 [ arrowhead=box, arrowtail=box] - 5 -> 4 [ arrowhead=box, arrowtail=box] - 6 -> 1 [ arrowhead=box, arrowtail=box] - 5 -> 6 [ arrowhead=box, arrowtail=box] - 7 -> 1 [ arrowhead=box, arrowtail=box] - 5 -> 7 [ arrowhead=box, arrowtail=box] - 3 -> 5 [ arrowhead=box, arrowtail=box] - 8 -> 2 [ arrowhead=box, arrowtail=box] - 8 -> 3 [ arrowhead=box, arrowtail=box] - 9 -> 2 [ arrowhead=box, arrowtail=box] - 9 -> 3 [ arrowhead=box, arrowtail=box] - 10 -> 2 [ arrowhead=box, arrowtail=box] - 10 -> 3 [ arrowhead=box, arrowtail=box] - 11 -> 2 [ arrowhead=box, arrowtail=box] - 11 -> 5 [ arrowhead=box, arrowtail=box] - 12 -> 2 [ arrowhead=box, arrowtail=box] - 12 -> 6 [ arrowhead=box, arrowtail=box] - 13 -> 2 [ arrowhead=box, arrowtail=box] - 13 -> 4 [ arrowhead=box, arrowtail=box] - 11 -> 12 [ arrowhead=box, arrowtail=box] - 11 -> 13 [ arrowhead=box, arrowtail=box] - 10 -> 11 [ arrowhead=box, arrowtail=box] - 9 -> 10 [ arrowhead=box, arrowtail=box] - 14 -> 2 [ arrowhead=box, arrowtail=box] - 14 -> 3 [ arrowhead=box, arrowtail=box] - 8 -> 9 [ arrowhead=box, arrowtail=box] - 8 -> 14 [ arrowhead=box, arrowtail=box] - 8 -> 13 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__runs_all_projects_for_target_all_scope.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__runs_all_projects_for_target_all_scope.snap deleted file mode 100644 index d89757aa9a0..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__runs_all_projects_for_target_all_scope.snap +++ /dev/null @@ -1,38 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(buildA)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="SyncNodeProject(buildC)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="SyncNodeProject(basic)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 6 [ label="SyncNodeProject(noTasks)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 7 [ label="RunTarget(buildA:build)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 8 [ label="RunTarget(basic:build)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 9 [ label="RunTarget(buildC:build)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 10 [ label="SyncNodeProject(buildB)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 11 [ label="RunTarget(buildB:build)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 4 [ arrowhead=box, arrowtail=box] - 5 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 5 [ arrowhead=box, arrowtail=box] - 6 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 6 [ arrowhead=box, arrowtail=box] - 7 -> 2 [ arrowhead=box, arrowtail=box] - 7 -> 3 [ arrowhead=box, arrowtail=box] - 8 -> 2 [ arrowhead=box, arrowtail=box] - 8 -> 5 [ arrowhead=box, arrowtail=box] - 9 -> 2 [ arrowhead=box, arrowtail=box] - 9 -> 4 [ arrowhead=box, arrowtail=box] - 7 -> 8 [ arrowhead=box, arrowtail=box] - 7 -> 9 [ arrowhead=box, arrowtail=box] - 10 -> 1 [ arrowhead=box, arrowtail=box] - 11 -> 2 [ arrowhead=box, arrowtail=box] - 11 -> 10 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__single_targets.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__single_targets.snap deleted file mode 100644 index 474248a08a3..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target__single_targets.snap +++ /dev/null @@ -1,19 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(tasks)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunTarget(tasks:test)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="RunTarget(tasks:lint)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] - 5 -> 2 [ arrowhead=box, arrowtail=box] - 5 -> 3 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target_if_touched__skips_if_untouched_project.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target_if_touched__skips_if_untouched_project.snap deleted file mode 100644 index e6651f5ad65..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target_if_touched__skips_if_untouched_project.snap +++ /dev/null @@ -1,16 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(inputA)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunTarget(inputA:a)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target_if_touched__skips_if_untouched_task.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target_if_touched__skips_if_untouched_task.snap deleted file mode 100644 index 2bd726be5e4..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__run_target_if_touched__skips_if_untouched_task.snap +++ /dev/null @@ -1,21 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="InstallNodeDeps(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(inputB)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="RunTarget(inputB:b2)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="SyncNodeProject(inputC)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 6 [ label="RunTarget(inputC:c)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 2 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] - 5 -> 1 [ arrowhead=box, arrowtail=box] - 6 -> 2 [ arrowhead=box, arrowtail=box] - 6 -> 5 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__avoids_dupe_projects.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__avoids_dupe_projects.snap deleted file mode 100644 index d30c9e696f1..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__avoids_dupe_projects.snap +++ /dev/null @@ -1,11 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="SyncNodeProject(advanced)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__isolated_projects.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__isolated_projects.snap deleted file mode 100644 index 8f5e9c77c16..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__isolated_projects.snap +++ /dev/null @@ -1,19 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="SyncNodeProject(advanced)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SyncNodeProject(basic)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="SetupSystemTool" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 5 [ label="SyncSystemProject(noConfig)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 6 [ label="SyncSystemProject(emptyConfig)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 3 -> 1 [ arrowhead=box, arrowtail=box] - 5 -> 4 [ arrowhead=box, arrowtail=box] - 3 -> 5 [ arrowhead=box, arrowtail=box] - 6 -> 4 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__projects_with_tasks.snap b/crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__projects_with_tasks.snap deleted file mode 100644 index 0366366dcd1..00000000000 --- a/crates/core/dep-graph/tests/snapshots/dep_graph_test__sync_project__projects_with_tasks.snap +++ /dev/null @@ -1,14 +0,0 @@ ---- -source: crates/core/dep-graph/tests/dep_graph_test.rs -expression: graph.to_dot() ---- -digraph { - 0 [ label="SyncWorkspace" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 1 [ label="SetupSystemTool" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 [ label="SyncSystemProject(noConfig)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 3 [ label="SetupNodeTool(16.0.0)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 4 [ label="SyncNodeProject(tasks)" style=filled, shape=oval, fillcolor=gray, fontcolor=black] - 2 -> 1 [ arrowhead=box, arrowtail=box] - 4 -> 3 [ arrowhead=box, arrowtail=box] -} - diff --git a/crates/core/moon/Cargo.toml b/crates/core/moon/Cargo.toml index 1a59b7dec0d..0aad8788f05 100644 --- a/crates/core/moon/Cargo.toml +++ b/crates/core/moon/Cargo.toml @@ -5,9 +5,9 @@ edition = "2021" publish = false [dependencies] +moon_action_graph = { path = "../../../nextgen/action-graph" } moon_config = { path = "../../../nextgen/config" } moon_deno_platform = { path = "../../deno/platform" } -moon_dep_graph = { path = "../dep-graph" } moon_node_platform = { path = "../../node/platform" } moon_platform = { path = "../platform" } moon_platform_detector = { path = "../platform-detector" } diff --git a/crates/core/moon/src/lib.rs b/crates/core/moon/src/lib.rs index 133add9662a..0203e9c6bbc 100644 --- a/crates/core/moon/src/lib.rs +++ b/crates/core/moon/src/lib.rs @@ -1,6 +1,6 @@ +use moon_action_graph::ActionGraphBuilder; use moon_config::LanguageType; use moon_deno_platform::DenoPlatform; -use moon_dep_graph::DepGraphBuilder; use moon_node_platform::NodePlatform; use moon_platform::{PlatformManager, PlatformType}; use moon_platform_detector::{detect_project_language, detect_task_platform}; @@ -127,8 +127,8 @@ pub async fn load_workspace_with_toolchain() -> miette::Result { Ok(workspace) } -pub fn build_dep_graph(project_graph: &ProjectGraph) -> DepGraphBuilder { - DepGraphBuilder::new(project_graph) +pub fn build_action_graph(project_graph: &ProjectGraph) -> miette::Result { + ActionGraphBuilder::new(project_graph) } pub async fn create_project_graph_context(workspace: &Workspace) -> ProjectGraphBuilderContext { diff --git a/nextgen/action-graph/Cargo.toml b/nextgen/action-graph/Cargo.toml index 72b0af7abc1..f46608af2df 100644 --- a/nextgen/action-graph/Cargo.toml +++ b/nextgen/action-graph/Cargo.toml @@ -19,11 +19,12 @@ moon_query = { path = "../query" } miette = { workspace = true } petgraph = { workspace = true } rustc-hash = { workspace = true } +serde = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } tracing = { workspace = true } [dev-dependencies] moon_config = { path = "../config" } moon_test_utils2 = { path = "../test-utils" } starbase_sandbox = { workspace = true } -tokio = { workspace = true } diff --git a/nextgen/action-graph/src/action_graph.rs b/nextgen/action-graph/src/action_graph.rs index 7cb832ab873..7723bc352c5 100644 --- a/nextgen/action-graph/src/action_graph.rs +++ b/nextgen/action-graph/src/action_graph.rs @@ -4,56 +4,32 @@ use moon_common::is_test_env; use petgraph::dot::{Config, Dot}; use petgraph::prelude::*; use petgraph::visit::{IntoEdgeReferences, IntoNodeReferences}; -use rustc_hash::{FxHashMap, FxHashSet}; -use std::collections::VecDeque; +use rustc_hash::FxHashSet; +use std::sync::{mpsc, Arc, RwLock}; +use std::thread::spawn; +use tracing::{debug, trace}; pub type GraphType = DiGraph; -pub type IndicesMap = FxHashMap; pub struct ActionGraph { graph: GraphType, - indices: IndicesMap, - - // States when iterating - queue: VecDeque, - visited: FxHashSet, } impl ActionGraph { - pub fn new(graph: GraphType, indices: IndicesMap) -> Self { - ActionGraph { - graph, - indices, - queue: VecDeque::default(), - visited: FxHashSet::default(), - } - } + pub fn new(graph: GraphType) -> Self { + debug!("Creating action graph"); - pub fn reset_iterator(&mut self) -> miette::Result<()> { - self.detect_cycle()?; - - self.queue.clear(); - self.visited.clear(); - - // Extract root/initial nodes (those without edges) - self.queue.extend(self.graph.node_indices().filter(|&idx| { - self.graph - .neighbors_directed(idx, Outgoing) - .next() - .is_none() - })); + ActionGraph { graph } + } - Ok(()) + pub fn try_iter(&self) -> miette::Result { + ActionGraphIter::new(&self.graph) } pub fn is_empty(&self) -> bool { self.get_node_count() == 0 } - pub fn get_index_from_node(&self, node: &ActionNode) -> Option<&NodeIndex> { - self.indices.get(node) - } - pub fn get_node_count(&self) -> usize { self.graph.node_count() } @@ -62,6 +38,10 @@ impl ActionGraph { self.graph.node_weight(*index) } + pub fn labeled_graph(&self) -> DiGraph { + self.graph.map(|_, n| n.label(), |_, _| String::new()) + } + pub fn to_dot(&self) -> String { type DotGraph = DiGraph; @@ -98,52 +78,107 @@ impl ActionGraph { format!("{dot:?}") } +} - pub fn to_labeled_graph(&self) -> DiGraph { - self.graph.map(|_, n| n.label(), |_, _| String::new()) - } +pub struct ActionGraphIter<'graph> { + graph: &'graph GraphType, + indices: Vec, + visited: FxHashSet, + completed: Arc>>, - fn detect_cycle(&self) -> miette::Result<()> { - if self.get_node_count() > 1 { - if let Err(cycle) = petgraph::algo::toposort(&self.graph, None) { - return Err(ActionGraphError::CycleDetected( - self.get_node_from_index(&cycle.node_id()) - .map(|n| n.label()) - .unwrap_or_else(|| "(unknown)".into()), - ) - .into()); + pub receiver: Option>, + pub sender: mpsc::Sender, +} + +impl<'graph> ActionGraphIter<'graph> { + pub fn new(graph: &'graph GraphType) -> miette::Result { + match petgraph::algo::toposort(graph, None) { + Ok(mut indices) => { + indices.reverse(); + + debug!( + order = ?indices.iter().map(|i| i.index()).collect::>(), + "Iterating action graph topologically", + ); + + let (sender, receiver) = mpsc::channel(); + + Ok(Self { + graph, + indices, + visited: FxHashSet::default(), + completed: Arc::new(RwLock::new(FxHashSet::default())), + receiver: Some(receiver), + sender, + }) } + Err(cycle) => Err(ActionGraphError::CycleDetected( + graph + .node_weight(cycle.node_id()) + .map(|n| n.label()) + .unwrap_or_else(|| "(unknown)".into()), + ) + .into()), } + } + + pub fn has_pending(&self) -> bool { + self.completed.read().unwrap().len() < self.graph.node_count() + } - Ok(()) + pub fn mark_completed(&mut self, index: NodeIndex) { + self.completed.write().unwrap().insert(index); + } + + pub fn monitor_completed(&mut self) { + let completed = Arc::clone(&self.completed); + let receiver = self.receiver.take().unwrap(); + + spawn(move || { + while let Ok(idx) = receiver.recv() { + let index = NodeIndex::new(idx); + + trace!(index = index.index(), "Marking action as complete"); + + completed.write().unwrap().insert(index); + } + }); } } // This is based on the `Topo` struct from petgraph! -impl Iterator for ActionGraph { - type Item = ActionNode; +impl<'graph> Iterator for ActionGraphIter<'graph> { + type Item = NodeIndex; fn next(&mut self) -> Option { - while let Some(idx) = self.queue.pop_front() { - if self.visited.contains(&idx) { + let completed = self.completed.read().unwrap(); + + for idx in &self.indices { + if self.visited.contains(idx) || completed.contains(idx) { continue; } - self.visited.insert(idx); - - for neighbor in self.graph.neighbors_directed(idx, Direction::Incoming) { - // Look at each neighbor, and those that only have incoming edges - // from the already ordered list, they are the next to visit. - if self - .graph - .neighbors_directed(neighbor, Direction::Outgoing) - .all(|b| self.visited.contains(&b)) - { - self.queue.push_back(neighbor); - } + // Ensure all dependencies of the index have completed + let mut deps = vec![]; + + if self + .graph + .neighbors_directed(*idx, Direction::Outgoing) + .all(|dep| { + deps.push(dep.index()); + completed.contains(&dep) + }) + { + self.visited.insert(*idx); + + trace!( + index = idx.index(), + deps = ?deps, + "Running action", + ); + + return Some(*idx); } - - return self.graph.node_weight(idx).map(|n| n.to_owned()); } None diff --git a/nextgen/action-graph/src/action_graph_builder.rs b/nextgen/action-graph/src/action_graph_builder.rs index ee51775871d..3dba99fe2c6 100644 --- a/nextgen/action-graph/src/action_graph_builder.rs +++ b/nextgen/action-graph/src/action_graph_builder.rs @@ -13,9 +13,8 @@ use tracing::{debug, trace}; type TouchedFilePaths = FxHashSet; pub struct ActionGraphBuilder<'app> { - pub include_dependents: bool, - all_query: Option, + dependents: bool, graph: DiGraph, indices: FxHashMap, platform_manager: &'app PlatformManager, @@ -31,10 +30,12 @@ impl<'app> ActionGraphBuilder<'app> { platform_manager: &'app PlatformManager, project_graph: &'app ProjectGraph, ) -> miette::Result { + debug!("Building action graph"); + Ok(ActionGraphBuilder { all_query: None, graph: DiGraph::new(), - include_dependents: false, + dependents: false, indices: FxHashMap::default(), platform_manager, project_graph, @@ -42,7 +43,7 @@ impl<'app> ActionGraphBuilder<'app> { } pub fn build(self) -> miette::Result { - Ok(ActionGraph::new(self.graph, self.indices)) + Ok(ActionGraph::new(self.graph)) } pub fn get_index_from_node(&self, node: &ActionNode) -> Option<&NodeIndex> { @@ -69,6 +70,10 @@ impl<'app> ActionGraphBuilder<'app> { Runtime::system() } + pub fn include_dependents(&mut self) { + self.dependents = true; + } + pub fn set_query(&mut self, input: &str) -> miette::Result<()> { self.all_query = Some(build_query(input)?); @@ -145,11 +150,6 @@ impl<'app> ActionGraphBuilder<'app> { // Compare against touched files if provided if let Some(touched) = touched_files { if !task.is_affected(touched)? { - trace!( - "Task {} not affected based on touched files, skipping", - color::label(&task.target), - ); - return Ok(None); } } @@ -168,17 +168,18 @@ impl<'app> ActionGraphBuilder<'app> { // And we also need to create edges for task dependencies if !task.deps.is_empty() { trace!( + task = task.target.as_str(), deps = ?task.deps.iter().map(|d| d.as_str()).collect::>(), - "Adding dependencies for task {}", - color::label(&task.target), + "Linking dependencies for task", ); + reqs.extend(self.run_task_dependencies(task)?); } self.link_requirements(index, reqs); // And possibly dependents - if self.include_dependents { + if self.dependents { self.run_task_dependents(task)?; } @@ -226,6 +227,10 @@ impl<'app> ActionGraphBuilder<'app> { // From self project for dep_task in project.tasks.values() { if dep_task.deps.contains(&task.target) { + if dep_task.is_persistent() { + continue; + } + if let Some(index) = self.run_task(&project, dep_task, None)? { indices.push(index); } @@ -237,6 +242,10 @@ impl<'app> ActionGraphBuilder<'app> { let dep_project = self.project_graph.get(dependent_id)?; for dep_task in dep_project.tasks.values() { + if dep_task.is_persistent() { + continue; + } + if dep_task.deps.contains(&task.target) { if let Some(index) = self.run_task(&dep_project, dep_task, None)? { indices.push(index); @@ -394,7 +403,7 @@ impl<'app> ActionGraphBuilder<'app> { fn link_requirements(&mut self, index: NodeIndex, reqs: Vec) { trace!( index = index.index(), - requires = ?reqs, + requires = ?reqs.iter().map(|i| i.index()).collect::>(), "Linking requirements for index" ); diff --git a/nextgen/action-graph/src/action_node.rs b/nextgen/action-graph/src/action_node.rs index a691c58de60..365e13ffde5 100644 --- a/nextgen/action-graph/src/action_node.rs +++ b/nextgen/action-graph/src/action_node.rs @@ -1,9 +1,10 @@ use moon_common::Id; use moon_platform_runtime::Runtime; use moon_task::Target; +use serde::Serialize; use std::hash::{Hash, Hasher}; -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] pub enum ActionNode { /// Install tool dependencies in the workspace root. InstallDeps { runtime: Runtime }, diff --git a/nextgen/action-graph/tests/action_graph_test.rs b/nextgen/action-graph/tests/action_graph_test.rs index de0b9e52d44..9bf27f8eee7 100644 --- a/nextgen/action-graph/tests/action_graph_test.rs +++ b/nextgen/action-graph/tests/action_graph_test.rs @@ -39,13 +39,15 @@ fn create_rust_runtime() -> Runtime { ) } -fn topo(mut graph: ActionGraph) -> Vec { +fn topo(graph: ActionGraph) -> Vec { let mut nodes = vec![]; + let mut iter = graph.try_iter().unwrap(); - graph.reset_iterator().unwrap(); - - for node in graph { - nodes.push(node); + while iter.has_pending() { + if let Some(index) = iter.next() { + nodes.push(graph.get_node_from_index(&index).unwrap().to_owned()); + iter.mark_completed(index); + } } nodes @@ -70,7 +72,7 @@ mod action_graph { .run_task(&project, project.get_task("cycle2").unwrap(), None) .unwrap(); - builder.build().unwrap().reset_iterator().unwrap(); + builder.build().unwrap().try_iter().unwrap(); } mod install_deps { @@ -151,11 +153,11 @@ mod action_graph { ActionNode::SetupTool { runtime: create_node_runtime() }, - ActionNode::InstallProjectDeps { - project: Id::raw("out"), + ActionNode::InstallDeps { runtime: create_node_runtime() }, - ActionNode::InstallDeps { + ActionNode::InstallProjectDeps { + project: Id::raw("out"), runtime: create_node_runtime() }, ] @@ -165,7 +167,6 @@ mod action_graph { mod run_task { use super::*; - use starbase_sandbox::pretty_assertions::assert_eq; #[tokio::test] async fn graphs() { @@ -190,11 +191,11 @@ mod action_graph { ActionNode::SetupTool { runtime: create_node_runtime() }, - ActionNode::SyncProject { - project: Id::raw("bar"), + ActionNode::InstallDeps { runtime: create_node_runtime() }, - ActionNode::InstallDeps { + ActionNode::SyncProject { + project: Id::raw("bar"), runtime: create_node_runtime() }, ActionNode::RunTask { @@ -231,11 +232,11 @@ mod action_graph { ActionNode::SetupTool { runtime: create_node_runtime() }, - ActionNode::SyncProject { - project: Id::raw("bar"), + ActionNode::InstallDeps { runtime: create_node_runtime() }, - ActionNode::InstallDeps { + ActionNode::SyncProject { + project: Id::raw("bar"), runtime: create_node_runtime() }, ActionNode::RunTask { @@ -314,18 +315,18 @@ mod action_graph { vec![ ActionNode::SyncWorkspace, ActionNode::SetupTool { - runtime: create_node_runtime() + runtime: create_rust_runtime() }, - ActionNode::SetupTool { + ActionNode::InstallDeps { runtime: create_rust_runtime() }, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, ActionNode::SyncProject { project: Id::raw("bar"), runtime: create_node_runtime() }, - ActionNode::InstallDeps { - runtime: create_rust_runtime() - }, ActionNode::RunTask { interactive: false, persistent: false, @@ -462,7 +463,7 @@ mod action_graph { let container = ActionGraphContainer::new(sandbox.path()).await; let mut builder = container.create_builder(); - builder.include_dependents = true; + builder.include_dependents(); let project = container.project_graph.get("deps").unwrap(); let task = project.get_task("base").unwrap(); @@ -703,8 +704,8 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, - ActionNode::SetupTool { runtime: node }, ActionNode::SetupTool { runtime: system }, + ActionNode::SetupTool { runtime: node }, ] ); } @@ -735,9 +736,9 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, - ActionNode::SetupTool { runtime: node3 }, - ActionNode::SetupTool { runtime: node2 }, ActionNode::SetupTool { runtime: node1 }, + ActionNode::SetupTool { runtime: node2 }, + ActionNode::SetupTool { runtime: node3 }, ] ); } @@ -847,15 +848,15 @@ mod action_graph { runtime: Runtime::system() }, ActionNode::SyncProject { - project: Id::raw("qux"), + project: Id::raw("bar"), runtime: Runtime::system() }, ActionNode::SyncProject { - project: Id::raw("bar"), + project: Id::raw("foo"), runtime: Runtime::system() }, ActionNode::SyncProject { - project: Id::raw("foo"), + project: Id::raw("qux"), runtime: Runtime::system() }, ] @@ -914,19 +915,19 @@ mod action_graph { vec![ ActionNode::SyncWorkspace, ActionNode::SetupTool { - runtime: create_rust_runtime() + runtime: create_node_runtime() }, - ActionNode::SetupTool { + ActionNode::SyncProject { + project: Id::raw("bar"), runtime: create_node_runtime() }, + ActionNode::SetupTool { + runtime: create_rust_runtime() + }, ActionNode::SyncProject { project: Id::raw("qux"), runtime: create_rust_runtime() }, - ActionNode::SyncProject { - project: Id::raw("bar"), - runtime: create_node_runtime() - } ] ); } @@ -950,15 +951,19 @@ mod action_graph { topo(graph), vec![ ActionNode::SyncWorkspace, + ActionNode::SetupTool { + runtime: create_node_runtime() + }, + ActionNode::SyncProject { + project: Id::raw("bar"), + runtime: create_node_runtime() + }, ActionNode::SetupTool { runtime: Runtime::new_override( PlatformType::Node, RuntimeReq::with_version(Version::new(18, 0, 0)) ) }, - ActionNode::SetupTool { - runtime: create_node_runtime() - }, ActionNode::SyncProject { project: Id::raw("baz"), runtime: Runtime::new_override( @@ -966,10 +971,6 @@ mod action_graph { RuntimeReq::with_version(Version::new(18, 0, 0)) ) }, - ActionNode::SyncProject { - project: Id::raw("bar"), - runtime: create_node_runtime() - }, ] ); } diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 6cdc6a341b4..1bfbbd6bd5c 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -44,11 +44,11 @@ arbitrarily. - Cleaned up dependency chains between actions, greatly reducing the number of nodes in the graph. - Renamed `RunTarget` to `RunTask`, including interactive and persistent variants. -- Updated the action graph to iterate using a topological queue, which executes ready-to-run actions - in the thread pool. Previously, we would sort topologically _into batches_, which worked, but - resulted in many threads uselessly waiting for an action to run, which was blocked waiting for the - current batch to complete. - - For large graphs, this should result in a significant performance improvement, upwards of 10x. +- Updated the action graph to iterate using a topological queue, which executes actions on-demand in + the thread pool when they are ready (dependencies have been met). Previously, we would sort + topologically _into batches_, which worked, but resulted in many threads uselessly waiting for an + action to run, which was blocked waiting for the current batch to complete. + - For large graphs, this should result in a significant performance improvement. - Persistent tasks will still be ran as a batch, but since it's the last operation, it's fine. - Released a new GitHub action, [`moonrepo/setup-toolchain`](https://github.com/marketplace/actions/setup-proto-and-moon-toolchains), @@ -57,6 +57,7 @@ #### 🚀 Updates - Added a `moon action-graph` command and deprecated `moon dep-graph`. +- Added a `--dependents` argument to `moon action-graph`. #### 🐞 Fixes diff --git a/website/docs/guides/webhooks.mdx b/website/docs/guides/webhooks.mdx index 00296fb6697..095f6461cbe 100644 --- a/website/docs/guides/webhooks.mdx +++ b/website/docs/guides/webhooks.mdx @@ -409,7 +409,8 @@ Triggered when a [target](../concepts/target) has finished running. If the run f field will be set with the error message. For more information about the action, refer to the [`action.finished`](#actionfinished) event. Ran -targets can be scoped with the `RunTarget(...)` and `RunPersistentTarget(...)` labels. +targets can be scoped with the `RunTask(...)`, `RunInteractiveTask(...)`, and +`RunPersistentTask(...)` labels. ```json {