From ae171b3feeb8905cd49e078eeed156e9a54f7cdb Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Wed, 2 Oct 2024 13:40:35 -0700 Subject: [PATCH] internal: Add plugin host data and sync workspace support. (#1655) * Add host data. * Update proto/extism. * Fix host funcs. * Add wasm bridge. * Remove deps. * Rename config. * Add sync workspace. * Fix format. * Improve affected. --- CHANGELOG.md | 2 + Cargo.lock | 45 +++--- .../action-graph/src/action_graph_builder.rs | 33 ++++- crates/action-pipeline/Cargo.toml | 1 + crates/action-pipeline/src/action_pipeline.rs | 10 +- crates/action-pipeline/src/action_runner.rs | 11 +- crates/action-pipeline/src/job.rs | 1 + crates/action-pipeline/src/job_context.rs | 4 + crates/action/src/operation.rs | 21 ++- crates/action/src/operation_list.rs | 1 + crates/action/src/operation_meta.rs | 5 + crates/actions/Cargo.toml | 3 + crates/actions/src/actions/sync_workspace.rs | 66 +++++++-- crates/actions/src/operations/mod.rs | 2 + .../src/operations/run_plugin_operation.rs | 32 +++++ crates/affected/src/affected_tracker.rs | 18 +-- crates/app/src/commands/ext.rs | 2 +- crates/app/src/commands/setup.rs | 2 +- crates/app/src/components.rs | 7 +- crates/app/src/session.rs | 26 ++-- crates/common/src/env.rs | 1 - crates/common/src/lib.rs | 1 + crates/common/src/serde.rs | 15 ++ crates/config/src/toolchain_config.rs | 8 +- .../src/extension_registry.rs | 12 +- crates/pdk-api/src/toolchain.rs | 36 ++--- crates/pdk-test-utils/src/lib.rs | 5 +- crates/plugin/Cargo.toml | 4 + crates/plugin/src/host.rs | 135 ++++++++++++++++++ crates/plugin/src/lib.rs | 2 + crates/plugin/src/plugin_registry.rs | 96 +++++++------ crates/plugin/tests/plugin_registry_test.rs | 11 +- crates/project/src/project.rs | 6 +- crates/target/src/target.rs | 8 +- crates/task/src/task.rs | 9 ++ crates/task/src/task_options.rs | 11 ++ .../toolchain-plugin/src/toolchain_plugin.rs | 17 ++- .../src/toolchain_registry.rs | 28 ++-- packages/types/src/project-config.ts | 2 +- 39 files changed, 527 insertions(+), 172 deletions(-) create mode 100644 crates/actions/src/operations/run_plugin_operation.rs create mode 100644 crates/common/src/serde.rs create mode 100644 crates/plugin/src/host.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eb90ff0c2a..6d486849151 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ - Added `--upstream` and `--downstream` options to `moon query projects`. - Coming soon for affected tasks as well! - Added a new task option, `cacheLifetime`, that controls how long a task will be cached for. +- Added "sync workspace action" support to toolchain plugins. This is our first step in supporting + toolchains via WASM plugins. - Updated task `outputs` to support token and environment variables. - Updated `moon query projects` to include the project description as a trailing value. - Updated `moon query tasks` to include the task type and platform, and the task description as a diff --git a/Cargo.lock b/Cargo.lock index 83e73d402f4..4cbf1607568 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -691,9 +691,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.22" +version = "1.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" +checksum = "3bbb537bb4a30b90362caddba8f360c0a56bc13d3a5570028e7197204cb54a17" dependencies = [ "jobserver", "libc", @@ -2057,9 +2057,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -2858,6 +2858,7 @@ dependencies = [ "moon_project_graph", "moon_task", "moon_toolchain", + "moon_toolchain_plugin", "num_cpus", "petgraph", "rustc-hash 2.0.0", @@ -2881,11 +2882,14 @@ dependencies = [ "moon_config", "moon_config_schema", "moon_hash", + "moon_pdk_api", "moon_platform", + "moon_process", "moon_project", "moon_project_graph", "moon_task_runner", "moon_time", + "moon_toolchain_plugin", "moon_vcs_hooks", "proto_core", "rustc-hash 2.0.0", @@ -3589,9 +3593,13 @@ version = "0.0.1" dependencies = [ "async-trait", "convert_case", + "extism", "miette", + "moon_common", "moon_env", "moon_pdk_api", + "moon_project_graph", + "moon_target", "proto_core", "scc", "serde_json", @@ -4891,9 +4899,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] @@ -5140,11 +5148,10 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] @@ -5640,9 +5647,9 @@ dependencies = [ [[package]] name = "starbase_macros" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27e0c7c62ceb4d73477855732b6727f01eabb78a13debfccf1bc1633938d7e2" +checksum = "c27886f95de94ee99af3c3f8fb3dd6e4d0bd12f02ff445a9ba32226ccb64ac40" dependencies = [ "darling", "proc-macro2", @@ -6665,9 +6672,9 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.217.0" +version = "0.218.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b88b0814c9a2b323a9b46c687e726996c255ac8b64aa237dd11c81ed4854760" +checksum = "22b896fa8ceb71091ace9bcb81e853f54043183a1c9667cf93422c40252ffa0a" dependencies = [ "leb128", ] @@ -6977,24 +6984,24 @@ dependencies = [ [[package]] name = "wast" -version = "217.0.0" +version = "218.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79004ecebded92d3c710d4841383368c7f04b63d0992ddd6b0c7d5029b7629b7" +checksum = "8a53cd1f0fa505df97557e36a58bddb8296e2fcdcd089529545ebfdb18a1b9d7" dependencies = [ "bumpalo", "leb128", "memchr", "unicode-width", - "wasm-encoder 0.217.0", + "wasm-encoder 0.218.0", ] [[package]] name = "wat" -version = "1.217.0" +version = "1.218.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c126271c3d92ca0f7c63e4e462e40c69cca52fd4245fcda730d1cf558fb55088" +checksum = "4f87f8e14e776762e07927c27c2054d2cf678aab9aae2d431a79b3e31e4dd391" dependencies = [ - "wast 217.0.0", + "wast 218.0.0", ] [[package]] diff --git a/crates/action-graph/src/action_graph_builder.rs b/crates/action-graph/src/action_graph_builder.rs index 04e19ad2e09..7db46f22482 100644 --- a/crates/action-graph/src/action_graph_builder.rs +++ b/crates/action-graph/src/action_graph_builder.rs @@ -27,6 +27,10 @@ pub struct RunRequirements { pub interactive: bool, pub skip_affected: bool, // Temporary until we support task dependents properly pub target_locators: FxHashSet, + + // Used to mark affected states + pub upstream_target: Option, + pub downstream_target: Option, } impl RunRequirements { @@ -273,7 +277,7 @@ impl<'app> ActionGraphBuilder<'app> { // but only if this task was affected if reqs.dependents { if let Some(affected) = &mut self.affected { - debug!( + trace!( task = task.target.as_str(), "But will run all dependent tasks if affected" ); @@ -324,8 +328,17 @@ impl<'app> ActionGraphBuilder<'app> { } // Compare against touched files if provided - if !reqs.skip_affected { - if let Some(affected) = &mut self.affected { + if let Some(affected) = &mut self.affected { + if let Some(downstream) = &reqs.downstream_target { + affected + .mark_task_affected(task, AffectedBy::DownstreamTask(downstream.to_owned()))?; + } + + if let Some(upstream) = &reqs.upstream_target { + affected.mark_task_affected(task, AffectedBy::UpstreamTask(upstream.to_owned()))?; + } + + if !reqs.skip_affected { match affected.is_task_affected(task)? { Some(by) => { affected.mark_task_affected(task, by)?; @@ -376,7 +389,14 @@ impl<'app> ActionGraphBuilder<'app> { #[instrument(skip_all)] pub fn run_task_dependencies(&mut self, task: &Task) -> miette::Result> { let parallel = task.options.run_deps_in_parallel; - let reqs = RunRequirements::default(); + let reqs = RunRequirements { + downstream_target: if self.affected.is_some() { + Some(task.target.clone()) + } else { + None + }, + ..Default::default() + }; let mut indices = vec![]; let mut previous_target_index = None; @@ -422,6 +442,11 @@ impl<'app> ActionGraphBuilder<'app> { ci: parent_reqs.ci, interactive: parent_reqs.interactive, skip_affected: true, + upstream_target: if self.affected.is_some() { + Some(task.target.clone()) + } else { + None + }, ..Default::default() }; diff --git a/crates/action-pipeline/Cargo.toml b/crates/action-pipeline/Cargo.toml index 8046dd730b8..b276bb5274a 100644 --- a/crates/action-pipeline/Cargo.toml +++ b/crates/action-pipeline/Cargo.toml @@ -20,6 +20,7 @@ moon_project = { path = "../project" } moon_project_graph = { path = "../project-graph" } moon_task = { path = "../task" } moon_toolchain = { path = "../toolchain" } +moon_toolchain_plugin = { path = "../toolchain-plugin" } async-trait = { workspace = true } miette = { workspace = true } num_cpus = "1.16.0" diff --git a/crates/action-pipeline/src/action_pipeline.rs b/crates/action-pipeline/src/action_pipeline.rs index b25801ae60a..73ab7afaf87 100644 --- a/crates/action-pipeline/src/action_pipeline.rs +++ b/crates/action-pipeline/src/action_pipeline.rs @@ -14,6 +14,7 @@ use moon_api::Moonbase; use moon_app_context::AppContext; use moon_common::{color, is_ci, is_test_env}; use moon_project_graph::ProjectGraph; +use moon_toolchain_plugin::ToolchainRegistry; use rustc_hash::{FxHashMap, FxHashSet}; use std::mem; use std::sync::Arc; @@ -39,10 +40,15 @@ pub struct ActionPipeline { action_context: Arc, emitter: Arc, project_graph: Arc, + toolchain_registry: Arc, } impl ActionPipeline { - pub fn new(app_context: Arc, project_graph: Arc) -> Self { + pub fn new( + app_context: Arc, + project_graph: Arc, + toolchain_registry: Arc, + ) -> Self { debug!("Creating pipeline to run actions"); Self { @@ -57,6 +63,7 @@ impl ActionPipeline { project_graph, report_name: "runReport.json".into(), summarize: false, + toolchain_registry, } } @@ -145,6 +152,7 @@ impl ActionPipeline { result_sender: sender, semaphore: Arc::new(Semaphore::new(self.concurrency)), running_jobs: Arc::new(RwLock::new(FxHashMap::default())), + toolchain_registry: Arc::clone(&self.toolchain_registry), }; // Monitor signals and ctrl+c diff --git a/crates/action-pipeline/src/action_runner.rs b/crates/action-pipeline/src/action_runner.rs index c1cc8e79a54..fb6bfcc681b 100644 --- a/crates/action-pipeline/src/action_runner.rs +++ b/crates/action-pipeline/src/action_runner.rs @@ -5,6 +5,7 @@ use moon_actions::actions::*; use moon_app_context::AppContext; use moon_common::color; use moon_project_graph::ProjectGraph; +use moon_toolchain_plugin::ToolchainRegistry; use std::sync::Arc; use tracing::{instrument, trace}; @@ -14,6 +15,7 @@ pub async fn run_action( action_context: Arc, app_context: Arc, project_graph: Arc, + toolchain_registry: Arc, emitter: Arc, ) -> miette::Result<()> { action.start(); @@ -36,7 +38,14 @@ pub async fn run_action( ActionNode::SyncWorkspace => { emitter.emit(Event::WorkspaceSyncing).await?; - let result = sync_workspace(action, action_context, app_context, project_graph).await; + let result = sync_workspace( + action, + action_context, + app_context, + project_graph, + toolchain_registry, + ) + .await; emitter .emit(Event::WorkspaceSynced { diff --git a/crates/action-pipeline/src/job.rs b/crates/action-pipeline/src/job.rs index 575e07d0b3a..e935cc6f2ab 100644 --- a/crates/action-pipeline/src/job.rs +++ b/crates/action-pipeline/src/job.rs @@ -73,6 +73,7 @@ impl Job { self.action_context, self.app_context, self.context.project_graph.clone(), + self.context.toolchain_registry.clone(), self.context.emitter.clone(), ) => {}, }; diff --git a/crates/action-pipeline/src/job_context.rs b/crates/action-pipeline/src/job_context.rs index 090e3560bc5..f4b983f1759 100644 --- a/crates/action-pipeline/src/job_context.rs +++ b/crates/action-pipeline/src/job_context.rs @@ -1,6 +1,7 @@ use crate::event_emitter::EventEmitter; use moon_action::Action; use moon_project_graph::ProjectGraph; +use moon_toolchain_plugin::ToolchainRegistry; use petgraph::graph::NodeIndex; use rustc_hash::{FxHashMap, FxHashSet}; use std::sync::Arc; @@ -32,6 +33,9 @@ pub struct JobContext { /// Acquires a permit for concurrency pub semaphore: Arc, + + /// The registry of all toolchain plugins + pub toolchain_registry: Arc, } impl JobContext { diff --git a/crates/action/src/operation.rs b/crates/action/src/operation.rs index 307728ecd1c..191b104b807 100644 --- a/crates/action/src/operation.rs +++ b/crates/action/src/operation.rs @@ -51,18 +51,18 @@ impl Operation { pub fn get_output(&self) -> Option<&OperationMetaOutput> { match &self.meta { - OperationMeta::OutputHydration(output) | OperationMeta::TaskExecution(output) => { - Some(output) - } + OperationMeta::OutputHydration(output) + | OperationMeta::ProcessExecution(output) + | OperationMeta::TaskExecution(output) => Some(output), _ => None, } } pub fn get_output_mut(&mut self) -> Option<&mut OperationMetaOutput> { match &mut self.meta { - OperationMeta::OutputHydration(output) | OperationMeta::TaskExecution(output) => { - Some(output) - } + OperationMeta::OutputHydration(output) + | OperationMeta::ProcessExecution(output) + | OperationMeta::TaskExecution(output) => Some(output), _ => None, } } @@ -216,6 +216,15 @@ impl Operation { Self::new(OperationMeta::OutputHydration(Default::default())) } + pub fn process_execution(command: impl AsRef) -> Self { + Self::new(OperationMeta::ProcessExecution(Box::new( + OperationMetaOutput { + command: Some(command.as_ref().to_owned()), + ..Default::default() + }, + ))) + } + pub fn sync_operation(label: impl AsRef) -> Self { Self::new(OperationMeta::SyncOperation(Box::new(OperationMetaLabel { label: label.as_ref().to_owned(), diff --git a/crates/action/src/operation_list.rs b/crates/action/src/operation_list.rs index 3e59e70f752..492db3e8581 100644 --- a/crates/action/src/operation_list.rs +++ b/crates/action/src/operation_list.rs @@ -38,6 +38,7 @@ impl OperationList { self.0.iter().rfind(|op| { op.meta.is_no_operation() || op.meta.is_output_hydration() + || op.meta.is_process_execution() || op.meta.is_sync_operation() || op.meta.is_task_execution() }) diff --git a/crates/action/src/operation_meta.rs b/crates/action/src/operation_meta.rs index b90ceb444ea..d5032ae7de0 100644 --- a/crates/action/src/operation_meta.rs +++ b/crates/action/src/operation_meta.rs @@ -56,6 +56,7 @@ pub enum OperationMeta { #[default] NoOperation, OutputHydration(Box), + ProcessExecution(Box), SyncOperation(Box), TaskExecution(Box), @@ -86,6 +87,10 @@ impl OperationMeta { matches!(self, Self::OutputHydration(_)) } + pub fn is_process_execution(&self) -> bool { + matches!(self, Self::ProcessExecution(_)) + } + pub fn is_sync_operation(&self) -> bool { matches!(self, Self::SyncOperation(_)) } diff --git a/crates/actions/Cargo.toml b/crates/actions/Cargo.toml index 74c0fec28c2..d474275f3b2 100644 --- a/crates/actions/Cargo.toml +++ b/crates/actions/Cargo.toml @@ -14,10 +14,13 @@ moon_common = { path = "../common" } moon_config = { path = "../config" } moon_config_schema = { path = "../config-schema" } moon_hash = { path = "../hash" } +moon_pdk_api = { path = "../pdk-api" } +moon_process = { path = "../process" } moon_project = { path = "../project" } moon_project_graph = { path = "../project-graph" } moon_task_runner = { path = "../task-runner" } moon_time = { path = "../time" } +moon_toolchain_plugin = { path = "../toolchain-plugin" } moon_vcs_hooks = { path = "../vcs-hooks" } miette = { workspace = true } proto_core = { workspace = true } diff --git a/crates/actions/src/actions/sync_workspace.rs b/crates/actions/src/actions/sync_workspace.rs index 63e0af36f4f..47d7127431a 100644 --- a/crates/actions/src/actions/sync_workspace.rs +++ b/crates/actions/src/actions/sync_workspace.rs @@ -1,4 +1,6 @@ -use crate::operations::{sync_codeowners, sync_config_schemas, sync_vcs_hooks}; +use crate::operations::{ + run_plugin_operation, sync_codeowners, sync_config_schemas, sync_vcs_hooks, +}; use crate::utils::should_skip_action; use miette::IntoDiagnostic; use moon_action::{Action, ActionStatus, Operation}; @@ -6,6 +8,7 @@ use moon_action_context::ActionContext; use moon_app_context::AppContext; use moon_common::color; use moon_project_graph::ProjectGraph; +use moon_toolchain_plugin::ToolchainRegistry; use std::sync::Arc; use tokio::task; use tracing::{debug, instrument}; @@ -16,6 +19,7 @@ pub async fn sync_workspace( _action_context: Arc, app_context: Arc, project_graph: Arc, + toolchain_registry: Arc, ) -> miette::Result { if should_skip_action("MOON_SKIP_SYNC_WORKSPACE").is_some() { debug!( @@ -29,20 +33,22 @@ pub async fn sync_workspace( debug!("Syncing workspace"); // Run operations in parallel - let mut futures = vec![]; + let mut operation_futures: Vec>>> = vec![]; { debug!("Syncing config schemas"); let app_context = Arc::clone(&app_context); - futures.push(task::spawn(async move { - Operation::sync_operation("Config schemas") + operation_futures.push(task::spawn(async move { + let op = Operation::sync_operation("Config schemas") .track_async_with_check( || sync_config_schemas(&app_context, false), |result| result, ) - .await + .await?; + + Ok(vec![op]) })); } @@ -55,13 +61,15 @@ pub async fn sync_workspace( let app_context = Arc::clone(&app_context); let project_graph = Arc::clone(&project_graph); - futures.push(task::spawn(async move { - Operation::sync_operation("Codeowners") + operation_futures.push(task::spawn(async move { + let op = Operation::sync_operation("Codeowners") .track_async_with_check( || sync_codeowners(&app_context, &project_graph, false), |result| result.is_some(), ) - .await + .await?; + + Ok(vec![op]) })); } @@ -74,15 +82,47 @@ pub async fn sync_workspace( let app_context = Arc::clone(&app_context); - futures.push(task::spawn(async move { - Operation::sync_operation("VCS hooks") + operation_futures.push(task::spawn(async move { + let op = Operation::sync_operation("VCS hooks") .track_async_with_check(|| sync_vcs_hooks(&app_context, false), |result| result) - .await + .await?; + + Ok(vec![op]) })); } - for future in futures { - action.operations.push(future.await.into_diagnostic()??); + if toolchain_registry.has_plugins() { + debug!("Syncing operations from toolchains"); + + let mut sync_results = vec![]; + let sync_context = toolchain_registry.create_context(); + + for plugin_id in toolchain_registry.get_plugin_ids() { + if let Some(result) = toolchain_registry + .load(plugin_id) + .await? + .sync_workspace(sync_context.clone()) + .await? + { + sync_results.push(result); + } + } + + for result in sync_results { + operation_futures.push(task::spawn(async move { + let mut ops = vec![]; + + for op in result.operations { + ops.push(run_plugin_operation(op).await?); + } + + Ok(ops) + })); + } + } + + for future in operation_futures { + action.operations.extend(future.await.into_diagnostic()??); } Ok(ActionStatus::Passed) diff --git a/crates/actions/src/operations/mod.rs b/crates/actions/src/operations/mod.rs index 9e7ea3b9eb0..ba58adca39f 100644 --- a/crates/actions/src/operations/mod.rs +++ b/crates/actions/src/operations/mod.rs @@ -1,7 +1,9 @@ +mod run_plugin_operation; mod sync_codeowners; mod sync_config_schemas; mod sync_vcs_hooks; +pub use run_plugin_operation::*; pub use sync_codeowners::*; pub use sync_config_schemas::*; pub use sync_vcs_hooks::*; diff --git a/crates/actions/src/operations/run_plugin_operation.rs b/crates/actions/src/operations/run_plugin_operation.rs new file mode 100644 index 00000000000..5a8a6af9dac --- /dev/null +++ b/crates/actions/src/operations/run_plugin_operation.rs @@ -0,0 +1,32 @@ +use moon_action::Operation; +use moon_pdk_api::Operation as PluginOperation; +use moon_process::Command; + +pub async fn run_plugin_operation(operation: PluginOperation) -> miette::Result { + match operation { + PluginOperation::ProcessExecution(process) => { + Operation::process_execution(&process.command) + .track_async_with_check( + || async { + let mut builder = Command::new(process.command); + builder.args(process.args); + builder.envs(process.env); + + if let Some(cwd) = process.working_dir.and_then(|cwd| cwd.real_path()) { + builder.cwd(cwd); + } + + let mut command = builder.create_async(); + + if process.stream { + command.exec_stream_output().await + } else { + command.exec_capture_output().await + } + }, + |result| result.status.success(), + ) + .await + } + } +} diff --git a/crates/affected/src/affected_tracker.rs b/crates/affected/src/affected_tracker.rs index 9938a0c1612..81d5d5610b7 100644 --- a/crates/affected/src/affected_tracker.rs +++ b/crates/affected/src/affected_tracker.rs @@ -51,11 +51,11 @@ impl<'app> AffectedTracker<'app> { let state = AffectedProjectState::from(list); debug!( - by_files = ?state.files.iter().collect::>(), - by_upstream = ?state.upstream.iter().map(|id| id.as_str()).collect::>(), - by_downstream = ?state.downstream.iter().map(|id| id.as_str()).collect::>(), + files = ?state.files.iter().collect::>(), + upstream = ?state.upstream.iter().map(|id| id.as_str()).collect::>(), + downstream = ?state.downstream.iter().map(|id| id.as_str()).collect::>(), other = state.other, - "Project {} is affected", color::id(&id), + "Project {} is affected by", color::id(&id), ); affected.projects.insert(id, state); @@ -65,12 +65,12 @@ impl<'app> AffectedTracker<'app> { let state = AffectedTaskState::from(list); debug!( - by_env = ?state.env.iter().collect::>(), - by_files = ?state.files.iter().collect::>(), - by_upstream = ?state.upstream.iter().map(|target| target.as_str()).collect::>(), - by_downstream = ?state.downstream.iter().map(|target| target.as_str()).collect::>(), + env = ?state.env.iter().collect::>(), + files = ?state.files.iter().collect::>(), + upstream = ?state.upstream.iter().map(|target| target.as_str()).collect::>(), + downstream = ?state.downstream.iter().map(|target| target.as_str()).collect::>(), other = state.other, - "Task {} is affected", color::label(&target), + "Task {} is affected by", color::label(&target), ); affected.tasks.insert(target, state); diff --git a/crates/app/src/commands/ext.rs b/crates/app/src/commands/ext.rs index 13e675f77b7..b0b038848fb 100644 --- a/crates/app/src/commands/ext.rs +++ b/crates/app/src/commands/ext.rs @@ -23,7 +23,7 @@ pub async fn ext(session: CliSession, args: ExtArgs) -> AppResult { } let id = PluginId::raw(&args.id); - let extensions = session.get_extension_registry()?; + let extensions = session.get_extension_registry().await?; // Load the plugin let plugin = extensions.load(&id).await?; diff --git a/crates/app/src/commands/setup.rs b/crates/app/src/commands/setup.rs index 5dd7ebb6942..7844f5cbb53 100644 --- a/crates/app/src/commands/setup.rs +++ b/crates/app/src/commands/setup.rs @@ -8,7 +8,7 @@ use tracing::instrument; pub async fn setup(session: CliSession) -> AppResult { let done = create_progress_bar("Downloading and installing tools..."); - analyze::load_toolchain(session.get_toolchain_registry()?).await?; + analyze::load_toolchain(session.get_toolchain_registry().await?).await?; done("Setup complete", true); diff --git a/crates/app/src/components.rs b/crates/app/src/components.rs index fee1f0db2e5..18a04a2ff99 100644 --- a/crates/app/src/components.rs +++ b/crates/app/src/components.rs @@ -19,7 +19,12 @@ pub async fn run_action_pipeline( action_graph: ActionGraph, ) -> miette::Result> { let project_graph = session.get_project_graph().await?; - let mut pipeline = ActionPipeline::new(session.get_app_context()?, project_graph); + let toolchain_registry = session.get_toolchain_registry().await?; + let mut pipeline = ActionPipeline::new( + session.get_app_context()?, + project_graph, + toolchain_registry, + ); if let Some(concurrency) = &session.cli.concurrency { pipeline.concurrency = *concurrency; diff --git a/crates/app/src/session.rs b/crates/app/src/session.rs index 1100c1b2099..37dcf50a1bf 100644 --- a/crates/app/src/session.rs +++ b/crates/app/src/session.rs @@ -13,7 +13,7 @@ use moon_console::Console; use moon_console_reporter::DefaultReporter; use moon_env::MoonEnvironment; use moon_extension_plugin::*; -use moon_plugin::PluginId; +use moon_plugin::{PluginHostData, PluginId}; use moon_project_graph::{ProjectGraph, ProjectGraphBuilder}; use moon_toolchain_plugin::*; use moon_vcs::{BoxedVcs, Git}; @@ -116,10 +116,15 @@ impl CliSession { Ok(Arc::new(self.console.clone())) } - pub fn get_extension_registry(&self) -> AppResult> { + pub async fn get_extension_registry(&self) -> AppResult> { + let project_graph = self.get_project_graph().await?; + let item = self.extension_registry.get_or_init(|| { - let mut registry = - ExtensionRegistry::new(Arc::clone(&self.moon_env), Arc::clone(&self.proto_env)); + let mut registry = ExtensionRegistry::new(PluginHostData { + moon_env: Arc::clone(&self.moon_env), + project_graph, + proto_env: Arc::clone(&self.proto_env), + }); // Convert moon IDs to plugin IDs for (id, config) in self.workspace_config.extensions.clone() { @@ -147,10 +152,15 @@ impl CliSession { Ok(graph) } - pub fn get_toolchain_registry(&self) -> AppResult> { + pub async fn get_toolchain_registry(&self) -> AppResult> { + let project_graph = self.get_project_graph().await?; + let item = self.toolchain_registry.get_or_init(|| { - let mut registry = - ToolchainRegistry::new(Arc::clone(&self.moon_env), Arc::clone(&self.proto_env)); + let mut registry = ToolchainRegistry::new(PluginHostData { + moon_env: Arc::clone(&self.moon_env), + project_graph, + proto_env: Arc::clone(&self.proto_env), + }); // Convert moon IDs to plugin IDs for (id, config) in self.toolchain_config.toolchains.clone() { @@ -269,7 +279,7 @@ impl AppSession for CliSession { .await?; if self.requires_toolchain_installed() { - analyze::load_toolchain(self.get_toolchain_registry()?).await?; + analyze::load_toolchain(self.get_toolchain_registry().await?).await?; } } diff --git a/crates/common/src/env.rs b/crates/common/src/env.rs index 97f74c9f737..31b552a3880 100644 --- a/crates/common/src/env.rs +++ b/crates/common/src/env.rs @@ -52,7 +52,6 @@ pub fn is_wsl() -> bool { }) } -#[inline] pub fn is_test_env() -> bool { static TEST_CACHE: OnceLock = OnceLock::new(); diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index e0088b396f6..744d10dd98f 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -4,6 +4,7 @@ mod env; mod id; mod macros; pub mod path; +pub mod serde; #[cfg(not(target_arch = "wasm32"))] pub use env::*; diff --git a/crates/common/src/serde.rs b/crates/common/src/serde.rs new file mode 100644 index 00000000000..a20cfc0981a --- /dev/null +++ b/crates/common/src/serde.rs @@ -0,0 +1,15 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +static BRIDGE_ENABLED: AtomicBool = AtomicBool::new(false); + +pub fn enable_wasm_bridge() { + BRIDGE_ENABLED.store(true, Ordering::Release); +} + +pub fn disable_wasm_bridge() { + BRIDGE_ENABLED.store(true, Ordering::Release); +} + +pub fn is_wasm_bridge(_: &T) -> bool { + BRIDGE_ENABLED.load(Ordering::Acquire) +} diff --git a/crates/config/src/toolchain_config.rs b/crates/config/src/toolchain_config.rs index 9092f22d53c..b0dbcc6ce25 100644 --- a/crates/config/src/toolchain_config.rs +++ b/crates/config/src/toolchain_config.rs @@ -14,15 +14,15 @@ use std::path::Path; #[cfg(feature = "proto")] use crate::{inherit_tool, inherit_tool_without_version, is_using_tool_version}; -/// Configures an individual platform. +/// Configures an individual toolchain. #[derive(Clone, Config, Debug, PartialEq)] #[config(allow_unknown_fields)] -pub struct ToolConfig { +pub struct ToolchainPluginConfig { /// Location of the WASM plugin to use. #[setting(required)] pub plugin: Option, - /// The version to download, install, and run tasks with. + /// The version of the toolchain to download and install. pub version: Option, /// Arbitrary configuration that'll be passed to the WASM plugin. @@ -68,7 +68,7 @@ pub struct ToolchainConfig { /// All configured toolchains by unique ID. #[setting(flatten, nested)] - pub toolchains: FxHashMap, + pub toolchains: FxHashMap, } impl ToolchainConfig { diff --git a/crates/extension-plugin/src/extension_registry.rs b/crates/extension-plugin/src/extension_registry.rs index e6ac63b817c..4d0b60783b2 100644 --- a/crates/extension-plugin/src/extension_registry.rs +++ b/crates/extension-plugin/src/extension_registry.rs @@ -1,8 +1,8 @@ use crate::extension_plugin::ExtensionPlugin; use moon_config::ExtensionConfig; use moon_plugin::{ - serialize_config, MoonEnvironment, PluginError, PluginId, PluginInstance, PluginRegistry, - PluginType, ProtoEnvironment, + serialize_config, PluginError, PluginHostData, PluginId, PluginInstance, PluginRegistry, + PluginType, }; use rustc_hash::FxHashMap; use std::ops::Deref; @@ -16,14 +16,10 @@ pub struct ExtensionRegistry { } impl ExtensionRegistry { - pub fn new(moon_env: Arc, proto_env: Arc) -> Self { + pub fn new(host_data: PluginHostData) -> Self { Self { configs: FxHashMap::default(), - registry: Arc::new(PluginRegistry::new( - PluginType::Extension, - moon_env, - proto_env, - )), + registry: Arc::new(PluginRegistry::new(PluginType::Extension, host_data)), } } diff --git a/crates/pdk-api/src/toolchain.rs b/crates/pdk-api/src/toolchain.rs index e7bf10eb1b9..0f7f11e2999 100644 --- a/crates/pdk-api/src/toolchain.rs +++ b/crates/pdk-api/src/toolchain.rs @@ -1,9 +1,6 @@ use crate::common::*; -use moon_common::Id; -use moon_config::{DependencyConfig, ProjectConfig}; -use rustc_hash::FxHashMap; use schematic::Schema; -use warpgate_api::{api_struct, VirtualPath}; +use warpgate_api::api_struct; // METADATA @@ -47,26 +44,13 @@ api_struct!( // SYNC PROJECT -api_struct!( - pub struct SyncProjectRecord { - pub config: ProjectConfig, - pub dependencies: Vec, - pub id: Id, - pub root: VirtualPath, - pub source: String, - } -); - -api_struct!( - /// Input passed to the `sync_project` function. - pub struct SyncProjectInput { - /// Current moon context. - pub context: MoonContext, - - /// Other projects that the project being synced depends on. - pub dependencies: FxHashMap, +// api_struct!( +// /// Input passed to the `sync_project` function. +// pub struct SyncProjectInput { +// /// Current moon context. +// pub context: MoonContext, - /// The project being synced. - pub project: SyncProjectRecord, - } -); +// /// Other projects that the project being synced depends on. +// // pub dependencies: FxHashMap, +// } +// ); diff --git a/crates/pdk-test-utils/src/lib.rs b/crates/pdk-test-utils/src/lib.rs index f326c7bcdf3..0771a2547e5 100644 --- a/crates/pdk-test-utils/src/lib.rs +++ b/crates/pdk-test-utils/src/lib.rs @@ -46,9 +46,8 @@ pub fn create_plugin_container_with_config( manifest = manifest.with_allowed_host("*"); manifest = manifest.with_allowed_paths( virtual_paths - .clone() - .into_iter() - .map(|(key, value)| (key.to_string_lossy().to_string(), value)), + .iter() + .map(|(key, value)| (key.to_string_lossy().to_string(), value.to_owned())), ); inject_default_manifest_config(&id, &sandbox.join(".home"), &mut manifest).unwrap(); diff --git a/crates/plugin/Cargo.toml b/crates/plugin/Cargo.toml index 682def59578..09483b94f23 100644 --- a/crates/plugin/Cargo.toml +++ b/crates/plugin/Cargo.toml @@ -9,10 +9,14 @@ repository = "https://github.com/moonrepo/moon" publish = false [dependencies] +moon_common = { path = "../common" } moon_env = { path = "../env" } moon_pdk_api = { version = "0.0.10", path = "../pdk-api" } +moon_project_graph = { path = "../project-graph" } +moon_target = { path = "../target" } async-trait = { workspace = true } convert_case = "0.6.0" +extism = { workspace = true } miette = { workspace = true } proto_core = { workspace = true } scc = { workspace = true } diff --git a/crates/plugin/src/host.rs b/crates/plugin/src/host.rs new file mode 100644 index 00000000000..d239301a139 --- /dev/null +++ b/crates/plugin/src/host.rs @@ -0,0 +1,135 @@ +use extism::{CurrentPlugin, Error, Function, UserData, Val, ValType}; +use moon_common::{color, serde::*, Id}; +use moon_env::MoonEnvironment; +use moon_project_graph::ProjectGraph; +use moon_target::{Target, TargetScope}; +use proto_core::ProtoEnvironment; +use std::fmt; +use std::sync::Arc; +use tracing::{instrument, trace}; +use warpgate::host::{create_host_functions as create_shared_host_functions, HostData}; + +#[derive(Clone, Default)] +pub struct PluginHostData { + pub moon_env: Arc, + pub project_graph: Arc, + pub proto_env: Arc, +} + +impl fmt::Debug for PluginHostData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PluginRegistry") + .field("moon_env", &self.moon_env) + .field("proto_env", &self.proto_env) + .finish() + } +} + +pub fn create_host_functions(data: PluginHostData, shared_data: HostData) -> Vec { + let mut functions = vec![]; + functions.extend(create_shared_host_functions(shared_data)); + functions.extend(vec![ + Function::new( + "load_project", + [ValType::I64], + [ValType::I64], + UserData::new(data.clone()), + load_project, + ), + Function::new( + "load_task", + [ValType::I64], + [ValType::I64], + UserData::new(data), + load_task, + ), + ]); + functions +} + +fn map_error(error: miette::Report) -> Error { + Error::msg(error.to_string()) +} + +#[instrument(name = "host_load_project", skip_all)] +fn load_project( + plugin: &mut CurrentPlugin, + inputs: &[Val], + outputs: &mut [Val], + user_data: UserData, +) -> Result<(), Error> { + let id_raw: String = plugin.memory_get_val(&inputs[0])?; + let id = Id::new(id_raw)?; + let uuid = plugin.id().to_string(); + + trace!( + plugin = &uuid, + project_id = id.as_str(), + "Calling host function {}", + color::label("load_project"), + ); + + let data = user_data.get()?; + let data = data.lock().unwrap(); + let project = data.project_graph.get(&id).map_err(map_error)?; + + trace!( + plugin = &uuid, + project_id = id.as_str(), + "Called host function {}", + color::label("load_project"), + ); + + enable_wasm_bridge(); + + plugin.memory_set_val(&mut outputs[0], serde_json::to_string(&project)?)?; + + disable_wasm_bridge(); + + Ok(()) +} + +#[instrument(name = "host_load_task", skip_all)] +fn load_task( + plugin: &mut CurrentPlugin, + inputs: &[Val], + outputs: &mut [Val], + user_data: UserData, +) -> Result<(), Error> { + let target_raw: String = plugin.memory_get_val(&inputs[0])?; + let target = Target::parse(&target_raw).map_err(map_error)?; + let uuid = plugin.id().to_string(); + + trace!( + plugin = &uuid, + target = target.as_str(), + "Calling host function {}", + color::label("load_task"), + ); + + let TargetScope::Project(project_id) = &target.scope else { + return Err(Error::msg( + "Unable to load task. Requires a fully-qualified target with a project scope.", + )); + }; + + let data = user_data.get()?; + let data = data.lock().unwrap(); + let project = data.project_graph.get(project_id).map_err(map_error)?; + let task = project.get_task(&target.task_id).map_err(map_error)?; + + trace!( + plugin = &uuid, + target = target.as_str(), + "Called host function {}", + color::label("load_task"), + ); + + enable_wasm_bridge(); + + plugin.memory_set_val(&mut outputs[0], serde_json::to_string(task)?)?; + + disable_wasm_bridge(); + + Ok(()) +} diff --git a/crates/plugin/src/lib.rs b/crates/plugin/src/lib.rs index 3ec554e6c93..105626e5876 100644 --- a/crates/plugin/src/lib.rs +++ b/crates/plugin/src/lib.rs @@ -1,7 +1,9 @@ +mod host; mod plugin; mod plugin_error; mod plugin_registry; +pub use host::*; pub use moon_env::MoonEnvironment; pub use plugin::*; pub use plugin_error::*; diff --git a/crates/plugin/src/plugin_registry.rs b/crates/plugin/src/plugin_registry.rs index f7ea7fe14b9..e482f8e9bb9 100644 --- a/crates/plugin/src/plugin_registry.rs +++ b/crates/plugin/src/plugin_registry.rs @@ -1,58 +1,50 @@ +use crate::host::*; use crate::plugin::*; use crate::plugin_error::PluginError; -use moon_env::MoonEnvironment; use moon_pdk_api::MoonContext; -use proto_core::{is_offline, ProtoEnvironment}; +use proto_core::is_offline; use scc::hash_map::OccupiedEntry; -use scc::HashMap; use starbase_utils::fs; use std::fmt; -use std::fmt::Debug; use std::ops::{Deref, DerefMut}; use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; use tracing::{debug, instrument}; use warpgate::{ - host::*, inject_default_manifest_config, to_virtual_path, Id, PluginContainer, PluginLoader, - PluginLocator, PluginManifest, Wasm, + host::HostData, inject_default_manifest_config, to_virtual_path, Id, PluginContainer, + PluginLoader, PluginLocator, PluginManifest, Wasm, }; #[allow(dead_code)] pub struct PluginRegistry { - pub moon_env: Arc, - pub proto_env: Arc, + pub host_data: PluginHostData, loader: PluginLoader, - plugins: Arc>, + plugins: Arc>, type_of: PluginType, virtual_paths: BTreeMap, } impl PluginRegistry { - pub fn new( - type_of: PluginType, - moon_env: Arc, - proto_env: Arc, - ) -> Self { + pub fn new(type_of: PluginType, host_data: PluginHostData) -> Self { debug!(plugin = type_of.get_label(), "Creating plugin registry"); // Create the loader let mut loader = PluginLoader::new( - moon_env.plugins_dir.join(type_of.get_dir_name()), - &moon_env.temp_dir, + host_data.moon_env.plugins_dir.join(type_of.get_dir_name()), + &host_data.moon_env.temp_dir, ); loader.set_offline_checker(is_offline); // Merge proto and moon virtual paths let mut paths = BTreeMap::new(); - paths.extend(proto_env.get_virtual_paths()); - paths.extend(moon_env.get_virtual_paths()); + paths.extend(host_data.proto_env.get_virtual_paths()); + paths.extend(host_data.moon_env.get_virtual_paths()); Self { loader, - plugins: Arc::new(HashMap::default()), - moon_env, - proto_env, + plugins: Arc::new(scc::HashMap::default()), + host_data, type_of, virtual_paths: paths, } @@ -60,10 +52,13 @@ impl PluginRegistry { pub fn create_context(&self) -> MoonContext { MoonContext { - working_dir: to_virtual_path(self.get_virtual_paths(), &self.moon_env.working_dir), + working_dir: to_virtual_path( + self.get_virtual_paths(), + &self.host_data.moon_env.working_dir, + ), workspace_root: to_virtual_path( self.get_virtual_paths(), - &self.moon_env.workspace_root, + &self.host_data.moon_env.workspace_root, ), } } @@ -85,9 +80,8 @@ impl PluginRegistry { // Inherit moon and proto virtual paths. manifest = manifest.with_allowed_paths( self.virtual_paths - .clone() - .into_iter() - .map(|(key, value)| (key.to_string_lossy().to_string(), value)), + .iter() + .map(|(key, value)| (key.to_string_lossy().to_string(), value.to_owned())), ); // Disable timeouts as some functions, like dependency installs, @@ -96,7 +90,7 @@ impl PluginRegistry { manifest.timeout_ms = None; // Inherit default configs, like host environment and ID. - inject_default_manifest_config(id, &self.moon_env.home, &mut manifest)?; + inject_default_manifest_config(id, &self.host_data.moon_env.home, &mut manifest)?; // Ensure virtual host paths exist, otherwise WASI (via extism) // will throw a cryptic file/directory not found error. @@ -107,7 +101,7 @@ impl PluginRegistry { Ok(manifest) } - pub fn get_cache(&self) -> Arc> { + pub fn get_cache(&self) -> Arc> { Arc::clone(&self.plugins) } @@ -131,6 +125,23 @@ impl PluginRegistry { self.plugins.contains(id) } + pub async fn load(&self, id: I) -> miette::Result> + where + I: AsRef + fmt::Debug, + { + let id = id.as_ref(); + + if self.is_registered(id) { + return self.get_instance(id).await; + } + + Err(PluginError::UnknownId { + id: id.to_string(), + ty: self.type_of, + } + .into()) + } + #[instrument(skip(self, op))] pub async fn load_with_config( &self, @@ -139,8 +150,8 @@ impl PluginRegistry { mut op: F, ) -> miette::Result<()> where - I: AsRef + Debug, - L: AsRef + Debug, + I: AsRef + fmt::Debug, + L: AsRef + fmt::Debug, F: FnMut(&mut PluginManifest) -> miette::Result<()>, { let id = id.as_ref(); @@ -163,11 +174,14 @@ impl PluginRegistry { let plugin_file = self.loader.load_plugin(id, locator).await?; // Create host functions (provided by warpgate) - let functions = create_host_functions(HostData { - http_client: self.loader.get_client()?.clone(), - virtual_paths: self.virtual_paths.clone(), - working_dir: self.moon_env.working_dir.clone(), - }); + let functions = create_host_functions( + self.host_data.clone(), + HostData { + http_client: self.loader.get_client()?.clone(), + virtual_paths: self.virtual_paths.clone(), + working_dir: self.host_data.moon_env.working_dir.clone(), + }, + ); // Create the manifest and let the consumer configure it let mut manifest = self.create_manifest(id, plugin_file)?; @@ -186,8 +200,8 @@ impl PluginRegistry { T::new(PluginRegistration { container: PluginContainer::new(id.to_owned(), manifest, functions)?, id: id.to_owned(), - moon_env: Arc::clone(&self.moon_env), - proto_env: Arc::clone(&self.proto_env), + moon_env: Arc::clone(&self.host_data.moon_env), + proto_env: Arc::clone(&self.host_data.proto_env), }) .await?, ); @@ -197,8 +211,8 @@ impl PluginRegistry { pub async fn load_without_config(&self, id: I, locator: L) -> miette::Result<()> where - I: AsRef + Debug, - L: AsRef + Debug, + I: AsRef + fmt::Debug, + L: AsRef + fmt::Debug, { self.load_with_config(id, locator, |_| Ok(())).await } @@ -217,14 +231,14 @@ impl PluginRegistry { impl fmt::Debug for PluginRegistry { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("PluginRegistry") - .field("moon_env", &self.moon_env) - .field("proto_env", &self.proto_env) + .field("host_data", &self.host_data) .field("plugins", &self.plugins) .field("type_of", &self.type_of) .field("virtual_paths", &self.virtual_paths) .finish() } } + pub struct PluginInstance<'l, T: Plugin> { entry: OccupiedEntry<'l, Id, T>, } diff --git a/crates/plugin/tests/plugin_registry_test.rs b/crates/plugin/tests/plugin_registry_test.rs index d73594ee478..a6b6dd84a40 100644 --- a/crates/plugin/tests/plugin_registry_test.rs +++ b/crates/plugin/tests/plugin_registry_test.rs @@ -1,8 +1,10 @@ use async_trait::async_trait; use moon_env::MoonEnvironment; use moon_plugin::{ - Plugin, PluginId as Id, PluginLocator, PluginRegistration, PluginRegistry, PluginType, + Plugin, PluginHostData, PluginId as Id, PluginLocator, PluginRegistration, PluginRegistry, + PluginType, }; +use moon_project_graph::ProjectGraph; use proto_core::{warpgate::FileLocator, ProtoEnvironment}; use starbase_sandbox::{create_empty_sandbox, create_sandbox}; use std::fs; @@ -26,8 +28,11 @@ impl Plugin for TestPlugin { fn create_registry(sandbox: &Path) -> PluginRegistry { let registry = PluginRegistry::new( PluginType::Extension, - Arc::new(MoonEnvironment::new_testing(sandbox)), - Arc::new(ProtoEnvironment::new_testing(sandbox).unwrap()), + PluginHostData { + moon_env: Arc::new(MoonEnvironment::new_testing(sandbox)), + project_graph: Arc::new(ProjectGraph::default()), + proto_env: Arc::new(ProtoEnvironment::new_testing(sandbox).unwrap()), + }, ); // These must exist or extism errors diff --git a/crates/project/src/project.rs b/crates/project/src/project.rs index 0c0aa8f5c89..65c471f28cb 100644 --- a/crates/project/src/project.rs +++ b/crates/project/src/project.rs @@ -1,5 +1,5 @@ use crate::project_error::ProjectError; -use moon_common::{cacheable, path::WorkspaceRelativePathBuf, Id}; +use moon_common::{cacheable, path::WorkspaceRelativePathBuf, serde::is_wasm_bridge, Id}; use moon_config::{ DependencyConfig, InheritedTasksResult, LanguageType, PlatformType, ProjectConfig, ProjectType, StackType, @@ -11,12 +11,15 @@ use std::path::PathBuf; cacheable!( #[derive(Clone, Debug, Default)] + #[serde(default)] pub struct Project { /// Unique alias of the project, alongside its official ID. /// This is typically for language specific semantics, like `name` from `package.json`. + #[serde(skip_serializing_if = "Option::is_none")] pub alias: Option, /// Project configuration loaded from "moon.yml", if it exists. + #[serde(skip_serializing_if = "is_wasm_bridge")] pub config: ProjectConfig, /// List of other projects this project depends on. @@ -29,6 +32,7 @@ cacheable!( pub id: Id, /// Task configuration that was inherited from ".moon/tasks". + #[serde(skip_serializing_if = "is_wasm_bridge")] pub inherited: Option, /// Primary programming language of the project. diff --git a/crates/target/src/target.rs b/crates/target/src/target.rs index ac1da7b6ea9..ba60106f180 100644 --- a/crates/target/src/target.rs +++ b/crates/target/src/target.rs @@ -18,7 +18,7 @@ pub static TARGET_PATTERN: Lazy = Lazy::new(|| { .unwrap() }); -#[derive(Clone, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Eq, Hash, PartialEq)] pub struct Target { pub id: CompactString, pub scope: TargetScope, @@ -163,6 +163,12 @@ impl Default for Target { } } +impl fmt::Debug for Target { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.id) + } +} + impl fmt::Display for Target { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.id) diff --git a/crates/task/src/task.rs b/crates/task/src/task.rs index 1569137ff7a..2b847edd326 100644 --- a/crates/task/src/task.rs +++ b/crates/task/src/task.rs @@ -32,6 +32,7 @@ cacheable!( cacheable!( #[derive(Clone, Debug, Default, Eq, PartialEq)] + #[serde(default)] pub struct Task { pub args: Vec, @@ -39,6 +40,7 @@ cacheable!( pub deps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, pub env: FxHashMap, @@ -47,10 +49,13 @@ cacheable!( pub inputs: Vec, + #[serde(skip_serializing_if = "FxHashSet::is_empty")] pub input_env: FxHashSet, + #[serde(skip_serializing_if = "FxHashSet::is_empty")] pub input_files: FxHashSet, + #[serde(skip_serializing_if = "FxHashSet::is_empty")] pub input_globs: FxHashSet, pub metadata: TaskMetadata, @@ -59,14 +64,18 @@ cacheable!( pub outputs: Vec, + #[serde(skip_serializing_if = "FxHashSet::is_empty")] pub output_files: FxHashSet, + #[serde(skip_serializing_if = "FxHashSet::is_empty")] pub output_globs: FxHashSet, pub platform: PlatformType, + #[serde(skip_serializing_if = "Option::is_none")] pub preset: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub script: Option, pub target: Target, diff --git a/crates/task/src/task_options.rs b/crates/task/src/task_options.rs index 3dcb8bbb6c3..883ae807b6f 100644 --- a/crates/task/src/task_options.rs +++ b/crates/task/src/task_options.rs @@ -6,7 +6,9 @@ use moon_config::{ cacheable!( #[derive(Clone, Debug, Eq, PartialEq)] + #[serde(default)] pub struct TaskOptions { + #[serde(skip_serializing_if = "Option::is_none")] pub affected_files: Option, pub affected_pass_inputs: bool, @@ -15,8 +17,10 @@ cacheable!( pub cache: bool, + #[serde(skip_serializing_if = "Option::is_none")] pub cache_lifetime: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub env_files: Option>, pub internal: bool, @@ -33,10 +37,13 @@ cacheable!( pub merge_outputs: TaskMergeStrategy, + #[serde(skip_serializing_if = "Option::is_none")] pub mutex: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub os: Option>, + #[serde(skip_serializing_if = "Option::is_none")] pub output_style: Option, pub persistent: bool, @@ -50,12 +57,16 @@ cacheable!( pub run_from_workspace_root: bool, + #[serde(skip_serializing_if = "Option::is_none")] pub shell: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub unix_shell: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub windows_shell: Option, } ); diff --git a/crates/toolchain-plugin/src/toolchain_plugin.rs b/crates/toolchain-plugin/src/toolchain_plugin.rs index 024249ce3d1..ea48a26b851 100644 --- a/crates/toolchain-plugin/src/toolchain_plugin.rs +++ b/crates/toolchain-plugin/src/toolchain_plugin.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use moon_pdk_api::{ - MoonContext, SyncWorkspaceInput, ToolchainMetadataInput, ToolchainMetadataOutput, + MoonContext, SyncWorkspaceInput, SyncWorkspaceOutput, ToolchainMetadataInput, + ToolchainMetadataOutput, }; use moon_plugin::{Plugin, PluginContainer, PluginId, PluginRegistration, PluginType}; use proto_core::Tool; @@ -20,18 +21,22 @@ pub struct ToolchainPlugin { impl ToolchainPlugin { #[instrument(skip_all)] - pub async fn sync_workspace(&self, context: MoonContext) -> miette::Result<()> { + pub async fn sync_workspace( + &self, + context: MoonContext, + ) -> miette::Result> { if !self.plugin.has_func("sync_workspace").await { - return Ok(()); + return Ok(None); } debug!(toolchain = self.id.as_str(), "Syncing workspace"); - self.plugin - .call_func_without_output("sync_workspace", SyncWorkspaceInput { context }) + let output: SyncWorkspaceOutput = self + .plugin + .call_func_with("sync_workspace", SyncWorkspaceInput { context }) .await?; - Ok(()) + Ok(Some(output)) } // #[instrument(skip_all)] diff --git a/crates/toolchain-plugin/src/toolchain_registry.rs b/crates/toolchain-plugin/src/toolchain_registry.rs index 46742114fa1..c4a4d4e310e 100644 --- a/crates/toolchain-plugin/src/toolchain_registry.rs +++ b/crates/toolchain-plugin/src/toolchain_registry.rs @@ -1,9 +1,7 @@ use crate::toolchain_plugin::ToolchainPlugin; use miette::IntoDiagnostic; -use moon_config::ToolConfig; -use moon_plugin::{ - serialize_config, MoonEnvironment, PluginId, PluginRegistry, PluginType, ProtoEnvironment, -}; +use moon_config::ToolchainPluginConfig; +use moon_plugin::{serialize_config, PluginHostData, PluginId, PluginRegistry, PluginType}; use proto_core::inject_proto_manifest_config; use rustc_hash::FxHashMap; use std::ops::Deref; @@ -13,24 +11,28 @@ use tracing::{debug, trace}; #[derive(Debug)] pub struct ToolchainRegistry { - pub configs: FxHashMap, + pub configs: FxHashMap, registry: Arc>, } impl ToolchainRegistry { - pub fn new(moon_env: Arc, proto_env: Arc) -> Self { + pub fn new(host_data: PluginHostData) -> Self { Self { configs: FxHashMap::default(), - registry: Arc::new(PluginRegistry::new( - PluginType::Toolchain, - moon_env, - proto_env, - )), + registry: Arc::new(PluginRegistry::new(PluginType::Toolchain, host_data)), } } + pub fn get_plugin_ids(&self) -> Vec<&PluginId> { + self.configs.keys().collect() + } + + pub fn has_plugins(&self) -> bool { + !self.configs.is_empty() + } + pub async fn load_all(&self) -> miette::Result<()> { - if self.configs.is_empty() { + if !self.has_plugins() { return Ok(()); } @@ -56,7 +58,7 @@ impl ToolchainRegistry { .config .insert("moon_toolchain_config".to_owned(), value); - inject_proto_manifest_config(&id, ®istry.proto_env, manifest)?; + inject_proto_manifest_config(&id, ®istry.host_data.proto_env, manifest)?; Ok(()) }) diff --git a/packages/types/src/project-config.ts b/packages/types/src/project-config.ts index 575d33fe8d1..52999651ecb 100644 --- a/packages/types/src/project-config.ts +++ b/packages/types/src/project-config.ts @@ -2,8 +2,8 @@ /* eslint-disable */ -import type { PartialTaskConfig, PlatformType, TaskConfig } from './tasks-config'; import type { UnresolvedVersionSpec } from './toolchain-config'; +import type { PartialTaskConfig, PlatformType, TaskConfig } from './tasks-config'; /** The scope and or relationship of the dependency. */ export type DependencyScope = 'build' | 'development' | 'peer' | 'production' | 'root';