internal: Add plugin host data and sync workspace support. (#1655)

* Add host data.

* Update proto/extism.

* Fix host funcs.

* Add wasm bridge.

* Remove deps.

* Rename config.

* Add sync workspace.

* Fix format.

* Improve affected.

CHANGELOG.md

@@ -10,6 +10,8 @@
   - Added `--upstream` and `--downstream` options to `moon query projects`.
   - Coming soon for affected tasks as well!
 - Added a new task option, `cacheLifetime`, that controls how long a task will be cached for.
+- Added "sync workspace action" support to toolchain plugins. This is our first step in supporting
+  toolchains via WASM plugins.
 - Updated task `outputs` to support token and environment variables.
 - Updated `moon query projects` to include the project description as a trailing value.
 - Updated `moon query tasks` to include the task type and platform, and the task description as a

crates/action-graph/src/action_graph_builder.rs

@@ -27,6 +27,10 @@ pub struct RunRequirements {
     pub interactive: bool,
     pub skip_affected: bool, // Temporary until we support task dependents properly
     pub target_locators: FxHashSet<TargetLocator>,
+
+    // Used to mark affected states
+    pub upstream_target: Option<Target>,
+    pub downstream_target: Option<Target>,
 }
 
 impl RunRequirements {
@@ -273,7 +277,7 @@ impl<'app> ActionGraphBuilder<'app> {
             // but only if this task was affected
             if reqs.dependents {
                 if let Some(affected) = &mut self.affected {
-                    debug!(
+                    trace!(
                         task = task.target.as_str(),
                         "But will run all dependent tasks if affected"
                     );
@@ -324,8 +328,17 @@ impl<'app> ActionGraphBuilder<'app> {
         }
 
         // Compare against touched files if provided
-        if !reqs.skip_affected {
-            if let Some(affected) = &mut self.affected {
+        if let Some(affected) = &mut self.affected {
+            if let Some(downstream) = &reqs.downstream_target {
+                affected
+                    .mark_task_affected(task, AffectedBy::DownstreamTask(downstream.to_owned()))?;
+            }
+
+            if let Some(upstream) = &reqs.upstream_target {
+                affected.mark_task_affected(task, AffectedBy::UpstreamTask(upstream.to_owned()))?;
+            }
+
+            if !reqs.skip_affected {
                 match affected.is_task_affected(task)? {
                     Some(by) => {
                         affected.mark_task_affected(task, by)?;
@@ -376,7 +389,14 @@ impl<'app> ActionGraphBuilder<'app> {
     #[instrument(skip_all)]
     pub fn run_task_dependencies(&mut self, task: &Task) -> miette::Result<Vec<NodeIndex>> {
         let parallel = task.options.run_deps_in_parallel;
-        let reqs = RunRequirements::default();
+        let reqs = RunRequirements {
+            downstream_target: if self.affected.is_some() {
+                Some(task.target.clone())
+            } else {
+                None
+            },
+            ..Default::default()
+        };
         let mut indices = vec![];
         let mut previous_target_index = None;
 
@@ -422,6 +442,11 @@ impl<'app> ActionGraphBuilder<'app> {
             ci: parent_reqs.ci,
             interactive: parent_reqs.interactive,
             skip_affected: true,
+            upstream_target: if self.affected.is_some() {
+                Some(task.target.clone())
+            } else {
+                None
+            },
             ..Default::default()
         };
 

crates/action-pipeline/Cargo.toml

@@ -20,6 +20,7 @@ moon_project = { path = "../project" }
 moon_project_graph = { path = "../project-graph" }
 moon_task = { path = "../task" }
 moon_toolchain = { path = "../toolchain" }
+moon_toolchain_plugin = { path = "../toolchain-plugin" }
 async-trait = { workspace = true }
 miette = { workspace = true }
 num_cpus = "1.16.0"

crates/action-pipeline/src/action_pipeline.rs

@@ -14,6 +14,7 @@ use moon_api::Moonbase;
 use moon_app_context::AppContext;
 use moon_common::{color, is_ci, is_test_env};
 use moon_project_graph::ProjectGraph;
+use moon_toolchain_plugin::ToolchainRegistry;
 use rustc_hash::{FxHashMap, FxHashSet};
 use std::mem;
 use std::sync::Arc;
@@ -39,10 +40,15 @@ pub struct ActionPipeline {
     action_context: Arc<ActionContext>,
     emitter: Arc<EventEmitter>,
     project_graph: Arc<ProjectGraph>,
+    toolchain_registry: Arc<ToolchainRegistry>,
 }
 
 impl ActionPipeline {
-    pub fn new(app_context: Arc<AppContext>, project_graph: Arc<ProjectGraph>) -> Self {
+    pub fn new(
+        app_context: Arc<AppContext>,
+        project_graph: Arc<ProjectGraph>,
+        toolchain_registry: Arc<ToolchainRegistry>,
+    ) -> Self {
         debug!("Creating pipeline to run actions");
 
         Self {
@@ -57,6 +63,7 @@ impl ActionPipeline {
             project_graph,
             report_name: "runReport.json".into(),
             summarize: false,
+            toolchain_registry,
         }
     }
 
@@ -145,6 +152,7 @@ impl ActionPipeline {
             result_sender: sender,
             semaphore: Arc::new(Semaphore::new(self.concurrency)),
             running_jobs: Arc::new(RwLock::new(FxHashMap::default())),
+            toolchain_registry: Arc::clone(&self.toolchain_registry),
         };
 
         // Monitor signals and ctrl+c

crates/action-pipeline/src/action_runner.rs

@@ -5,6 +5,7 @@ use moon_actions::actions::*;
 use moon_app_context::AppContext;
 use moon_common::color;
 use moon_project_graph::ProjectGraph;
+use moon_toolchain_plugin::ToolchainRegistry;
 use std::sync::Arc;
 use tracing::{instrument, trace};
 
@@ -14,6 +15,7 @@ pub async fn run_action(
     action_context: Arc<ActionContext>,
     app_context: Arc<AppContext>,
     project_graph: Arc<ProjectGraph>,
+    toolchain_registry: Arc<ToolchainRegistry>,
     emitter: Arc<EventEmitter>,
 ) -> miette::Result<()> {
     action.start();
@@ -36,7 +38,14 @@ pub async fn run_action(
         ActionNode::SyncWorkspace => {
             emitter.emit(Event::WorkspaceSyncing).await?;
 
-            let result = sync_workspace(action, action_context, app_context, project_graph).await;
+            let result = sync_workspace(
+                action,
+                action_context,
+                app_context,
+                project_graph,
+                toolchain_registry,
+            )
+            .await;
 
             emitter
                 .emit(Event::WorkspaceSynced {

crates/action-pipeline/src/job.rs

@@ -73,6 +73,7 @@ impl Job {
                 self.action_context,
                 self.app_context,
                 self.context.project_graph.clone(),
+                self.context.toolchain_registry.clone(),
                 self.context.emitter.clone(),
             ) => {},
         };

crates/action-pipeline/src/job_context.rs

@@ -1,6 +1,7 @@
 use crate::event_emitter::EventEmitter;
 use moon_action::Action;
 use moon_project_graph::ProjectGraph;
+use moon_toolchain_plugin::ToolchainRegistry;
 use petgraph::graph::NodeIndex;
 use rustc_hash::{FxHashMap, FxHashSet};
 use std::sync::Arc;
@@ -32,6 +33,9 @@ pub struct JobContext {
 
     /// Acquires a permit for concurrency
     pub semaphore: Arc<Semaphore>,
+
+    /// The registry of all toolchain plugins
+    pub toolchain_registry: Arc<ToolchainRegistry>,
 }
 
 impl JobContext {

crates/action/src/operation.rs

@@ -51,18 +51,18 @@ impl Operation {
 
     pub fn get_output(&self) -> Option<&OperationMetaOutput> {
         match &self.meta {
-            OperationMeta::OutputHydration(output) | OperationMeta::TaskExecution(output) => {
-                Some(output)
-            }
+            OperationMeta::OutputHydration(output)
+            | OperationMeta::ProcessExecution(output)
+            | OperationMeta::TaskExecution(output) => Some(output),
             _ => None,
         }
     }
 
     pub fn get_output_mut(&mut self) -> Option<&mut OperationMetaOutput> {
         match &mut self.meta {
-            OperationMeta::OutputHydration(output) | OperationMeta::TaskExecution(output) => {
-                Some(output)
-            }
+            OperationMeta::OutputHydration(output)
+            | OperationMeta::ProcessExecution(output)
+            | OperationMeta::TaskExecution(output) => Some(output),
             _ => None,
         }
     }
@@ -216,6 +216,15 @@ impl Operation {
         Self::new(OperationMeta::OutputHydration(Default::default()))
     }
 
+    pub fn process_execution(command: impl AsRef<str>) -> Self {
+        Self::new(OperationMeta::ProcessExecution(Box::new(
+            OperationMetaOutput {
+                command: Some(command.as_ref().to_owned()),
+                ..Default::default()
+            },
+        )))
+    }
+
     pub fn sync_operation(label: impl AsRef<str>) -> Self {
         Self::new(OperationMeta::SyncOperation(Box::new(OperationMetaLabel {
             label: label.as_ref().to_owned(),

crates/action/src/operation_list.rs

@@ -38,6 +38,7 @@ impl OperationList {
         self.0.iter().rfind(|op| {
             op.meta.is_no_operation()
                 || op.meta.is_output_hydration()
+                || op.meta.is_process_execution()
                 || op.meta.is_sync_operation()
                 || op.meta.is_task_execution()
         })

crates/action/src/operation_meta.rs

@@ -56,6 +56,7 @@ pub enum OperationMeta {
     #[default]
     NoOperation,
     OutputHydration(Box<OperationMetaOutput>),
+    ProcessExecution(Box<OperationMetaOutput>),
     SyncOperation(Box<OperationMetaLabel>),
     TaskExecution(Box<OperationMetaOutput>),
 
@@ -86,6 +87,10 @@ impl OperationMeta {
         matches!(self, Self::OutputHydration(_))
     }
 
+    pub fn is_process_execution(&self) -> bool {
+        matches!(self, Self::ProcessExecution(_))
+    }
+
     pub fn is_sync_operation(&self) -> bool {
         matches!(self, Self::SyncOperation(_))
     }