new: Setup groundwork for toolchain plugins. (#1611)

* new: Replace old pipeline with new pipeline. (#1600)

* Remove old crates.

* Update callsites.

* Rename crate.

* Resolve experiments.

* Remove CI insights.

* Update versions.

* Fix tests.

* Update schematic.

* Add register.

* Add registry to session.

* Enable via config.

* Get config loading working.

* Some clean up.

* Fix APIs.

* Polish.

* Fix tests.

.github/workflows/rust.yml

@@ -90,7 +90,7 @@ jobs:
         with:
           auto-install: true
           cache: ${{ runner.os == 'Linux' }}
-          proto-version: '0.40.0' # Keep in sync
+          proto-version: '0.40.2' # Keep in sync
       - uses: mozilla-actions/[email protected]
         if: ${{ vars.ENABLE_SCCACHE == 'true' }}
       - name: Checking coverage status

CHANGELOG.md

@@ -15,7 +15,7 @@
 
 - We've sunset the CI insights feature in moonbase. We will no longer be tracking CI run history.
   This is a retroactive change that applies to all moon versions.
-- Updated proto to v0.40.0 (from v0.39.7).
+- Updated proto to v0.40.2 (from v0.39.7).
 
 ## 1.27.10
 

Cargo.toml

@@ -54,8 +54,8 @@ schematic = { version = "0.17.2", default-features = false, features = [
 ] }
 serial_test = "3.1.1"
 semver = "1.0.23"
-serde = { version = "1.0.204", features = ["derive"] }
-serde_json = "1.0.122"
+serde = { version = "1.0.209", features = ["derive"] }
+serde_json = "1.0.127"
 serde_yaml = "0.9.34"
 starbase = { version = "0.8.2" }
 starbase_archive = { version = "0.8.4", default-features = false, features = [
@@ -93,7 +93,7 @@ uuid = { version = "1.10.0", features = ["v4"] }
 # proto/plugin related
 extism = "=1.3.0"
 extism-pdk = "1.2.1"
-proto_core = "0.40.1"
+proto_core = "0.40.3"
 proto_installer = "0.7.0"
 system_env = "0.6.0"
 version_spec = "0.7.0"

crates/app/Cargo.toml

@@ -27,6 +27,7 @@ moon_project_graph = { path = "../project-graph" }
 moon_query = { path = "../query" }
 moon_task = { path = "../task" }
 moon_toolchain = { path = "../toolchain" }
+moon_toolchain_plugin = { path = "../toolchain-plugin" }
 moon_vcs = { path = "../vcs" }
 async-recursion = { workspace = true }
 async-trait = { workspace = true }

crates/app/src/commands/ext.rs

@@ -2,7 +2,7 @@ use crate::app_error::AppError;
 use crate::session::CliSession;
 use clap::Args;
 use moon_common::Id;
-use moon_plugin::{serialize_config, PluginId};
+use moon_plugin::PluginId;
 use starbase::AppResult;
 use tracing::instrument;
 
@@ -18,39 +18,20 @@ pub struct ExtArgs {
 
 #[instrument(skip_all)]
 pub async fn ext(session: CliSession, args: ExtArgs) -> AppResult {
-    let Some(config) = session.workspace_config.extensions.get(&args.id) else {
+    if !session.workspace_config.extensions.contains_key(&args.id) {
         return Err(AppError::UnknownExtension { id: args.id }.into());
-    };
+    }
 
     let id = PluginId::raw(&args.id);
     let extensions = session.get_extension_registry()?;
 
-    // Load and configure the plugin
-    extensions
-        .load_with_config(&id, config.get_plugin_locator(), move |manifest| {
-            manifest.config.insert(
-                "moon_extension_config".to_owned(),
-                serialize_config(config.config.iter())?,
-            );
-
-            Ok(())
-        })
-        .await?;
+    // Load the plugin
+    let plugin = extensions.load(&id).await?;
 
     // Execute the plugin
-    let plugin = extensions.get(&id).await?;
-
     plugin
         .execute(args.passthrough, extensions.create_context())
         .await?;
 
-    // let passthrough_args = args.passthrough.clone();
-
-    // extensions
-    //     .perform(&id, |plugin, context| async move {
-    //         plugin.execute(passthrough_args, context).await
-    //     })
-    //     .await?;
-
     Ok(())
 }

crates/app/src/commands/setup.rs

@@ -1,13 +1,14 @@
 use crate::helpers::create_progress_bar;
+use crate::session::CliSession;
 use crate::systems::analyze;
 use starbase::AppResult;
 use tracing::instrument;
 
 #[instrument]
-pub async fn setup() -> AppResult {
+pub async fn setup(session: CliSession) -> AppResult {
     let done = create_progress_bar("Downloading and installing tools...");
 
-    analyze::load_toolchain().await?;
+    analyze::load_toolchain(session.get_toolchain_registry()?).await?;
 
     done("Setup complete", true);
 

crates/app/src/session.rs

@@ -12,9 +12,10 @@ use moon_config::{InheritedTasksManager, ToolchainConfig, WorkspaceConfig};
 use moon_console::Console;
 use moon_console_reporter::DefaultReporter;
 use moon_env::MoonEnvironment;
-use moon_extension_plugin::ExtensionPlugin;
-use moon_plugin::{PluginRegistry, PluginType};
+use moon_extension_plugin::*;
+use moon_plugin::PluginId;
 use moon_project_graph::{ProjectGraph, ProjectGraphBuilder};
+use moon_toolchain_plugin::*;
 use moon_vcs::{BoxedVcs, Git};
 use once_cell::sync::OnceCell;
 use proto_core::ProtoEnvironment;
@@ -27,8 +28,6 @@ use std::sync::Arc;
 use tokio::try_join;
 use tracing::debug;
 
-pub type ExtensionRegistry = PluginRegistry<ExtensionPlugin>;
-
 #[derive(Clone)]
 pub struct CliSession {
     pub cli: Cli,
@@ -44,6 +43,7 @@ pub struct CliSession {
     cache_engine: OnceCell<Arc<CacheEngine>>,
     extension_registry: OnceCell<Arc<ExtensionRegistry>>,
     project_graph: OnceCell<Arc<ProjectGraph>>,
+    toolchain_registry: OnceCell<Arc<ToolchainRegistry>>,
     vcs_adapter: OnceCell<Arc<BoxedVcs>>,
 
     // Configs
@@ -71,6 +71,7 @@ impl CliSession {
             proto_env: Arc::new(ProtoEnvironment::default()),
             tasks_config: Arc::new(InheritedTasksManager::default()),
             toolchain_config: Arc::new(ToolchainConfig::default()),
+            toolchain_registry: OnceCell::new(),
             working_dir: PathBuf::new(),
             workspace_root: PathBuf::new(),
             workspace_config: Arc::new(WorkspaceConfig::default()),
@@ -117,11 +118,15 @@ impl CliSession {
 
     pub fn get_extension_registry(&self) -> AppResult<Arc<ExtensionRegistry>> {
         let item = self.extension_registry.get_or_init(|| {
-            Arc::new(PluginRegistry::new(
-                PluginType::Extension,
-                Arc::clone(&self.moon_env),
-                Arc::clone(&self.proto_env),
-            ))
+            let mut registry =
+                ExtensionRegistry::new(Arc::clone(&self.moon_env), Arc::clone(&self.proto_env));
+
+            // Convert moon IDs to plugin IDs
+            for (id, config) in self.workspace_config.extensions.clone() {
+                registry.configs.insert(PluginId::raw(id), config);
+            }
+
+            Arc::new(registry)
         });
 
         Ok(Arc::clone(item))
@@ -142,6 +147,22 @@ impl CliSession {
         Ok(graph)
     }
 
+    pub fn get_toolchain_registry(&self) -> AppResult<Arc<ToolchainRegistry>> {
+        let item = self.toolchain_registry.get_or_init(|| {
+            let mut registry =
+                ToolchainRegistry::new(Arc::clone(&self.moon_env), Arc::clone(&self.proto_env));
+
+            // Convert moon IDs to plugin IDs
+            for (id, config) in self.toolchain_config.toolchains.clone() {
+                registry.configs.insert(PluginId::raw(id), config);
+            }
+
+            Arc::new(registry)
+        });
+
+        Ok(Arc::clone(item))
+    }
+
     pub fn get_vcs_adapter(&self) -> AppResult<Arc<BoxedVcs>> {
         let item = self.vcs_adapter.get_or_try_init(|| {
             let config = &self.workspace_config.vcs;
@@ -248,7 +269,7 @@ impl AppSession for CliSession {
             .await?;
 
             if self.requires_toolchain_installed() {
-                analyze::load_toolchain().await?;
+                analyze::load_toolchain(self.get_toolchain_registry()?).await?;
             }
         }
 

crates/app/src/systems/analyze.rs

@@ -9,6 +9,7 @@ use moon_node_platform::NodePlatform;
 use moon_platform::PlatformManager;
 use moon_rust_platform::RustPlatform;
 use moon_system_platform::SystemPlatform;
+use moon_toolchain_plugin::ToolchainRegistry;
 use proto_core::{is_offline, ProtoEnvironment, ProtoError};
 use proto_installer::*;
 use semver::{Version, VersionReq};
@@ -186,13 +187,15 @@ pub async fn register_platforms(
     Ok(())
 }
 
-#[instrument]
-pub async fn load_toolchain() -> AppResult {
+#[instrument(skip(registry))]
+pub async fn load_toolchain(registry: Arc<ToolchainRegistry>) -> AppResult {
     // This isn't an action but we should also support skipping here!
     if should_skip_action("MOON_SKIP_SETUP_TOOLCHAIN").is_some() {
         return Ok(());
     }
 
+    registry.load_all().await?;
+
     for platform in PlatformManager::write().list_mut() {
         platform.setup_toolchain().await?;
     }

crates/cli/src/main.rs

@@ -170,7 +170,7 @@ async fn main() -> MainResult {
                     }
                 },
                 Commands::Run(args) => commands::run::run(session, args).await,
-                Commands::Setup => commands::setup::setup().await,
+                Commands::Setup => commands::setup::setup(session).await,
                 Commands::Sync { command } => match command {
                     Some(SyncCommands::Codeowners(args)) => {
                         commands::syncs::codeowners::sync(session, args).await

crates/common/src/consts.rs

@@ -16,4 +16,4 @@ pub const CONFIG_PROJECT_FILENAME: &str = "moon.yml";
 
 pub const CONFIG_TEMPLATE_FILENAME: &str = "template.yml";
 
-pub const PROTO_CLI_VERSION: &str = "0.40.0";
+pub const PROTO_CLI_VERSION: &str = "0.40.2";

crates/config/src/config_cache.rs

@@ -62,6 +62,8 @@ impl Cacher for ConfigCache {
                     return Ok(Some(contents));
                 }
             }
+
+            let _ = fs::remove_file(&file);
         }
 
         Ok(None)

crates/config/src/toolchain_config.rs

@@ -2,6 +2,7 @@
 
 use crate::language_platform::*;
 use crate::toolchain::*;
+use moon_common::Id;
 use rustc_hash::FxHashMap;
 use schematic::{validate, Config};
 use version_spec::UnresolvedVersionSpec;
@@ -32,6 +33,7 @@ pub struct ToolConfig {
 /// Configures all tools and platforms required for tasks.
 /// Docs: https://moonrepo.dev/docs/config/toolchain
 #[derive(Clone, Config, Debug)]
+#[config(allow_unknown_fields)]
 pub struct ToolchainConfig {
     #[setting(
         default = "https://moonrepo.dev/schemas/toolchain.json",
@@ -63,9 +65,10 @@ pub struct ToolchainConfig {
     /// Configures and enables the TypeScript platform.
     #[setting(nested)]
     pub typescript: Option<TypeScriptConfig>,
-    // TODO: enable once platforms are live!
-    // #[setting(flatten, nested)]
-    // pub tools: FxHashMap<Id, ToolConfig>,
+
+    /// All configured toolchains by unique ID.
+    #[setting(flatten, nested)]
+    pub toolchains: FxHashMap<Id, ToolConfig>,
 }
 
 impl ToolchainConfig {