From a584eeefcbd68ec1f82ca2396f81075ee63159a9 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Sat, 12 Oct 2024 16:53:13 +0200 Subject: [PATCH 01/28] Save my work --- Cargo.lock | 71 ++++ crates/app/Cargo.toml | 3 + crates/app/src/systems/analyze.rs | 13 + crates/config/src/language_platform.rs | 1 + crates/config/src/project/overrides_config.rs | 4 + crates/config/src/toolchain/mod.rs | 2 + crates/config/src/toolchain/python_config.rs | 35 ++ crates/config/src/toolchain_config.rs | 18 + .../toolchain/src/detect/project_platform.rs | 7 + legacy/python/lang/Cargo.toml | 23 ++ legacy/python/lang/src/lib.rs | 5 + legacy/python/lang/src/pip_requirements.rs | 15 + legacy/python/platform/Cargo.toml | 36 ++ .../platform/src/actions/install_deps.rs | 78 +++++ legacy/python/platform/src/actions/mod.rs | 3 + legacy/python/platform/src/lib.rs | 16 + legacy/python/platform/src/manifest_hash.rs | 10 + legacy/python/platform/src/python_platform.rs | 320 ++++++++++++++++++ legacy/python/platform/src/target_hash.rs | 18 + legacy/python/platform/src/toolchain_hash.rs | 21 ++ legacy/python/tool/Cargo.toml | 25 ++ legacy/python/tool/src/lib.rs | 3 + legacy/python/tool/src/python_tool.rs | 292 ++++++++++++++++ packages/types/src/project-config.ts | 28 +- packages/types/src/tasks-config.ts | 15 +- packages/types/src/template-config.ts | 12 +- packages/types/src/toolchain-config.ts | 55 ++- packages/types/src/workspace-config.ts | 5 +- website/static/schemas/project.json | 14 + website/static/schemas/tasks.json | 1 + website/static/schemas/toolchain.json | 90 +++++ 31 files changed, 1182 insertions(+), 57 deletions(-) create mode 100644 crates/config/src/toolchain/python_config.rs create mode 100644 legacy/python/lang/Cargo.toml create mode 100644 legacy/python/lang/src/lib.rs create mode 100644 legacy/python/lang/src/pip_requirements.rs create mode 100644 legacy/python/platform/Cargo.toml create mode 100644 legacy/python/platform/src/actions/install_deps.rs create mode 100644 legacy/python/platform/src/actions/mod.rs create mode 100644 legacy/python/platform/src/lib.rs create mode 100644 legacy/python/platform/src/manifest_hash.rs create mode 100644 legacy/python/platform/src/python_platform.rs create mode 100644 legacy/python/platform/src/target_hash.rs create mode 100644 legacy/python/platform/src/toolchain_hash.rs create mode 100644 legacy/python/tool/Cargo.toml create mode 100644 legacy/python/tool/src/lib.rs create mode 100644 legacy/python/tool/src/python_tool.rs diff --git a/Cargo.lock b/Cargo.lock index 138d74508f8..291f3377a62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2994,6 +2994,9 @@ dependencies = [ "moon_plugin", "moon_project", "moon_project_graph", + "moon_python_lang", + "moon_python_platform", + "moon_python_tool", "moon_query", "moon_rust_lang", "moon_rust_platform", @@ -3733,6 +3736,74 @@ dependencies = [ "tracing", ] +[[package]] +name = "moon_python_lang" +version = "0.0.1" +dependencies = [ + "cached", + "cargo-lock", + "cargo_toml", + "miette", + "moon_lang", + "moon_logger", + "moon_test_utils", + "rustc-hash 2.0.0", + "serde", + "starbase_styles", + "starbase_utils", +] + +[[package]] +name = "moon_python_platform" +version = "0.0.1" +dependencies = [ + "miette", + "moon_action", + "moon_action_context", + "moon_common", + "moon_config", + "moon_console", + "moon_hash", + "moon_logger", + "moon_platform", + "moon_process", + "moon_project", + "moon_python_lang", + "moon_python_tool", + "moon_task", + "moon_test_utils", + "moon_tool", + "moon_utils", + "proto_core", + "rustc-hash 2.0.0", + "serde", + "starbase_styles", + "starbase_utils", + "tokio", + "tracing", +] + +[[package]] +name = "moon_python_tool" +version = "0.0.1" +dependencies = [ + "miette", + "moon_common", + "moon_config", + "moon_console", + "moon_logger", + "moon_process", + "moon_python_lang", + "moon_tool", + "moon_toolchain", + "moon_utils", + "proto_core", + "rustc-hash 2.0.0", + "starbase_styles", + "starbase_utils", + "tracing", +] + [[package]] name = "moon_query" version = "0.0.1" diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index 9f72f4bc000..2b6630ee7aa 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -86,6 +86,9 @@ moon_deno_platform = { path = "../../legacy/deno/platform" } moon_node_lang = { path = "../../legacy/node/lang" } moon_node_tool = { path = "../../legacy/node/tool" } moon_node_platform = { path = "../../legacy/node/platform" } +moon_python_lang = { path = "../../legacy/python/lang" } +moon_python_tool = { path = "../../legacy/python/tool" } +moon_python_platform = { path = "../../legacy/python/platform" } moon_rust_lang = { path = "../../legacy/rust/lang" } moon_rust_tool = { path = "../../legacy/rust/tool" } moon_rust_platform = { path = "../../legacy/rust/platform" } diff --git a/crates/app/src/systems/analyze.rs b/crates/app/src/systems/analyze.rs index 391105d4faa..aafc7d8eb00 100644 --- a/crates/app/src/systems/analyze.rs +++ b/crates/app/src/systems/analyze.rs @@ -8,6 +8,7 @@ use moon_console::{Checkpoint, Console}; use moon_deno_platform::DenoPlatform; use moon_node_platform::NodePlatform; use moon_platform::PlatformManager; +use moon_python_platform::PythonPlatform; use moon_rust_platform::RustPlatform; use moon_system_platform::SystemPlatform; use moon_toolchain_plugin::ToolchainRegistry; @@ -172,6 +173,18 @@ pub async fn register_platforms( ); } + if let Some(python_config) = &toolchain_config.python { + registry.register( + PlatformType::Python, + Box::new(PythonPlatform::new( + python_config, + workspace_root, + Arc::clone(proto_env), + Arc::clone(&console), + )), + ); + } + if let Some(rust_config) = &toolchain_config.rust { registry.register( PlatformType::Rust, diff --git a/crates/config/src/language_platform.rs b/crates/config/src/language_platform.rs index c7179e3f572..2c9377b7fd7 100644 --- a/crates/config/src/language_platform.rs +++ b/crates/config/src/language_platform.rs @@ -70,6 +70,7 @@ derive_enum!( Bun, Deno, Node, + Python, Rust, System, #[default] diff --git a/crates/config/src/project/overrides_config.rs b/crates/config/src/project/overrides_config.rs index 0c87bcda69c..ea8982aa8d8 100644 --- a/crates/config/src/project/overrides_config.rs +++ b/crates/config/src/project/overrides_config.rs @@ -48,6 +48,10 @@ cacheable!( /// Overrides `deno` settings. #[setting(nested)] pub deno: Option, + + /// Overrides `python` settings. + #[setting(nested)] + pub python: Option, /// Overrides `node` settings. #[setting(nested)] diff --git a/crates/config/src/toolchain/mod.rs b/crates/config/src/toolchain/mod.rs index a3986c5f874..5c5a717520d 100644 --- a/crates/config/src/toolchain/mod.rs +++ b/crates/config/src/toolchain/mod.rs @@ -2,6 +2,7 @@ mod bin_config; mod bun_config; mod deno_config; mod node_config; +mod python_config; mod rust_config; mod typescript_config; @@ -9,6 +10,7 @@ pub use bin_config::*; pub use bun_config::*; pub use deno_config::*; pub use node_config::*; +pub use python_config::*; pub use rust_config::*; pub use typescript_config::*; diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs new file mode 100644 index 00000000000..5214f52cca1 --- /dev/null +++ b/crates/config/src/toolchain/python_config.rs @@ -0,0 +1,35 @@ +// use super::bin_config::BinEntry; +use schematic::Config; +use serde::Serialize; +use version_spec::UnresolvedVersionSpec; +use warpgate_api::PluginLocator; + + +#[derive(Clone, Config, Debug, PartialEq, Serialize)] +pub struct PipConfig { + /// List of arguments to append to `pip install` commands. + pub install_args: Option>, + + /// The version of pip to download, install, and run `pip` tasks with. + pub version: Option, +} + + +#[derive(Clone, Config, Debug, PartialEq)] +pub struct PythonConfig { + /// Location of the WASM plugin to use for Python support. + pub plugin: Option, + + /// Options for pnpm, when used as a package manager. + #[setting(nested)] + pub pip: Option, + + /// The relative root of the virtual environment workspace. Default to moon's + /// workspace root + #[setting(default = ".", skip)] + pub venv_root: String, + + /// The version of Python to download, install, and run `python` tasks with. + #[setting(env = "MOON_PYTHON_VERSION")] + pub version: Option, +} diff --git a/crates/config/src/toolchain_config.rs b/crates/config/src/toolchain_config.rs index f482a54c379..7d2bb26deda 100644 --- a/crates/config/src/toolchain_config.rs +++ b/crates/config/src/toolchain_config.rs @@ -53,6 +53,10 @@ pub struct ToolchainConfig { #[setting(nested)] pub node: Option, + /// Configures and enables the Python platform. + #[setting(nested)] + pub python: Option, + /// Configures and enables the Rust platform. #[setting(nested)] pub rust: Option, @@ -82,6 +86,10 @@ impl ToolchainConfig { tools.push(PlatformType::Node); } + if self.python.is_some() { + tools.push(PlatformType::Python) + } + if self.rust.is_some() { tools.push(PlatformType::Rust); } @@ -137,6 +145,12 @@ impl ToolchainConfig { } } + if let Some(python_config) = &self.python { + if let Some(version) = &python_config.version { + inject("PROTO_PYTHON_VERSION", version); + } + } + // We don't include Rust since it's a special case! env @@ -151,6 +165,8 @@ impl ToolchainConfig { inherit_tool!(NodeConfig, node, "node", inherit_proto_node); + inherit_tool!(PythonConfig, python, "python", inherit_proto_python); + inherit_tool!(RustConfig, rust, "rust", inherit_proto_rust); inherit_tool_without_version!( @@ -167,6 +183,7 @@ impl ToolchainConfig { is_using_tool_version!(self, node, bun); is_using_tool_version!(self, node, pnpm); is_using_tool_version!(self, node, yarn); + is_using_tool_version!(self, python); is_using_tool_version!(self, rust); // Special case @@ -185,6 +202,7 @@ impl ToolchainConfig { self.inherit_proto_bun(proto_config)?; self.inherit_proto_deno(proto_config)?; self.inherit_proto_node(proto_config)?; + self.inherit_proto_python(proto_config)?; self.inherit_proto_rust(proto_config)?; self.inherit_proto_typescript(proto_config)?; diff --git a/crates/toolchain/src/detect/project_platform.rs b/crates/toolchain/src/detect/project_platform.rs index 163e4ccfcc8..54e1c6cb1b0 100644 --- a/crates/toolchain/src/detect/project_platform.rs +++ b/crates/toolchain/src/detect/project_platform.rs @@ -28,6 +28,13 @@ pub fn detect_project_platform( PlatformType::System } } + LanguageType::Python => { + if enabled_platforms.contains(&PlatformType::Python) { + PlatformType::Python + } else { + PlatformType::System + } + } LanguageType::Rust => { if enabled_platforms.contains(&PlatformType::Rust) { PlatformType::Rust diff --git a/legacy/python/lang/Cargo.toml b/legacy/python/lang/Cargo.toml new file mode 100644 index 00000000000..6074c3866eb --- /dev/null +++ b/legacy/python/lang/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "moon_python_lang" +version = "0.0.1" +edition = "2021" +publish = false + +[dependencies] +moon_lang = { path = "../../core/lang" } +moon_logger = { path = "../../core/logger" } +cached = { workspace = true } +cargo-lock = "9.0.0" +cargo_toml = "0.20.4" +miette = { workspace = true } +rustc-hash = { workspace = true } +serde = { workspace = true } +starbase_styles = { workspace = true } +starbase_utils = { workspace = true, features = ["glob", "toml"] } + +[dev-dependencies] +moon_test_utils = { path = "../../core/test-utils" } + +[lints] +workspace = true diff --git a/legacy/python/lang/src/lib.rs b/legacy/python/lang/src/lib.rs new file mode 100644 index 00000000000..d55284cc8f7 --- /dev/null +++ b/legacy/python/lang/src/lib.rs @@ -0,0 +1,5 @@ +pub mod pip_requirements; + + +pub use pip_requirements::*; +pub use moon_lang::LockfileDependencyVersions; diff --git a/legacy/python/lang/src/pip_requirements.rs b/legacy/python/lang/src/pip_requirements.rs new file mode 100644 index 00000000000..d4c7aa3ea25 --- /dev/null +++ b/legacy/python/lang/src/pip_requirements.rs @@ -0,0 +1,15 @@ +use cached::proc_macro::cached; +use moon_lang::LockfileDependencyVersions; +use rustc_hash::FxHashMap; +use std::path::PathBuf; +use std::fs; + +#[cached(result)] +pub fn load_lockfile_dependencies(path: PathBuf) -> miette::Result { + let mut deps: LockfileDependencyVersions = FxHashMap::default(); + + let lockfile = fs::read_to_string(path.as_path()).expect("Unable to read file"); + let dep = deps.entry(path.display().to_string()).or_default(); + dep.push(lockfile); + Ok(deps) +} diff --git a/legacy/python/platform/Cargo.toml b/legacy/python/platform/Cargo.toml new file mode 100644 index 00000000000..c797eaa1e1b --- /dev/null +++ b/legacy/python/platform/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "moon_python_platform" +version = "0.0.1" +edition = "2021" +publish = false + +[dependencies] +moon_action = { path = "../../../crates/action" } +moon_action_context = { path = "../../../crates/action-context" } +moon_common = { path = "../../../crates/common" } +moon_config = { path = "../../../crates/config" } +moon_console = { path = "../../../crates/console" } +moon_hash = { path = "../../../crates/hash" } +moon_logger = { path = "../../core/logger" } +moon_platform = { path = "../../core/platform" } +moon_process = { path = "../../../crates/process" } +moon_project = { path = "../../../crates/project" } +moon_python_lang = { path = "../lang" } +moon_python_tool = { path = "../tool" } +moon_task = { path = "../../../crates/task" } +moon_tool = { path = "../../core/tool" } +moon_utils = { path = "../../core/utils" } +miette = { workspace = true } +proto_core = { workspace = true } +rustc-hash = { workspace = true } +serde = { workspace = true } +starbase_styles = { workspace = true } +starbase_utils = { workspace = true, features = ["glob"] } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +moon_test_utils = { path = "../../core/test-utils" } + +[lints] +workspace = true diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs new file mode 100644 index 00000000000..a7ed87a496e --- /dev/null +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -0,0 +1,78 @@ +use moon_action::Operation; +use moon_console::{Checkpoint, Console}; +use moon_python_tool::PythonTool; +use moon_utils::get_workspace_root; +use std::path::Path; + +use crate::find_requirements_txt; + +pub async fn install_deps( + python: &PythonTool, + working_dir: &Path, + console: &Console, +) -> miette::Result> { + let mut operations = vec![]; + + if let Some(pip_config) = &python.config.pip { + + + + if let Some(pip_version) = &pip_config.version { + console + .out + .print_checkpoint(Checkpoint::Setup, format!("install pip {pip_version}"))?; + + let p_version: String = if pip_version.is_latest() { + format!("pip") + } else { + format!("pip{}", pip_version.to_owned().to_string().replace("~", "~=")) + }; + let args = vec!["-m", "pip", "install","--quiet", "-U", &p_version]; + + operations.push( + Operation::task_execution(format!("python {} ", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); + } + + console + .out + .print_checkpoint(Checkpoint::Setup, format!("activate virtual environment"))?; + let virtual_environment = &get_workspace_root().join(".venv"); + + if !virtual_environment.exists() { + let args = vec!["-m", "venv", virtual_environment.as_os_str().to_str().unwrap()]; + operations.push( + Operation::task_execution(format!("python {} ", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); + } + + + + + if let Some(req) = find_requirements_txt(working_dir, &get_workspace_root()) { + console + .out + .print_checkpoint(Checkpoint::Setup, format!("pip dependencies from {}", req.as_os_str().to_str().unwrap()))?; + + let mut args = vec!["-m", "pip", "install", "--quiet"]; + + if pip_config.install_args.is_some() { + args.extend(pip_config.install_args.as_ref().unwrap().iter().map(|c| c.as_str())); + } + + args.extend(["-r", req.as_os_str().to_str().unwrap()]); + + operations.push( + Operation::task_execution(format!("python {}", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); + } + } + + Ok(operations) +} \ No newline at end of file diff --git a/legacy/python/platform/src/actions/mod.rs b/legacy/python/platform/src/actions/mod.rs new file mode 100644 index 00000000000..b7b5cf56da6 --- /dev/null +++ b/legacy/python/platform/src/actions/mod.rs @@ -0,0 +1,3 @@ +mod install_deps; + +pub use install_deps::*; \ No newline at end of file diff --git a/legacy/python/platform/src/lib.rs b/legacy/python/platform/src/lib.rs new file mode 100644 index 00000000000..bb487c34931 --- /dev/null +++ b/legacy/python/platform/src/lib.rs @@ -0,0 +1,16 @@ +pub mod actions; +mod manifest_hash; +mod python_platform; +mod target_hash; +mod toolchain_hash; + + +pub use python_platform::*; + +use starbase_utils::fs; +use std::path::{Path, PathBuf}; + +fn find_requirements_txt(starting_dir: &Path, workspace_root: &Path) -> Option { + fs::find_upwards_until("requirements.txt", starting_dir, workspace_root) +} + diff --git a/legacy/python/platform/src/manifest_hash.rs b/legacy/python/platform/src/manifest_hash.rs new file mode 100644 index 00000000000..89acff80801 --- /dev/null +++ b/legacy/python/platform/src/manifest_hash.rs @@ -0,0 +1,10 @@ +use moon_hash::hash_content; +// use moon_python_lang::pipfile::DependencyDetail; +// use std::collections::BTreeMap; + +hash_content!( + pub struct PythonManifestHash { + // pub dependencies: BTreeMap, + pub name: String, + } +); diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs new file mode 100644 index 00000000000..1ef0f635198 --- /dev/null +++ b/legacy/python/platform/src/python_platform.rs @@ -0,0 +1,320 @@ +use crate::{ + actions, find_requirements_txt, target_hash::PythonTargetHash, toolchain_hash::PythonToolchainHash +}; +use moon_action::Operation; +use moon_action_context::ActionContext; +use moon_common::Id; +use moon_config::{ + HasherConfig, PlatformType, ProjectConfig, ProjectsAliasesList, ProjectsSourcesList, + PythonConfig, UnresolvedVersionSpec, +}; +use moon_console::{Checkpoint, Console}; +use moon_hash::ContentHasher; +use moon_logger::debug; +use moon_platform::{Platform, Runtime, RuntimeReq}; +use moon_process::Command; +use moon_project::Project; +use moon_python_lang::pip_requirements::load_lockfile_dependencies; +use moon_python_tool::{get_python_tool_paths, PythonTool}; +use moon_task::Task; +use moon_tool::{get_proto_version_env, prepend_path_env_var, Tool, ToolManager}; +use moon_utils::{async_trait, get_workspace_root}; +use proto_core::ProtoEnvironment; +use rustc_hash::FxHashMap; +use std::{ + collections::BTreeMap, path::{Path, PathBuf}, sync::Arc +}; +use tracing::instrument; + +const LOG_TARGET: &str = "moon:python-platform"; + +pub struct PythonPlatform { + pub config: PythonConfig, + + console: Arc, + + proto_env: Arc, + + toolchain: ToolManager, + + #[allow(dead_code)] + pub workspace_root: PathBuf, +} + +impl PythonPlatform { + pub fn new( + config: &PythonConfig, + workspace_root: &Path, + proto_env: Arc, + console: Arc, + ) -> Self { + PythonPlatform { + config: config.to_owned(), + proto_env, + toolchain: ToolManager::new(Runtime::new(PlatformType::Python, RuntimeReq::Global)), + workspace_root: workspace_root.to_path_buf(), + console, + } + } +} + +#[async_trait] +impl Platform for PythonPlatform { + fn get_type(&self) -> PlatformType { + PlatformType::Python + } + + fn get_runtime_from_config(&self, project_config: Option<&ProjectConfig>) -> Runtime { + if let Some(config) = &project_config { + if let Some(python_config) = &config.toolchain.python { + if let Some(version) = &python_config.version { + return Runtime::new_override( + PlatformType::Python, + RuntimeReq::Toolchain(version.to_owned()), + ); + } + } + } + + if let Some(version) = &self.config.version { + return Runtime::new( + PlatformType::Python, + RuntimeReq::Toolchain(version.to_owned()), + ); + } + + Runtime::new(PlatformType::Python, RuntimeReq::Global) + } + + fn matches(&self, platform: &PlatformType, runtime: Option<&Runtime>) -> bool { + if matches!(platform, PlatformType::Python) { + return true; + } + + if let Some(runtime) = &runtime { + return matches!(runtime.platform, PlatformType::Python); + } + + false + } + + // PROJECT GRAPH + + fn is_project_in_dependency_workspace(&self, _project_source: &str) -> miette::Result { + + Ok(false) + } + + #[instrument(skip_all)] + fn load_project_graph_aliases( + &mut self, + _projects_list: &ProjectsSourcesList, + _aliases_list: &mut ProjectsAliasesList, + ) -> miette::Result<()> { + // Extract the alias from the Cargo project relative to the lockfile + // for (id, source) in projects_list { + // let project_root = source.to_path(&self.workspace_root); + + // } + + Ok(()) + } + + // TOOLCHAIN + + fn is_toolchain_enabled(&self) -> miette::Result { + Ok(self.config.version.is_some()) + } + + fn get_tool(&self) -> miette::Result> { + let tool = self.toolchain.get()?; + + Ok(Box::new(tool)) + } + + fn get_tool_for_version(&self, req: RuntimeReq) -> miette::Result> { + let tool = self.toolchain.get_for_version(&req)?; + + Ok(Box::new(tool)) + } + + fn get_dependency_configs(&self) -> miette::Result> { + Ok(Some(("requirements.txt".to_owned(), "requirements.txt".to_owned()))) + } + + async fn setup_toolchain(&mut self) -> miette::Result<()> { + let req = match &self.config.version { + Some(v) => RuntimeReq::Toolchain(v.to_owned()), + None => RuntimeReq::Global, + }; + + let mut last_versions = FxHashMap::default(); + + if !self.toolchain.has(&req) { + self.toolchain.register( + &req, + PythonTool::new( + Arc::clone(&self.proto_env), + Arc::clone(&self.console), + &self.config, + &req, + ) + .await?, + ); + } + + + + self.toolchain.setup(&req, &mut last_versions).await?; + + // info!( + // target: LOG_TARGET, + // "Setup toolchain" + // ); + + Ok(()) + } + + async fn teardown_toolchain(&mut self) -> miette::Result<()> { + self.toolchain.teardown_all().await?; + + Ok(()) + } + + // ACTIONS + + #[instrument(skip_all)] + async fn setup_tool( + &mut self, + _context: &ActionContext, + runtime: &Runtime, + last_versions: &mut FxHashMap, + ) -> miette::Result { + let req = &runtime.requirement; + + + + + + if !self.toolchain.has(req) { + self.toolchain.register( + req, + PythonTool::new( + Arc::clone(&self.proto_env), + Arc::clone(&self.console), + &self.config, + req, + ) + .await?, + ); + } + Ok(self.toolchain.setup(req, last_versions).await?) + } + + #[instrument(skip_all)] + async fn install_deps( + &self, + _context: &ActionContext, + runtime: &Runtime, + _working_dir: &Path, + ) -> miette::Result> { + actions::install_deps( + self.toolchain.get_for_version(&runtime.requirement)?, + &get_workspace_root(), + &self.console, + ) + .await + } + + #[instrument(skip_all)] + async fn sync_project( + &self, + _context: &ActionContext, + _project: &Project, + _dependencies: &FxHashMap>, + ) -> miette::Result { + let mutated_files = false; + //TODO: Here we can modifiy something + + Ok(mutated_files) + } + + + // # Lockfile or manifests have not changed since last run, skipping dependency install + #[instrument(skip_all)] + async fn hash_manifest_deps( + &self, + _manifest_path: &Path, + hasher: &mut ContentHasher, + _hasher_config: &HasherConfig, + ) -> miette::Result<()> { + hasher.hash_content(PythonToolchainHash { + pip: Some(self.config.pip).expect("S"), + version: Some(self.config.version).expect("S"), + requirements_dependencies: "".into(), + })?; + + Ok(()) + } + + #[instrument(skip_all)] + async fn hash_run_target( + &self, + project: &Project, + _runtime: &Runtime, + hasher: &mut ContentHasher, + _hasher_config: &HasherConfig, + ) -> miette::Result<()> { + let lockfile_path = project.root.join("requirements.txt"); + + + debug!( + target: LOG_TARGET, + "{} does not exist, installing", + lockfile_path.display() + ); + + // Not running in the Cargo workspace root, not sure how to handle! + if !lockfile_path.exists() { + return Ok(()); + } + + let mut hash = PythonTargetHash::new(None); + + // Use the resolved dependencies from the lockfile directly, + // since it also takes into account features and workspace members. + hash.locked_dependencies = BTreeMap::from_iter(load_lockfile_dependencies(lockfile_path)?); + + hasher.hash_content(hash)?; + + Ok(()) + } + + #[instrument(skip_all)] + async fn create_run_target_command( + &self, + _context: &ActionContext, + _project: &Project, + task: &Task, + runtime: &Runtime, + _working_dir: &Path, + ) -> miette::Result { + + let mut command = Command::new(&task.command); + command.with_console(self.console.clone()); + command.args(&task.args); + command.envs(&task.env); + + if let Ok(python) = self.toolchain.get_for_version(&runtime.requirement) { + if let Some(version) = get_proto_version_env(&python.tool) { + command.env("PROTO_PYTHON_VERSION", version); + command.env( + "PATH", + prepend_path_env_var(get_python_tool_paths(&python)), + ); + } + } + + Ok(command) + } +} diff --git a/legacy/python/platform/src/target_hash.rs b/legacy/python/platform/src/target_hash.rs new file mode 100644 index 00000000000..e32a6b3f9a1 --- /dev/null +++ b/legacy/python/platform/src/target_hash.rs @@ -0,0 +1,18 @@ +use moon_hash::hash_content; +use std::collections::BTreeMap; + +hash_content!( + pub struct PythonTargetHash { + pub python_version: String, + pub locked_dependencies: BTreeMap>, + } +); + +impl PythonTargetHash { + pub fn new(python_version: Option) -> Self { + PythonTargetHash { + python_version: python_version.unwrap_or_else(|| "unknown".into()), + locked_dependencies: BTreeMap::new(), + } + } +} diff --git a/legacy/python/platform/src/toolchain_hash.rs b/legacy/python/platform/src/toolchain_hash.rs new file mode 100644 index 00000000000..540bb06c372 --- /dev/null +++ b/legacy/python/platform/src/toolchain_hash.rs @@ -0,0 +1,21 @@ +use moon_config::{PipConfig, UnresolvedVersionSpec}; +use moon_hash::hash_content; +use std::collections::BTreeMap; + +hash_content!( + pub struct PythonToolchainHash { + pub pip: Option, + pub version: Option, + pub requirements_dependencies: String, + } +); + +impl PythonToolchainHash { + pub fn new(python_version: Option, pip_config: Option) -> Self { + PythonToolchainHash { + version: python_version, + requirements_dependencies: "".into(), + pip: pip_config, + } + } +} \ No newline at end of file diff --git a/legacy/python/tool/Cargo.toml b/legacy/python/tool/Cargo.toml new file mode 100644 index 00000000000..8e9cba3f79b --- /dev/null +++ b/legacy/python/tool/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "moon_python_tool" +version = "0.0.1" +edition = "2021" +publish = false + +[dependencies] +moon_python_lang = { path = "../lang" } +moon_common = { path = "../../../crates/common" } +moon_config = { path = "../../../crates/config" } +moon_console = { path = "../../../crates/console" } +moon_logger = { path = "../../core/logger" } +moon_process = { path = "../../../crates/process" } +moon_tool = { path = "../../core/tool" } +moon_utils = { path = "../../core/utils" } +moon_toolchain = { path = "../../../crates/toolchain" } +starbase_styles = { workspace = true } +miette = { workspace = true } +proto_core = { workspace = true } +rustc-hash = { workspace = true } +starbase_utils = { workspace = true } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/legacy/python/tool/src/lib.rs b/legacy/python/tool/src/lib.rs new file mode 100644 index 00000000000..84807fe6f11 --- /dev/null +++ b/legacy/python/tool/src/lib.rs @@ -0,0 +1,3 @@ +mod python_tool; + +pub use python_tool::*; diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs new file mode 100644 index 00000000000..75bb82e7732 --- /dev/null +++ b/legacy/python/tool/src/python_tool.rs @@ -0,0 +1,292 @@ +use moon_config::PythonConfig; +use moon_python_lang::pip_requirements::{load_lockfile_dependencies}; +use moon_python_lang::{LockfileDependencyVersions}; +use moon_console::{Checkpoint, Console}; +// use moon_logger::debug; +use moon_utils::get_workspace_root; +use moon_process::Command; +use moon_tool::{ + async_trait, get_proto_paths, load_tool_plugin, prepend_path_env_var, use_global_tool_on_path, + Tool, get_proto_version_env, get_proto_env_vars, DependencyManager +}; +use moon_toolchain::RuntimeReq; +use proto_core::flow::install::InstallOptions; +use proto_core::{Id, ProtoEnvironment, Tool as ProtoTool, UnresolvedVersionSpec}; +use rustc_hash::FxHashMap; +use std::env; +use std::path::PathBuf; +use std::sync::Arc; +use std::{ffi::OsStr, path::Path}; +use starbase_utils::fs; +use tracing::instrument; +use starbase_styles::color; +use moon_logger::{debug, map_list}; + +const LOG_TARGET: &str = "moon:python-tool"; + + +pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { + // let mut paths = get_proto_paths(proto_env); + // let mut paths:Vec = []; + + let paths = python_tool.tool.get_globals_dirs() + .iter() + .cloned() + .collect::>(); + + paths +} + +pub fn get_python_env_paths(proto_env: &ProtoEnvironment) -> Vec { + let paths = get_proto_paths(proto_env); + + paths +} + +pub struct PythonTool { + pub config: PythonConfig, + + pub global: bool, + + pub tool: ProtoTool, + + console: Arc, + + proto_env: Arc, +} + +impl PythonTool { + pub async fn new( + proto_env: Arc, + console: Arc, + config: &PythonConfig, + req: &RuntimeReq, + ) -> miette::Result { + let mut python = PythonTool { + config: config.to_owned(), + global: false, + tool: load_tool_plugin( + &Id::raw("python"), + &proto_env, + config.plugin.as_ref().unwrap(), + ) + .await?, + proto_env, + console, + }; + + if use_global_tool_on_path("python") || req.is_global() { + python.global = true; + python.config.version = None; + } else { + python.config.version = req.to_spec(); + }; + + Ok(python) + } + + #[instrument(skip_all)] + pub async fn exec_python(&self, args: I, working_dir: &Path) -> miette::Result<()> + where + I: IntoIterator, + S: AsRef, + { + Command::new("python") + .args(args) + .envs(get_proto_env_vars()) + .env( + "PATH", + prepend_path_env_var(get_python_env_paths(&self.proto_env)), + ) + .cwd(working_dir) + .with_console(self.console.clone()) + .create_async() + .exec_stream_output() + .await?; + + Ok(()) + + + // let mut cmd = Command::new("python"); + // cmd.with_console(self.console.clone()); + // cmd.envs(get_proto_env_vars()); + // cmd.envs(get_proto_env_vars()); + // if !self.global { + // cmd.env( + // "PATH", + // prepend_path_env_var(get_python_env_paths(&self.proto_env)), + // ); + // } + } + +} + +#[async_trait] +impl Tool for PythonTool { + fn as_any(&self) -> &(dyn std::any::Any + Send + Sync) { + self + } + + #[instrument(skip_all)] + async fn setup( + &mut self, + last_versions: &mut FxHashMap, + ) -> miette::Result { + let mut installed = 0; + + let Some(version) = &self.config.version else { + return Ok(installed); + }; + + if self.global { + debug!("Using global binary in PATH"); + } else if self.tool.is_setup(version).await? { + debug!("Python has already been setup"); + + // When offline and the tool doesn't exist, fallback to the global binary + } else if proto_core::is_offline() { + debug!( + "No internet connection and Python has not been setup, falling back to global binary in PATH" + ); + + self.global = true; + + // Otherwise try and install the tool + } else { + let setup = match last_versions.get("python") { + Some(last) => version != last, + None => true, + }; + + if setup || !self.tool.get_product_dir().exists() { + self.console + .out + .print_checkpoint(Checkpoint::Setup, format!("installing python {version}"))?; + + if self.tool.setup(version, InstallOptions::default()).await? { + last_versions.insert("python".into(), version.to_owned()); + installed += 1; + } + } + } + self.tool.locate_globals_dirs().await?; + + + + + + Ok(installed) + } + + async fn teardown(&mut self) -> miette::Result<()> { + self.tool.teardown().await?; + + Ok(()) + } +} + + +#[async_trait] +impl DependencyManager for PythonTool { + fn create_command(&self, python: &PythonTool) -> miette::Result { + let mut cmd = Command::new("python"); + cmd.with_console(self.console.clone()); + cmd.envs(get_proto_env_vars()); + if !self.global { + cmd.env( + "PATH", + prepend_path_env_var(get_python_env_paths(&self.proto_env)), + ); + } + + if let Some(version) = get_proto_version_env(&self.tool) { + cmd.env("PROTO_PYTHON_VERSION", version); + } + + if let Some(version) = get_proto_version_env(&python.tool) { + cmd.env("PROTO_PYTHON_VERSION", version); + } + + Ok(cmd) + } + + #[instrument(skip_all)] + async fn dedupe_dependencies( + &self, + _python: &PythonTool, + _working_dir: &Path, + _log: bool, + ) -> miette::Result<()> { + // Not supported! + + Ok(()) + } + + fn get_lock_filename(&self) -> String { + String::from("requirements.txt") + } + + fn get_manifest_filename(&self) -> String { + String::from("requirements.txt") + } + + #[instrument(skip_all)] + async fn get_resolved_dependencies( + &self, + project_root: &Path, + ) -> miette::Result { + let Some(lockfile_path) = + fs::find_upwards_until("requirements.txt", project_root, get_workspace_root()) + else { + return Ok(FxHashMap::default()); + }; + + Ok(load_lockfile_dependencies(lockfile_path)?) + } + #[instrument(skip_all)] + async fn install_dependencies( + &self, + python: &PythonTool, + working_dir: &Path, + log: bool, + ) -> miette::Result<()> { + let mut cmd = self.create_command(python)?; + + // TODO: DONT KNOW IF CORRECT LOCATION, BECAUSE IT IS HANDLING THE TOOL INSTALLATION + if let Some(pip_config) = &self.config.pip { + + cmd.args(["install"]) + // .args(&args) + .cwd(working_dir) + .set_print_command(log); + + //TODO: only read from root, but ready for sub virtual environments + if let Some(requirements_path) = fs::find_upwards_until("requirements.txt", get_workspace_root(), get_workspace_root()) { + cmd.args(["-r", &requirements_path.as_os_str().to_str().unwrap()]); + } + if let Some(install_args) = &pip_config.install_args { + cmd.args(install_args); + } + + let mut cmd = cmd.create_async(); + if env::var("MOON_TEST_HIDE_INSTALL_OUTPUT").is_ok() { + cmd.exec_capture_output().await?; + } else { + cmd.exec_stream_output().await?; + } + } + + Ok(()) + } + + #[instrument(skip_all)] + async fn install_focused_dependencies( + &self, + _python: &PythonTool, + _packages: &[String], + _production_only: bool, + ) -> miette::Result<()> { + // TODO: Implement for docker purposes + Ok(()) + } +} diff --git a/packages/types/src/project-config.ts b/packages/types/src/project-config.ts index de23f2100c3..578432efbed 100644 --- a/packages/types/src/project-config.ts +++ b/packages/types/src/project-config.ts @@ -2,8 +2,8 @@ /* eslint-disable */ -import type { UnresolvedVersionSpec } from './toolchain-config'; import type { PartialTaskConfig, PlatformType, TaskConfig } from './tasks-config'; +import type { UnresolvedVersionSpec } from './toolchain-config'; /** The scope and or relationship of the dependency. */ export type DependencyScope = 'build' | 'development' | 'peer' | 'production' | 'root'; @@ -66,18 +66,7 @@ export interface ProjectDockerConfig { } /** Supported programming languages that each project can be written in. */ -export type LanguageType = - | 'bash' - | 'batch' - | 'go' - | 'javascript' - | 'php' - | 'python' - | 'ruby' - | 'rust' - | 'typescript' - | 'unknown' - | string; +export type LanguageType = 'bash' | 'batch' | 'go' | 'javascript' | 'php' | 'python' | 'ruby' | 'rust' | 'typescript' | 'unknown' | string; export type OwnersPaths = string[] | Record; @@ -164,6 +153,8 @@ export interface ProjectToolchainConfig { deno: ProjectToolchainCommonToolConfig | null; /** Overrides `node` settings. */ node: ProjectToolchainCommonToolConfig | null; + /** Overrides `python` settings. */ + python: ProjectToolchainCommonToolConfig | null; /** Overrides `rust` settings. */ rust: ProjectToolchainCommonToolConfig | null; /** Overrides `typescript` settings. */ @@ -171,14 +162,7 @@ export interface ProjectToolchainConfig { } /** The type of project, for categorizing. */ -export type ProjectType = - | 'application' - | 'automation' - | 'configuration' - | 'library' - | 'scaffolding' - | 'tool' - | 'unknown'; +export type ProjectType = 'application' | 'automation' | 'configuration' | 'library' | 'scaffolding' | 'tool' | 'unknown'; /** Controls how tasks are inherited. */ export interface ProjectWorkspaceInheritedTasksConfig { @@ -405,6 +389,8 @@ export interface PartialProjectToolchainConfig { deno?: PartialProjectToolchainCommonToolConfig | null; /** Overrides `node` settings. */ node?: PartialProjectToolchainCommonToolConfig | null; + /** Overrides `python` settings. */ + python?: PartialProjectToolchainCommonToolConfig | null; /** Overrides `rust` settings. */ rust?: PartialProjectToolchainCommonToolConfig | null; /** Overrides `typescript` settings. */ diff --git a/packages/types/src/tasks-config.ts b/packages/types/src/tasks-config.ts index 3019f12b7b3..2b5ed1d2ee9 100644 --- a/packages/types/src/tasks-config.ts +++ b/packages/types/src/tasks-config.ts @@ -32,16 +32,7 @@ export type TaskOperatingSystem = 'linux' | 'macos' | 'windows'; export type TaskOutputStyle = 'buffer' | 'buffer-only-failure' | 'hash' | 'none' | 'stream'; /** A list of available shells on Unix. */ -export type TaskUnixShell = - | 'bash' - | 'elvish' - | 'fish' - | 'ion' - | 'murex' - | 'nu' - | 'pwsh' - | 'xonsh' - | 'zsh'; +export type TaskUnixShell = 'bash' | 'elvish' | 'fish' | 'ion' | 'murex' | 'nu' | 'pwsh' | 'xonsh' | 'zsh'; /** A list of available shells on Windows. */ export type TaskWindowsShell = 'bash' | 'elvish' | 'fish' | 'murex' | 'nu' | 'pwsh' | 'xonsh'; @@ -157,7 +148,7 @@ export interface TaskOptionsConfig { } /** Platforms that each programming language can belong to. */ -export type PlatformType = 'bun' | 'deno' | 'node' | 'rust' | 'system' | 'unknown'; +export type PlatformType = 'bun' | 'deno' | 'node' | 'python' | 'rust' | 'system' | 'unknown'; /** Preset options to inherit. */ export type TaskPreset = 'server' | 'watcher'; @@ -220,7 +211,7 @@ export interface TaskConfig { * be automatically detected. * * @default 'unknown' - * @type {'bun' | 'deno' | 'node' | 'rust' | 'system' | 'unknown'} + * @type {'bun' | 'deno' | 'node' | 'python' | 'rust' | 'system' | 'unknown'} */ platform: PlatformType; /** The preset to apply for the task. Will inherit default options. */ diff --git a/packages/types/src/template-config.ts b/packages/types/src/template-config.ts index 771c72fcfe5..68864da4a37 100644 --- a/packages/types/src/template-config.ts +++ b/packages/types/src/template-config.ts @@ -92,11 +92,7 @@ export interface TemplateVariableStringSetting { type: 'string'; } -export type TemplateVariable = - | TemplateVariableBoolSetting - | TemplateVariableEnumSetting - | TemplateVariableNumberSetting - | TemplateVariableStringSetting; +export type TemplateVariable = TemplateVariableBoolSetting | TemplateVariableEnumSetting | TemplateVariableNumberSetting | TemplateVariableStringSetting; /** * Configures a template and its files to be scaffolded. @@ -197,11 +193,7 @@ export interface PartialTemplateVariableStringSetting { type?: 'string' | null; } -export type PartialTemplateVariable = - | PartialTemplateVariableBoolSetting - | PartialTemplateVariableEnumSetting - | PartialTemplateVariableNumberSetting - | PartialTemplateVariableStringSetting; +export type PartialTemplateVariable = PartialTemplateVariableBoolSetting | PartialTemplateVariableEnumSetting | PartialTemplateVariableNumberSetting | PartialTemplateVariableStringSetting; /** * Configures a template and its files to be scaffolded. diff --git a/packages/types/src/toolchain-config.ts b/packages/types/src/toolchain-config.ts index 618e6296f8c..4b53e575185 100644 --- a/packages/types/src/toolchain-config.ts +++ b/packages/types/src/toolchain-config.ts @@ -3,16 +3,7 @@ /* eslint-disable */ /** Formats that a `package.json` version dependency can be. */ -export type NodeVersionFormat = - | 'file' - | 'link' - | 'star' - | 'version' - | 'version-caret' - | 'version-tilde' - | 'workspace' - | 'workspace-caret' - | 'workspace-tilde'; +export type NodeVersionFormat = 'file' | 'link' | 'star' | 'version' | 'version-caret' | 'version-tilde' | 'workspace' | 'workspace-caret' | 'workspace-tilde'; export type PluginLocator = string; @@ -243,6 +234,26 @@ export interface NodeConfig { yarn: YarnConfig | null; } +export interface PipConfig { + /** List of arguments to append to `pip install` commands. */ + installArgs: string[] | null; + /** The version of pip to download, install, and run `pip` tasks with. */ + version: UnresolvedVersionSpec | null; +} + +export interface PythonConfig { + /** Options for pnpm, when used as a package manager. */ + pip: PipConfig | null; + /** Location of the WASM plugin to use for Python support. */ + plugin: PluginLocator | null; + /** + * The version of Python to download, install, and run `python` tasks with. + * + * @envvar MOON_PYTHON_VERSION + */ + version: UnresolvedVersionSpec | null; +} + /** * Configures and enables the Rust platform. * Docs: https://moonrepo.dev/docs/config/toolchain#rust @@ -358,6 +369,8 @@ export interface ToolchainConfig { extends: string | null; /** Configures and enables the Node.js platform. */ node: NodeConfig | null; + /** Configures and enables the Python platform. */ + python: PythonConfig | null; /** Configures and enables the Rust platform. */ rust: RustConfig | null; /** All configured toolchains by unique ID. */ @@ -582,6 +595,26 @@ export interface PartialNodeConfig { yarn?: PartialYarnConfig | null; } +export interface PartialPipConfig { + /** List of arguments to append to `pip install` commands. */ + installArgs?: string[] | null; + /** The version of pip to download, install, and run `pip` tasks with. */ + version?: UnresolvedVersionSpec | null; +} + +export interface PartialPythonConfig { + /** Options for pnpm, when used as a package manager. */ + pip?: PartialPipConfig | null; + /** Location of the WASM plugin to use for Python support. */ + plugin?: PluginLocator | null; + /** + * The version of Python to download, install, and run `python` tasks with. + * + * @envvar MOON_PYTHON_VERSION + */ + version?: UnresolvedVersionSpec | null; +} + /** * Configures and enables the Rust platform. * Docs: https://moonrepo.dev/docs/config/toolchain#rust @@ -697,6 +730,8 @@ export interface PartialToolchainConfig { extends?: string | null; /** Configures and enables the Node.js platform. */ node?: PartialNodeConfig | null; + /** Configures and enables the Python platform. */ + python?: PartialPythonConfig | null; /** Configures and enables the Rust platform. */ rust?: PartialRustConfig | null; /** All configured toolchains by unique ID. */ diff --git a/packages/types/src/workspace-config.ts b/packages/types/src/workspace-config.ts index d9d0fb6248b..541caa9730c 100644 --- a/packages/types/src/workspace-config.ts +++ b/packages/types/src/workspace-config.ts @@ -519,10 +519,7 @@ export interface PartialWorkspaceProjectsConfig { sources?: Record | null; } -export type PartialWorkspaceProjects = - | PartialWorkspaceProjectsConfig - | string[] - | Record; +export type PartialWorkspaceProjects = PartialWorkspaceProjectsConfig | string[] | Record; /** Configures aspects of the task runner (also known as the action pipeline). */ export interface PartialRunnerConfig { diff --git a/website/static/schemas/project.json b/website/static/schemas/project.json index 1becab652bd..67b3a089832 100644 --- a/website/static/schemas/project.json +++ b/website/static/schemas/project.json @@ -376,6 +376,7 @@ "bun", "deno", "node", + "python", "rust", "system", "unknown" @@ -616,6 +617,19 @@ ], "markdownDescription": "Overrides `node` settings." }, + "python": { + "title": "python", + "description": "Overrides python settings.", + "anyOf": [ + { + "$ref": "#/definitions/ProjectToolchainCommonToolConfig" + }, + { + "type": "null" + } + ], + "markdownDescription": "Overrides `python` settings." + }, "rust": { "title": "rust", "description": "Overrides rust settings.", diff --git a/website/static/schemas/tasks.json b/website/static/schemas/tasks.json index d24fcb9122a..0b1a63feeec 100644 --- a/website/static/schemas/tasks.json +++ b/website/static/schemas/tasks.json @@ -83,6 +83,7 @@ "bun", "deno", "node", + "python", "rust", "system", "unknown" diff --git a/website/static/schemas/toolchain.json b/website/static/schemas/toolchain.json index cb5fd0cdd96..902b478dfb8 100644 --- a/website/static/schemas/toolchain.json +++ b/website/static/schemas/toolchain.json @@ -56,6 +56,18 @@ } ] }, + "python": { + "title": "python", + "description": "Configures and enables the Python platform.", + "anyOf": [ + { + "$ref": "#/definitions/PythonConfig" + }, + { + "type": "null" + } + ] + }, "rust": { "title": "rust", "description": "Configures and enables the Rust platform.", @@ -540,6 +552,41 @@ }, "additionalProperties": false }, + "PipConfig": { + "type": "object", + "properties": { + "installArgs": { + "title": "installArgs", + "description": "List of arguments to append to pip install commands.", + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ], + "markdownDescription": "List of arguments to append to `pip install` commands." + }, + "version": { + "title": "version", + "description": "The version of pip to download, install, and run pip tasks with.", + "anyOf": [ + { + "$ref": "#/definitions/UnresolvedVersionSpec" + }, + { + "type": "null" + } + ], + "markdownDescription": "The version of pip to download, install, and run `pip` tasks with." + } + }, + "additionalProperties": false + }, "PluginLocator": { "description": "Strategies and protocols for locating plugins.", "type": "string" @@ -585,6 +632,49 @@ }, "additionalProperties": false }, + "PythonConfig": { + "type": "object", + "properties": { + "pip": { + "title": "pip", + "description": "Options for pnpm, when used as a package manager.", + "anyOf": [ + { + "$ref": "#/definitions/PipConfig" + }, + { + "type": "null" + } + ] + }, + "plugin": { + "title": "plugin", + "description": "Location of the WASM plugin to use for Python support.", + "anyOf": [ + { + "$ref": "#/definitions/PluginLocator" + }, + { + "type": "null" + } + ] + }, + "version": { + "title": "version", + "description": "The version of Python to download, install, and run python tasks with.", + "anyOf": [ + { + "$ref": "#/definitions/UnresolvedVersionSpec" + }, + { + "type": "null" + } + ], + "markdownDescription": "The version of Python to download, install, and run `python` tasks with." + } + }, + "additionalProperties": false + }, "RustConfig": { "description": "Configures and enables the Rust platform. Docs: https://moonrepo.dev/docs/config/toolchain#rust", "type": "object", From 6e9ca3ac2e8694f05fe3e2e8ab996f7ab46375f6 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:09:21 +0200 Subject: [PATCH 02/28] next iteration --- legacy/python/platform/src/python_platform.rs | 48 +++++++++---------- legacy/python/platform/src/toolchain_hash.rs | 12 ++--- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 1ef0f635198..850ae978169 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -14,7 +14,7 @@ use moon_logger::debug; use moon_platform::{Platform, Runtime, RuntimeReq}; use moon_process::Command; use moon_project::Project; -use moon_python_lang::pip_requirements::load_lockfile_dependencies; +use moon_python_lang::pip_requirements::{self, load_lockfile_dependencies}; use moon_python_tool::{get_python_tool_paths, PythonTool}; use moon_task::Task; use moon_tool::{get_proto_version_env, prepend_path_env_var, Tool, ToolManager}; @@ -248,11 +248,18 @@ impl Platform for PythonPlatform { hasher: &mut ContentHasher, _hasher_config: &HasherConfig, ) -> miette::Result<()> { - hasher.hash_content(PythonToolchainHash { - pip: Some(self.config.pip).expect("S"), - version: Some(self.config.version).expect("S"), - requirements_dependencies: "".into(), - })?; + + if let Some(python_version) = &self.config.version { + let mut deps = BTreeMap::new(); + if let Some(pip_requirements) = find_requirements_txt(&get_workspace_root(), &get_workspace_root()) { + deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); + } + + hasher.hash_content(PythonToolchainHash { + version: python_version.clone(), + dependencies: deps, + })?; + } Ok(()) } @@ -265,27 +272,20 @@ impl Platform for PythonPlatform { hasher: &mut ContentHasher, _hasher_config: &HasherConfig, ) -> miette::Result<()> { - let lockfile_path = project.root.join("requirements.txt"); - - - debug!( - target: LOG_TARGET, - "{} does not exist, installing", - lockfile_path.display() - ); - // Not running in the Cargo workspace root, not sure how to handle! - if !lockfile_path.exists() { - return Ok(()); + if let Some(python_version) = &self.config.version { + let mut deps = BTreeMap::new(); + if let Some(pip_requirements) = find_requirements_txt(&get_workspace_root(), &get_workspace_root()) { + deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); + } + + hasher.hash_content(PythonToolchainHash { + version: python_version.clone(), + dependencies: deps, + })?; } - let mut hash = PythonTargetHash::new(None); - - // Use the resolved dependencies from the lockfile directly, - // since it also takes into account features and workspace members. - hash.locked_dependencies = BTreeMap::from_iter(load_lockfile_dependencies(lockfile_path)?); - - hasher.hash_content(hash)?; + Ok(()) } diff --git a/legacy/python/platform/src/toolchain_hash.rs b/legacy/python/platform/src/toolchain_hash.rs index 540bb06c372..9ce77a8ea07 100644 --- a/legacy/python/platform/src/toolchain_hash.rs +++ b/legacy/python/platform/src/toolchain_hash.rs @@ -4,18 +4,18 @@ use std::collections::BTreeMap; hash_content!( pub struct PythonToolchainHash { - pub pip: Option, - pub version: Option, - pub requirements_dependencies: String, + // pub pip: Option, + pub version: UnresolvedVersionSpec, + pub dependencies: BTreeMap>, } ); impl PythonToolchainHash { - pub fn new(python_version: Option, pip_config: Option) -> Self { + pub fn new(python_version: UnresolvedVersionSpec, pip_config: Option) -> Self { PythonToolchainHash { version: python_version, - requirements_dependencies: "".into(), - pip: pip_config, + dependencies: BTreeMap::new(), + // pip: pip_config, } } } \ No newline at end of file From 431e197d9654c639e2acb06e30a8296e9338c8d4 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:44:27 +0200 Subject: [PATCH 03/28] Latest version --- crates/config/src/toolchain/python_config.rs | 3 + .../platform/src/actions/install_deps.rs | 44 +++++++------- legacy/python/platform/src/python_platform.rs | 12 ++-- legacy/python/platform/src/target_hash.rs | 16 +++--- legacy/python/platform/src/toolchain_hash.rs | 22 ++++--- legacy/python/tool/src/python_tool.rs | 57 +++++++++++++++---- packages/types/src/project-config.ts | 2 +- 7 files changed, 98 insertions(+), 58 deletions(-) diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs index 5214f52cca1..b4dddf50659 100644 --- a/crates/config/src/toolchain/python_config.rs +++ b/crates/config/src/toolchain/python_config.rs @@ -29,6 +29,9 @@ pub struct PythonConfig { #[setting(default = ".", skip)] pub venv_root: String, + #[setting(default = ".venv", skip)] + pub venv_name: String, + /// The version of Python to download, install, and run `python` tasks with. #[setting(env = "MOON_PYTHON_VERSION")] pub version: Option, diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs index a7ed87a496e..b60aae71574 100644 --- a/legacy/python/platform/src/actions/install_deps.rs +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -13,8 +13,27 @@ pub async fn install_deps( ) -> miette::Result> { let mut operations = vec![]; + // python.exec_python(args, working_dir) + + if let Some(pip_config) = &python.config.pip { + // Very first step: Activate virtual environment + console + .out + .print_checkpoint(Checkpoint::Setup, format!("activate virtual environment"))?; + let virtual_environment = &get_workspace_root().join(python.config.venv_name.clone()); + + if !virtual_environment.exists() { + let args = vec!["-m", "venv", virtual_environment.as_os_str().to_str().unwrap()]; + operations.push( + Operation::task_execution(format!("python {} ", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); + } + + if let Some(pip_version) = &pip_config.version { @@ -27,29 +46,16 @@ pub async fn install_deps( } else { format!("pip{}", pip_version.to_owned().to_string().replace("~", "~=")) }; - let args = vec!["-m", "pip", "install","--quiet", "-U", &p_version]; - + let args = vec!["-m", "pip", "install", "-U", &p_version]; +// #"--quiet", operations.push( - Operation::task_execution(format!("python {} ", args.join(" "))) + Operation::task_execution(format!(" {} ", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, ); } - console - .out - .print_checkpoint(Checkpoint::Setup, format!("activate virtual environment"))?; - let virtual_environment = &get_workspace_root().join(".venv"); - if !virtual_environment.exists() { - let args = vec!["-m", "venv", virtual_environment.as_os_str().to_str().unwrap()]; - operations.push( - Operation::task_execution(format!("python {} ", args.join(" "))) - .track_async(|| python.exec_python(args, working_dir)) - .await?, - ); - } - @@ -58,8 +64,8 @@ pub async fn install_deps( .out .print_checkpoint(Checkpoint::Setup, format!("pip dependencies from {}", req.as_os_str().to_str().unwrap()))?; - let mut args = vec!["-m", "pip", "install", "--quiet"]; - + let mut args = vec!["-m", "pip", "install"]; + // #, "--quiet" if pip_config.install_args.is_some() { args.extend(pip_config.install_args.as_ref().unwrap().iter().map(|c| c.as_str())); } @@ -67,7 +73,7 @@ pub async fn install_deps( args.extend(["-r", req.as_os_str().to_str().unwrap()]); operations.push( - Operation::task_execution(format!("python {}", args.join(" "))) + Operation::task_execution(format!(" {}", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, ); diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 850ae978169..33a395cb007 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -1,5 +1,5 @@ use crate::{ - actions, find_requirements_txt, target_hash::PythonTargetHash, toolchain_hash::PythonToolchainHash + actions, find_requirements_txt, toolchain_hash::PythonToolchainHash }; use moon_action::Operation; use moon_action_context::ActionContext; @@ -8,13 +8,13 @@ use moon_config::{ HasherConfig, PlatformType, ProjectConfig, ProjectsAliasesList, ProjectsSourcesList, PythonConfig, UnresolvedVersionSpec, }; -use moon_console::{Checkpoint, Console}; +use moon_console::Console; use moon_hash::ContentHasher; -use moon_logger::debug; +// use moon_logger::debug; use moon_platform::{Platform, Runtime, RuntimeReq}; use moon_process::Command; use moon_project::Project; -use moon_python_lang::pip_requirements::{self, load_lockfile_dependencies}; +use moon_python_lang::pip_requirements::load_lockfile_dependencies; use moon_python_tool::{get_python_tool_paths, PythonTool}; use moon_task::Task; use moon_tool::{get_proto_version_env, prepend_path_env_var, Tool, ToolManager}; @@ -26,7 +26,7 @@ use std::{ }; use tracing::instrument; -const LOG_TARGET: &str = "moon:python-platform"; +// const LOG_TARGET: &str = "moon:python-platform"; pub struct PythonPlatform { pub config: PythonConfig, @@ -267,7 +267,7 @@ impl Platform for PythonPlatform { #[instrument(skip_all)] async fn hash_run_target( &self, - project: &Project, + _project: &Project, _runtime: &Runtime, hasher: &mut ContentHasher, _hasher_config: &HasherConfig, diff --git a/legacy/python/platform/src/target_hash.rs b/legacy/python/platform/src/target_hash.rs index e32a6b3f9a1..155f3e1d195 100644 --- a/legacy/python/platform/src/target_hash.rs +++ b/legacy/python/platform/src/target_hash.rs @@ -8,11 +8,11 @@ hash_content!( } ); -impl PythonTargetHash { - pub fn new(python_version: Option) -> Self { - PythonTargetHash { - python_version: python_version.unwrap_or_else(|| "unknown".into()), - locked_dependencies: BTreeMap::new(), - } - } -} +// impl PythonTargetHash { +// pub fn new(python_version: Option) -> Self { +// PythonTargetHash { +// python_version: python_version.unwrap_or_else(|| "unknown".into()), +// locked_dependencies: BTreeMap::new(), +// } +// } +// } diff --git a/legacy/python/platform/src/toolchain_hash.rs b/legacy/python/platform/src/toolchain_hash.rs index 9ce77a8ea07..22f0b8c376c 100644 --- a/legacy/python/platform/src/toolchain_hash.rs +++ b/legacy/python/platform/src/toolchain_hash.rs @@ -1,21 +1,19 @@ -use moon_config::{PipConfig, UnresolvedVersionSpec}; +use moon_config::UnresolvedVersionSpec; use moon_hash::hash_content; use std::collections::BTreeMap; hash_content!( - pub struct PythonToolchainHash { - // pub pip: Option, + pub struct PythonToolchainHash { pub version: UnresolvedVersionSpec, pub dependencies: BTreeMap>, } ); -impl PythonToolchainHash { - pub fn new(python_version: UnresolvedVersionSpec, pip_config: Option) -> Self { - PythonToolchainHash { - version: python_version, - dependencies: BTreeMap::new(), - // pip: pip_config, - } - } -} \ No newline at end of file +// impl PythonToolchainHash { +// pub fn new(python_version: UnresolvedVersionSpec) -> Self { +// PythonToolchainHash { +// version: python_version, +// dependencies: BTreeMap::new(), +// } +// } +// } \ No newline at end of file diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs index 75bb82e7732..98a102317e7 100644 --- a/legacy/python/tool/src/python_tool.rs +++ b/legacy/python/tool/src/python_tool.rs @@ -6,8 +6,7 @@ use moon_console::{Checkpoint, Console}; use moon_utils::get_workspace_root; use moon_process::Command; use moon_tool::{ - async_trait, get_proto_paths, load_tool_plugin, prepend_path_env_var, use_global_tool_on_path, - Tool, get_proto_version_env, get_proto_env_vars, DependencyManager + async_trait, get_proto_env_vars, get_proto_paths, get_proto_version_env, load_tool_plugin, prepend_path_env_var, use_global_tool_on_path, DependencyManager, Tool }; use moon_toolchain::RuntimeReq; use proto_core::flow::install::InstallOptions; @@ -29,20 +28,48 @@ pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { // let mut paths = get_proto_paths(proto_env); // let mut paths:Vec = []; - let paths = python_tool.tool.get_globals_dirs() - .iter() - .cloned() - .collect::>(); + // let mut python_command = "python"; - paths -} + let venv_python = &get_workspace_root().join(python_tool.config.venv_name.clone()); + + let paths; + + if venv_python.exists() { + paths = vec![venv_python.join("Scripts").clone()]; + } else { + // paths = python_tool.tool.get_globals_dirs() + // .iter() + // .cloned() + // .collect::>(); + paths = get_proto_paths(&python_tool.proto_env); + } -pub fn get_python_env_paths(proto_env: &ProtoEnvironment) -> Vec { - let paths = get_proto_paths(proto_env); + // for p in python_tool.tool.get_globals_dirs().iter() { + // debug!( + // target: LOG_TARGET, + // "Proto Env {} ", + // p.clone().display(), + // ); + // } + + + + + debug!( + target: LOG_TARGET, + "Proto Env {} ", + map_list(&paths, |c| color::label(c.display().to_string())), + ); paths } +// pub fn get_python_env_paths(proto_env: &ProtoEnvironment) -> Vec { +// let paths = get_proto_paths(proto_env); + +// paths +// } + pub struct PythonTool { pub config: PythonConfig, @@ -91,12 +118,18 @@ impl PythonTool { I: IntoIterator, S: AsRef, { + + // Check if venv is already created. + + + + Command::new("python") .args(args) .envs(get_proto_env_vars()) .env( "PATH", - prepend_path_env_var(get_python_env_paths(&self.proto_env)), + prepend_path_env_var(get_python_tool_paths(&self)), ) .cwd(working_dir) .with_console(self.console.clone()) @@ -195,7 +228,7 @@ impl DependencyManager for PythonTool { if !self.global { cmd.env( "PATH", - prepend_path_env_var(get_python_env_paths(&self.proto_env)), + prepend_path_env_var(get_python_tool_paths(&self)), ); } diff --git a/packages/types/src/project-config.ts b/packages/types/src/project-config.ts index 578432efbed..9be59614c27 100644 --- a/packages/types/src/project-config.ts +++ b/packages/types/src/project-config.ts @@ -2,8 +2,8 @@ /* eslint-disable */ -import type { PartialTaskConfig, PlatformType, TaskConfig } from './tasks-config'; import type { UnresolvedVersionSpec } from './toolchain-config'; +import type { PartialTaskConfig, PlatformType, TaskConfig } from './tasks-config'; /** The scope and or relationship of the dependency. */ export type DependencyScope = 'build' | 'development' | 'peer' | 'production' | 'root'; From b0b9a81bab8b47e44e7317c1342d2ed5a1868a16 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:18:09 +0200 Subject: [PATCH 04/28] Run linter --- crates/config/src/project/overrides_config.rs | 2 +- crates/config/src/toolchain/python_config.rs | 4 +- legacy/python/lang/src/lib.rs | 3 +- legacy/python/lang/src/pip_requirements.rs | 4 +- .../platform/src/actions/install_deps.rs | 59 ++++++----- legacy/python/platform/src/actions/mod.rs | 2 +- legacy/python/platform/src/lib.rs | 2 - legacy/python/platform/src/python_platform.rs | 51 ++++------ legacy/python/platform/src/toolchain_hash.rs | 6 +- legacy/python/tool/src/python_tool.rs | 99 +++++-------------- 10 files changed, 89 insertions(+), 143 deletions(-) diff --git a/crates/config/src/project/overrides_config.rs b/crates/config/src/project/overrides_config.rs index ea8982aa8d8..0430d22900a 100644 --- a/crates/config/src/project/overrides_config.rs +++ b/crates/config/src/project/overrides_config.rs @@ -48,7 +48,7 @@ cacheable!( /// Overrides `deno` settings. #[setting(nested)] pub deno: Option, - + /// Overrides `python` settings. #[setting(nested)] pub python: Option, diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs index b4dddf50659..1dd8d8577f9 100644 --- a/crates/config/src/toolchain/python_config.rs +++ b/crates/config/src/toolchain/python_config.rs @@ -4,7 +4,6 @@ use serde::Serialize; use version_spec::UnresolvedVersionSpec; use warpgate_api::PluginLocator; - #[derive(Clone, Config, Debug, PartialEq, Serialize)] pub struct PipConfig { /// List of arguments to append to `pip install` commands. @@ -14,7 +13,6 @@ pub struct PipConfig { pub version: Option, } - #[derive(Clone, Config, Debug, PartialEq)] pub struct PythonConfig { /// Location of the WASM plugin to use for Python support. @@ -23,7 +21,7 @@ pub struct PythonConfig { /// Options for pnpm, when used as a package manager. #[setting(nested)] pub pip: Option, - + /// The relative root of the virtual environment workspace. Default to moon's /// workspace root #[setting(default = ".", skip)] diff --git a/legacy/python/lang/src/lib.rs b/legacy/python/lang/src/lib.rs index d55284cc8f7..90a0e45a446 100644 --- a/legacy/python/lang/src/lib.rs +++ b/legacy/python/lang/src/lib.rs @@ -1,5 +1,4 @@ pub mod pip_requirements; - -pub use pip_requirements::*; pub use moon_lang::LockfileDependencyVersions; +pub use pip_requirements::*; diff --git a/legacy/python/lang/src/pip_requirements.rs b/legacy/python/lang/src/pip_requirements.rs index d4c7aa3ea25..c636b7dc347 100644 --- a/legacy/python/lang/src/pip_requirements.rs +++ b/legacy/python/lang/src/pip_requirements.rs @@ -1,13 +1,13 @@ use cached::proc_macro::cached; use moon_lang::LockfileDependencyVersions; use rustc_hash::FxHashMap; -use std::path::PathBuf; use std::fs; +use std::path::PathBuf; #[cached(result)] pub fn load_lockfile_dependencies(path: PathBuf) -> miette::Result { let mut deps: LockfileDependencyVersions = FxHashMap::default(); - + let lockfile = fs::read_to_string(path.as_path()).expect("Unable to read file"); let dep = deps.entry(path.display().to_string()).or_default(); dep.push(lockfile); diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs index b60aae71574..4bdb05fd8c3 100644 --- a/legacy/python/platform/src/actions/install_deps.rs +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -14,71 +14,80 @@ pub async fn install_deps( let mut operations = vec![]; // python.exec_python(args, working_dir) - if let Some(pip_config) = &python.config.pip { - // Very first step: Activate virtual environment console .out .print_checkpoint(Checkpoint::Setup, format!("activate virtual environment"))?; let virtual_environment = &get_workspace_root().join(python.config.venv_name.clone()); - - if !virtual_environment.exists() { - let args = vec!["-m", "venv", virtual_environment.as_os_str().to_str().unwrap()]; + + if !virtual_environment.exists() { + let args = vec![ + "-m", + "venv", + virtual_environment.as_os_str().to_str().unwrap(), + ]; operations.push( Operation::task_execution(format!("python {} ", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, - ); + ); } - - - if let Some(pip_version) = &pip_config.version { console .out .print_checkpoint(Checkpoint::Setup, format!("install pip {pip_version}"))?; - + let p_version: String = if pip_version.is_latest() { format!("pip") } else { - format!("pip{}", pip_version.to_owned().to_string().replace("~", "~=")) + format!( + "pip{}", + pip_version.to_owned().to_string().replace("~", "~=") + ) }; let args = vec!["-m", "pip", "install", "-U", &p_version]; -// #"--quiet", + // #"--quiet", operations.push( Operation::task_execution(format!(" {} ", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, - ); + ); } - - - - if let Some(req) = find_requirements_txt(working_dir, &get_workspace_root()) { - console - .out - .print_checkpoint(Checkpoint::Setup, format!("pip dependencies from {}", req.as_os_str().to_str().unwrap()))?; + console.out.print_checkpoint( + Checkpoint::Setup, + format!( + "pip dependencies from {}", + req.as_os_str().to_str().unwrap() + ), + )?; let mut args = vec!["-m", "pip", "install"]; // #, "--quiet" if pip_config.install_args.is_some() { - args.extend(pip_config.install_args.as_ref().unwrap().iter().map(|c| c.as_str())); + args.extend( + pip_config + .install_args + .as_ref() + .unwrap() + .iter() + .map(|c| c.as_str()), + ); } - + args.extend(["-r", req.as_os_str().to_str().unwrap()]); - + operations.push( Operation::task_execution(format!(" {}", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, - ); + ); } } Ok(operations) -} \ No newline at end of file +} diff --git a/legacy/python/platform/src/actions/mod.rs b/legacy/python/platform/src/actions/mod.rs index b7b5cf56da6..31a65cfdfdf 100644 --- a/legacy/python/platform/src/actions/mod.rs +++ b/legacy/python/platform/src/actions/mod.rs @@ -1,3 +1,3 @@ mod install_deps; -pub use install_deps::*; \ No newline at end of file +pub use install_deps::*; diff --git a/legacy/python/platform/src/lib.rs b/legacy/python/platform/src/lib.rs index bb487c34931..d08d7820a8c 100644 --- a/legacy/python/platform/src/lib.rs +++ b/legacy/python/platform/src/lib.rs @@ -4,7 +4,6 @@ mod python_platform; mod target_hash; mod toolchain_hash; - pub use python_platform::*; use starbase_utils::fs; @@ -13,4 +12,3 @@ use std::path::{Path, PathBuf}; fn find_requirements_txt(starting_dir: &Path, workspace_root: &Path) -> Option { fs::find_upwards_until("requirements.txt", starting_dir, workspace_root) } - diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 33a395cb007..96cf4871da1 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -1,6 +1,4 @@ -use crate::{ - actions, find_requirements_txt, toolchain_hash::PythonToolchainHash -}; +use crate::{actions, find_requirements_txt, toolchain_hash::PythonToolchainHash}; use moon_action::Operation; use moon_action_context::ActionContext; use moon_common::Id; @@ -22,7 +20,9 @@ use moon_utils::{async_trait, get_workspace_root}; use proto_core::ProtoEnvironment; use rustc_hash::FxHashMap; use std::{ - collections::BTreeMap, path::{Path, PathBuf}, sync::Arc + collections::BTreeMap, + path::{Path, PathBuf}, + sync::Arc, }; use tracing::instrument; @@ -101,7 +101,6 @@ impl Platform for PythonPlatform { // PROJECT GRAPH fn is_project_in_dependency_workspace(&self, _project_source: &str) -> miette::Result { - Ok(false) } @@ -139,7 +138,10 @@ impl Platform for PythonPlatform { } fn get_dependency_configs(&self) -> miette::Result> { - Ok(Some(("requirements.txt".to_owned(), "requirements.txt".to_owned()))) + Ok(Some(( + "requirements.txt".to_owned(), + "requirements.txt".to_owned(), + ))) } async fn setup_toolchain(&mut self) -> miette::Result<()> { @@ -163,8 +165,6 @@ impl Platform for PythonPlatform { ); } - - self.toolchain.setup(&req, &mut last_versions).await?; // info!( @@ -192,10 +192,6 @@ impl Platform for PythonPlatform { ) -> miette::Result { let req = &runtime.requirement; - - - - if !self.toolchain.has(req) { self.toolchain.register( req, @@ -207,7 +203,7 @@ impl Platform for PythonPlatform { ) .await?, ); - } + } Ok(self.toolchain.setup(req, last_versions).await?) } @@ -239,7 +235,6 @@ impl Platform for PythonPlatform { Ok(mutated_files) } - // # Lockfile or manifests have not changed since last run, skipping dependency install #[instrument(skip_all)] async fn hash_manifest_deps( @@ -248,14 +243,15 @@ impl Platform for PythonPlatform { hasher: &mut ContentHasher, _hasher_config: &HasherConfig, ) -> miette::Result<()> { - if let Some(python_version) = &self.config.version { let mut deps = BTreeMap::new(); - if let Some(pip_requirements) = find_requirements_txt(&get_workspace_root(), &get_workspace_root()) { + if let Some(pip_requirements) = + find_requirements_txt(&get_workspace_root(), &get_workspace_root()) + { deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); } - - hasher.hash_content(PythonToolchainHash { + + hasher.hash_content(PythonToolchainHash { version: python_version.clone(), dependencies: deps, })?; @@ -272,21 +268,20 @@ impl Platform for PythonPlatform { hasher: &mut ContentHasher, _hasher_config: &HasherConfig, ) -> miette::Result<()> { - if let Some(python_version) = &self.config.version { let mut deps = BTreeMap::new(); - if let Some(pip_requirements) = find_requirements_txt(&get_workspace_root(), &get_workspace_root()) { + if let Some(pip_requirements) = + find_requirements_txt(&get_workspace_root(), &get_workspace_root()) + { deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); } - - hasher.hash_content(PythonToolchainHash { + + hasher.hash_content(PythonToolchainHash { version: python_version.clone(), dependencies: deps, })?; } - - Ok(()) } @@ -299,19 +294,15 @@ impl Platform for PythonPlatform { runtime: &Runtime, _working_dir: &Path, ) -> miette::Result { - let mut command = Command::new(&task.command); command.with_console(self.console.clone()); command.args(&task.args); - command.envs(&task.env); + command.envs(&task.env); if let Ok(python) = self.toolchain.get_for_version(&runtime.requirement) { if let Some(version) = get_proto_version_env(&python.tool) { command.env("PROTO_PYTHON_VERSION", version); - command.env( - "PATH", - prepend_path_env_var(get_python_tool_paths(&python)), - ); + command.env("PATH", prepend_path_env_var(get_python_tool_paths(&python))); } } diff --git a/legacy/python/platform/src/toolchain_hash.rs b/legacy/python/platform/src/toolchain_hash.rs index 22f0b8c376c..63df731d848 100644 --- a/legacy/python/platform/src/toolchain_hash.rs +++ b/legacy/python/platform/src/toolchain_hash.rs @@ -3,7 +3,7 @@ use moon_hash::hash_content; use std::collections::BTreeMap; hash_content!( - pub struct PythonToolchainHash { + pub struct PythonToolchainHash { pub version: UnresolvedVersionSpec, pub dependencies: BTreeMap>, } @@ -13,7 +13,7 @@ hash_content!( // pub fn new(python_version: UnresolvedVersionSpec) -> Self { // PythonToolchainHash { // version: python_version, -// dependencies: BTreeMap::new(), +// dependencies: BTreeMap::new(), // } // } -// } \ No newline at end of file +// } diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs index 98a102317e7..e88caa8ee56 100644 --- a/legacy/python/tool/src/python_tool.rs +++ b/legacy/python/tool/src/python_tool.rs @@ -1,29 +1,29 @@ use moon_config::PythonConfig; -use moon_python_lang::pip_requirements::{load_lockfile_dependencies}; -use moon_python_lang::{LockfileDependencyVersions}; use moon_console::{Checkpoint, Console}; +use moon_python_lang::pip_requirements::load_lockfile_dependencies; +use moon_python_lang::LockfileDependencyVersions; // use moon_logger::debug; -use moon_utils::get_workspace_root; +use moon_logger::{debug, map_list}; use moon_process::Command; use moon_tool::{ - async_trait, get_proto_env_vars, get_proto_paths, get_proto_version_env, load_tool_plugin, prepend_path_env_var, use_global_tool_on_path, DependencyManager, Tool + async_trait, get_proto_env_vars, get_proto_paths, get_proto_version_env, load_tool_plugin, + prepend_path_env_var, use_global_tool_on_path, DependencyManager, Tool, }; use moon_toolchain::RuntimeReq; +use moon_utils::get_workspace_root; use proto_core::flow::install::InstallOptions; use proto_core::{Id, ProtoEnvironment, Tool as ProtoTool, UnresolvedVersionSpec}; use rustc_hash::FxHashMap; +use starbase_styles::color; +use starbase_utils::fs; use std::env; use std::path::PathBuf; use std::sync::Arc; use std::{ffi::OsStr, path::Path}; -use starbase_utils::fs; use tracing::instrument; -use starbase_styles::color; -use moon_logger::{debug, map_list}; const LOG_TARGET: &str = "moon:python-tool"; - pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { // let mut paths = get_proto_paths(proto_env); // let mut paths:Vec = []; @@ -31,7 +31,7 @@ pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { // let mut python_command = "python"; let venv_python = &get_workspace_root().join(python_tool.config.venv_name.clone()); - + let paths; if venv_python.exists() { @@ -39,37 +39,19 @@ pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { } else { // paths = python_tool.tool.get_globals_dirs() // .iter() - // .cloned() + // .cloned() // .collect::>(); paths = get_proto_paths(&python_tool.proto_env); } - // for p in python_tool.tool.get_globals_dirs().iter() { - // debug!( - // target: LOG_TARGET, - // "Proto Env {} ", - // p.clone().display(), - // ); - // } - - - - - debug!( target: LOG_TARGET, "Proto Env {} ", - map_list(&paths, |c| color::label(c.display().to_string())), + map_list(&paths, |c| color::label(c.display().to_string())), ); paths } -// pub fn get_python_env_paths(proto_env: &ProtoEnvironment) -> Vec { -// let paths = get_proto_paths(proto_env); - -// paths -// } - pub struct PythonTool { pub config: PythonConfig, @@ -118,19 +100,10 @@ impl PythonTool { I: IntoIterator, S: AsRef, { - - // Check if venv is already created. - - - - Command::new("python") .args(args) .envs(get_proto_env_vars()) - .env( - "PATH", - prepend_path_env_var(get_python_tool_paths(&self)), - ) + .env("PATH", prepend_path_env_var(get_python_tool_paths(&self))) .cwd(working_dir) .with_console(self.console.clone()) .create_async() @@ -138,20 +111,7 @@ impl PythonTool { .await?; Ok(()) - - - // let mut cmd = Command::new("python"); - // cmd.with_console(self.console.clone()); - // cmd.envs(get_proto_env_vars()); - // cmd.envs(get_proto_env_vars()); - // if !self.global { - // cmd.env( - // "PATH", - // prepend_path_env_var(get_python_env_paths(&self.proto_env)), - // ); - // } } - } #[async_trait] @@ -203,33 +163,23 @@ impl Tool for PythonTool { } } self.tool.locate_globals_dirs().await?; - - - - - Ok(installed) } async fn teardown(&mut self) -> miette::Result<()> { self.tool.teardown().await?; - Ok(()) } } - #[async_trait] impl DependencyManager for PythonTool { fn create_command(&self, python: &PythonTool) -> miette::Result { let mut cmd = Command::new("python"); cmd.with_console(self.console.clone()); - cmd.envs(get_proto_env_vars()); + cmd.envs(get_proto_env_vars()); if !self.global { - cmd.env( - "PATH", - prepend_path_env_var(get_python_tool_paths(&self)), - ); + cmd.env("PATH", prepend_path_env_var(get_python_tool_paths(&self))); } if let Some(version) = get_proto_version_env(&self.tool) { @@ -284,23 +234,24 @@ impl DependencyManager for PythonTool { log: bool, ) -> miette::Result<()> { let mut cmd = self.create_command(python)?; - - // TODO: DONT KNOW IF CORRECT LOCATION, BECAUSE IT IS HANDLING THE TOOL INSTALLATION - if let Some(pip_config) = &self.config.pip { + if let Some(pip_config) = &self.config.pip { cmd.args(["install"]) - // .args(&args) - .cwd(working_dir) - .set_print_command(log); + .cwd(working_dir) + .set_print_command(log); //TODO: only read from root, but ready for sub virtual environments - if let Some(requirements_path) = fs::find_upwards_until("requirements.txt", get_workspace_root(), get_workspace_root()) { - cmd.args(["-r", &requirements_path.as_os_str().to_str().unwrap()]); + if let Some(requirements_path) = fs::find_upwards_until( + "requirements.txt", + get_workspace_root(), + get_workspace_root(), + ) { + cmd.args(["-r", &requirements_path.as_os_str().to_str().unwrap()]); } if let Some(install_args) = &pip_config.install_args { - cmd.args(install_args); + cmd.args(install_args); } - + let mut cmd = cmd.create_async(); if env::var("MOON_TEST_HIDE_INSTALL_OUTPUT").is_ok() { cmd.exec_capture_output().await?; From 3ce75277098eb49b82e22e5c471adfe040edf87a Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:00:15 +0200 Subject: [PATCH 05/28] run pip with install args --- .../platform/src/actions/install_deps.rs | 38 ++++++++++++++++++- legacy/python/platform/src/python_platform.rs | 3 +- legacy/python/tool/src/python_tool.rs | 5 ++- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs index 4bdb05fd8c3..8c2c20129c5 100644 --- a/legacy/python/platform/src/actions/install_deps.rs +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -57,7 +57,43 @@ pub async fn install_deps( ); } - if let Some(req) = find_requirements_txt(working_dir, &get_workspace_root()) { + let requirements_path = find_requirements_txt(working_dir, &get_workspace_root()); + + if let Some(install_args) = &pip_config.install_args { + if install_args.iter().any(|x| !x.starts_with("-")) { + if let Some(_) = requirements_path { + console.out.print_checkpoint( + Checkpoint::Setup, + "Skip installation via install args, found requirements.txt additional.", + )?; + } else { + console.out.print_checkpoint( + Checkpoint::Setup, + "pip dependencies from install args", + )?; + + let mut args = vec!["-m", "pip", "install"]; + if pip_config.install_args.is_some() { + args.extend( + pip_config + .install_args + .as_ref() + .unwrap() + .iter() + .map(|c| c.as_str()), + ); + } + + operations.push( + Operation::task_execution(format!(" {}", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); + } + } + } + + if let Some(req) = requirements_path { console.out.print_checkpoint( Checkpoint::Setup, format!( diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 96cf4871da1..45692908179 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -101,7 +101,8 @@ impl Platform for PythonPlatform { // PROJECT GRAPH fn is_project_in_dependency_workspace(&self, _project_source: &str) -> miette::Result { - Ok(false) + // Single version policy / only a root package.json + Ok(true) } #[instrument(skip_all)] diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs index e88caa8ee56..fc6ce537131 100644 --- a/legacy/python/tool/src/python_tool.rs +++ b/legacy/python/tool/src/python_tool.rs @@ -35,7 +35,10 @@ pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { let paths; if venv_python.exists() { - paths = vec![venv_python.join("Scripts").clone()]; + paths = vec![ + venv_python.join("Scripts").clone(), + venv_python.join("bin").clone(), + ]; } else { // paths = python_tool.tool.get_globals_dirs() // .iter() From 287fb778b7efc4097bd1b0b54f8b5c9eb6360f88 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Fri, 18 Oct 2024 11:02:38 -0700 Subject: [PATCH 06/28] build: Prepare v1.30 release. --- .yarn/versions/3fbef0f1.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .yarn/versions/3fbef0f1.yml diff --git a/.yarn/versions/3fbef0f1.yml b/.yarn/versions/3fbef0f1.yml new file mode 100644 index 00000000000..28fac241db0 --- /dev/null +++ b/.yarn/versions/3fbef0f1.yml @@ -0,0 +1,9 @@ +releases: + "@moonrepo/cli": minor + "@moonrepo/core-linux-arm64-gnu": minor + "@moonrepo/core-linux-arm64-musl": minor + "@moonrepo/core-linux-x64-gnu": minor + "@moonrepo/core-linux-x64-musl": minor + "@moonrepo/core-macos-arm64": minor + "@moonrepo/core-macos-x64": minor + "@moonrepo/core-windows-x64-msvc": minor From 542ded8452ee0053a70e038625f1306256da8650 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Wed, 23 Oct 2024 20:41:15 -0700 Subject: [PATCH 07/28] new: Add new workspace builder. (#1697) * Add workspace crate. * Add projects loading. * Add build. * Clean up build data. * Flesh out build. * Add caching. * Add to session. * Use focused graphs. * Add workspace mocker. * Delete old graph builder. * Polish. * Fix tests. * Fix tests. --- Cargo.lock | 30 +- .../action-graph/tests/action_graph_test.rs | 4 +- ...n_graph__run_task_by_target__runs_all.snap | 22 +- crates/app/Cargo.toml | 1 + crates/app/src/commands/docker/scaffold.rs | 4 +- crates/app/src/commands/node/run_script.rs | 5 +- crates/app/src/commands/project.rs | 8 +- crates/app/src/commands/task.rs | 5 +- crates/app/src/components.rs | 10 +- crates/app/src/session.rs | 31 +- crates/cache-item/src/lib.rs | 2 +- crates/cli/tests/action_graph_test.rs | 2 +- crates/cli/tests/docker_file_test.rs | 2 +- .../project_test__unknown_project.snap | 4 +- .../run_test__errors_for_unknown_project.snap | 4 +- crates/project-builder/src/project_builder.rs | 24 +- .../project-expander/src/project_expander.rs | 2 +- crates/project-graph/Cargo.toml | 11 +- crates/project-graph/src/lib.rs | 9 - crates/project-graph/src/project_graph.rs | 40 +- .../src/project_graph_builder.rs | 665 ----------------- .../project-graph/src/project_graph_cache.rs | 11 - .../project-graph/src/project_graph_error.rs | 36 +- .../project-graph/tests/project_graph_test.rs | 256 +++---- crates/task-builder/src/tasks_builder.rs | 4 +- crates/task-hasher/tests/task_hasher_test.rs | 10 +- crates/test-utils/Cargo.toml | 1 + crates/test-utils/src/lib.rs | 2 + crates/test-utils/src/project_graph.rs | 156 +--- crates/test-utils/src/workspace_mocker.rs | 175 +++++ crates/workspace/Cargo.toml | 31 + crates/workspace/src/lib.rs | 12 + .../src/project_build_data.rs} | 24 +- .../src/projects_locator.rs | 28 +- crates/workspace/src/repo_type.rs | 17 + crates/workspace/src/workspace_builder.rs | 681 ++++++++++++++++++ .../workspace/src/workspace_builder_error.rs | 38 + .../src/workspace_cache.rs} | 37 +- 38 files changed, 1280 insertions(+), 1124 deletions(-) delete mode 100644 crates/project-graph/src/project_graph_builder.rs delete mode 100644 crates/project-graph/src/project_graph_cache.rs create mode 100644 crates/test-utils/src/workspace_mocker.rs create mode 100644 crates/workspace/Cargo.toml create mode 100644 crates/workspace/src/lib.rs rename crates/{project-graph/src/project_events.rs => workspace/src/project_build_data.rs} (59%) rename crates/{project-graph => workspace}/src/projects_locator.rs (84%) create mode 100644 crates/workspace/src/repo_type.rs create mode 100644 crates/workspace/src/workspace_builder.rs create mode 100644 crates/workspace/src/workspace_builder_error.rs rename crates/{project-graph/src/project_graph_hash.rs => workspace/src/workspace_cache.rs} (66%) diff --git a/Cargo.lock b/Cargo.lock index cce432e6623..ef50600c05d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3006,6 +3006,7 @@ dependencies = [ "moon_toolchain_plugin", "moon_typescript_lang", "moon_vcs", + "moon_workspace", "once_cell", "open", "petgraph", @@ -3713,15 +3714,12 @@ dependencies = [ "moon_cache", "moon_common", "moon_config", - "moon_hash", "moon_project", - "moon_project_builder", - "moon_project_constraints", "moon_project_expander", "moon_query", "moon_task", "moon_test_utils2", - "moon_vcs", + "moon_workspace", "petgraph", "rustc-hash 2.0.0", "scc", @@ -3988,6 +3986,7 @@ dependencies = [ "moon_rust_platform", "moon_system_platform", "moon_vcs", + "moon_workspace", "proto_core", "starbase_events", "starbase_sandbox", @@ -4141,6 +4140,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "moon_workspace" +version = "0.0.1" +dependencies = [ + "miette", + "moon_cache", + "moon_common", + "moon_config", + "moon_hash", + "moon_project", + "moon_project_builder", + "moon_project_constraints", + "moon_project_graph", + "moon_vcs", + "petgraph", + "rustc-hash 2.0.0", + "serde", + "starbase_events", + "starbase_utils", + "thiserror", + "tracing", +] + [[package]] name = "native-tls" version = "0.2.12" diff --git a/crates/action-graph/tests/action_graph_test.rs b/crates/action-graph/tests/action_graph_test.rs index d7758dadbb0..4e5b60f5e4e 100644 --- a/crates/action-graph/tests/action_graph_test.rs +++ b/crates/action-graph/tests/action_graph_test.rs @@ -1472,7 +1472,9 @@ mod action_graph { } #[tokio::test] - #[should_panic(expected = "No project has been configured with the name or alias unknown.")] + #[should_panic( + expected = "No project has been configured with the identifier or alias unknown." + )] async fn errors_for_unknown_project() { let sandbox = create_sandbox("tasks"); let container = ActionGraphContainer::new(sandbox.path()).await; diff --git a/crates/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap b/crates/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap index 2e6f50c8634..36cd2a7a823 100644 --- a/crates/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap +++ b/crates/action-graph/tests/snapshots/action_graph_test__action_graph__run_task_by_target__runs_all.snap @@ -5,26 +5,26 @@ expression: graph.to_dot() digraph { 0 [ label="SyncWorkspace" ] 1 [ label="SetupToolchain(system)" ] - 2 [ label="SyncProject(system, server)" ] - 3 [ label="RunTask(server:build)" ] + 2 [ label="SyncProject(system, base)" ] + 3 [ label="RunTask(base:build)" ] 4 [ label="SyncProject(system, client)" ] - 5 [ label="SyncProject(system, common)" ] - 6 [ label="SyncProject(system, base)" ] + 5 [ label="SyncProject(system, server)" ] + 6 [ label="SyncProject(system, common)" ] 7 [ label="RunTask(client:build)" ] 8 [ label="RunTask(common:build)" ] - 9 [ label="RunTask(base:build)" ] + 9 [ label="RunTask(server:build)" ] 1 -> 0 [ ] 2 -> 1 [ ] 3 -> 2 [ ] - 6 -> 1 [ ] 5 -> 1 [ ] - 5 -> 6 [ ] + 6 -> 1 [ ] + 6 -> 2 [ ] 4 -> 1 [ ] - 4 -> 2 [ ] 4 -> 5 [ ] - 8 -> 5 [ ] + 4 -> 6 [ ] + 8 -> 6 [ ] + 9 -> 5 [ ] 7 -> 4 [ ] 7 -> 8 [ ] - 7 -> 3 [ ] - 9 -> 6 [ ] + 7 -> 9 [ ] } diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index 9f72f4bc000..e4095646c8e 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -30,6 +30,7 @@ moon_task = { path = "../task" } moon_toolchain = { path = "../toolchain" } moon_toolchain_plugin = { path = "../toolchain-plugin" } moon_vcs = { path = "../vcs" } +moon_workspace = { path = "../workspace" } async-recursion = { workspace = true } async-trait = { workspace = true } bytes = "1.7.2" diff --git a/crates/app/src/commands/docker/scaffold.rs b/crates/app/src/commands/docker/scaffold.rs index 470ae66d526..f174319704a 100644 --- a/crates/app/src/commands/docker/scaffold.rs +++ b/crates/app/src/commands/docker/scaffold.rs @@ -300,7 +300,7 @@ async fn scaffold_sources_project( } debug!( - id = project_id.as_str(), + project_id = project_id.as_str(), globs = ?include_globs, "Copying sources from project {}", color::id(project_id), @@ -318,7 +318,7 @@ async fn scaffold_sources_project( // they can be explicit in config or on the command line! if !dep_cfg.is_root_scope() { debug!( - id = project_id.as_str(), + project_id = project_id.as_str(), dep_id = dep_cfg.id.as_str(), "Including dependency project" ); diff --git a/crates/app/src/commands/node/run_script.rs b/crates/app/src/commands/node/run_script.rs index 570bc2f0404..191c994cd0f 100644 --- a/crates/app/src/commands/node/run_script.rs +++ b/crates/app/src/commands/node/run_script.rs @@ -44,10 +44,9 @@ pub async fn run_script(session: CliSession, args: RunScriptArgs) -> AppResult { // Otherwise try and find the project in the graph } else if let Some(project_id) = &args.project { - let mut project_graph = session.build_project_graph().await?; - project_graph.load(project_id).await?; + let project_graph = session.get_project_graph().await?; - command.cwd(&project_graph.build().await?.get(project_id)?.root); + command.cwd(&project_graph.get(project_id)?.root); // This should rarely happen... } else { diff --git a/crates/app/src/commands/project.rs b/crates/app/src/commands/project.rs index 049e07222f3..6f49b42de54 100644 --- a/crates/app/src/commands/project.rs +++ b/crates/app/src/commands/project.rs @@ -17,10 +17,10 @@ pub struct ProjectArgs { #[instrument(skip_all)] pub async fn project(session: CliSession, args: ProjectArgs) -> AppResult { - let mut project_graph_builder = session.build_project_graph().await?; - project_graph_builder.load(&args.id).await?; - - let project_graph = project_graph_builder.build().await?; + let project_graph = session + .get_project_graph() + .await? + .into_focused(&args.id, false)?; let project = project_graph.get(&args.id)?; let config = &project.config; diff --git a/crates/app/src/commands/task.rs b/crates/app/src/commands/task.rs index 1fbc87cdcb0..bb0be3f8305 100644 --- a/crates/app/src/commands/task.rs +++ b/crates/app/src/commands/task.rs @@ -22,10 +22,7 @@ pub async fn task(session: CliSession, args: TaskArgs) -> AppResult { return Err(AppError::ProjectIdRequired.into()); }; - let mut project_graph_builder = session.build_project_graph().await?; - project_graph_builder.load(project_locator).await?; - - let project_graph = project_graph_builder.build().await?; + let project_graph = session.get_project_graph().await?; let project = project_graph.get(project_locator)?; let task = project.get_task(&args.target.task_id)?; diff --git a/crates/app/src/components.rs b/crates/app/src/components.rs index cfba514a92b..dc1bfb94800 100644 --- a/crates/app/src/components.rs +++ b/crates/app/src/components.rs @@ -5,9 +5,9 @@ use moon_action_context::ActionContext; use moon_action_graph::ActionGraph; use moon_action_pipeline::ActionPipeline; use moon_platform::PlatformManager; -use moon_project_graph::{ +use moon_workspace::{ ExtendProjectData, ExtendProjectEvent, ExtendProjectGraphData, ExtendProjectGraphEvent, - ProjectGraphBuilderContext, + WorkspaceBuilderContext, }; use starbase_events::{Emitter, EventState}; use std::sync::Arc; @@ -53,10 +53,10 @@ pub async fn run_action_pipeline( Ok(results) } -pub async fn create_project_graph_context( +pub async fn create_workspace_graph_context( session: &CliSession, -) -> miette::Result { - let context = ProjectGraphBuilderContext { +) -> miette::Result { + let context = WorkspaceBuilderContext { config_loader: &session.config_loader, extend_project: Emitter::::new(), extend_project_graph: Emitter::::new(), diff --git a/crates/app/src/session.rs b/crates/app/src/session.rs index bdb9d178841..7e91869ab24 100644 --- a/crates/app/src/session.rs +++ b/crates/app/src/session.rs @@ -14,9 +14,10 @@ use moon_console_reporter::DefaultReporter; use moon_env::MoonEnvironment; use moon_extension_plugin::*; use moon_plugin::{PluginHostData, PluginId}; -use moon_project_graph::{ProjectGraph, ProjectGraphBuilder}; +use moon_project_graph::ProjectGraph; use moon_toolchain_plugin::*; use moon_vcs::{BoxedVcs, Git}; +use moon_workspace::WorkspaceBuilder; use once_cell::sync::OnceCell; use proto_core::ProtoEnvironment; use semver::Version; @@ -89,10 +90,6 @@ impl CliSession { ActionGraphBuilder::new(project_graph) } - pub async fn build_project_graph(&self) -> AppResult { - ProjectGraphBuilder::new(create_project_graph_context(self).await?).await - } - pub fn get_app_context(&self) -> AppResult> { Ok(Arc::new(AppContext { cli_version: self.cli_version.clone(), @@ -140,18 +137,11 @@ impl CliSession { } pub async fn get_project_graph(&self) -> AppResult> { - if let Some(item) = self.project_graph.get() { - return Ok(Arc::clone(item)); + if self.project_graph.get().is_none() { + self.load_workspace_graph().await?; } - let cache_engine = self.get_cache_engine()?; - let context = create_project_graph_context(self).await?; - let builder = ProjectGraphBuilder::generate(context, &cache_engine).await?; - let graph = Arc::new(builder.build().await?); - - let _ = self.project_graph.set(Arc::clone(&graph)); - - Ok(graph) + Ok(self.project_graph.get().map(Arc::clone).unwrap()) } pub async fn get_toolchain_registry(&self) -> AppResult> { @@ -207,6 +197,17 @@ impl CliSession { Commands::Bin(_) | Commands::Docker { .. } | Commands::Node { .. } | Commands::Teardown ) } + + async fn load_workspace_graph(&self) -> AppResult<()> { + let cache_engine = self.get_cache_engine()?; + let context = create_workspace_graph_context(self).await?; + let builder = WorkspaceBuilder::new_with_cache(context, &cache_engine).await?; + let result = builder.build().await?; + + let _ = self.project_graph.set(Arc::new(result.project_graph)); + + Ok(()) + } } #[async_trait] diff --git a/crates/cache-item/src/lib.rs b/crates/cache-item/src/lib.rs index 1cb90bf7473..9a261abf4ab 100644 --- a/crates/cache-item/src/lib.rs +++ b/crates/cache-item/src/lib.rs @@ -7,7 +7,7 @@ pub use cache_mode::*; #[macro_export] macro_rules! cache_item { ($item:item) => { - #[derive(Debug, Default, Eq, PartialEq, serde::Deserialize, serde::Serialize)] + #[derive(Debug, Default, /* Eq, */ PartialEq, serde::Deserialize, serde::Serialize)] #[serde(default, rename_all = "camelCase")] $item }; diff --git a/crates/cli/tests/action_graph_test.rs b/crates/cli/tests/action_graph_test.rs index fb3a1f22cd2..ddaaa530445 100644 --- a/crates/cli/tests/action_graph_test.rs +++ b/crates/cli/tests/action_graph_test.rs @@ -24,7 +24,7 @@ mod action_graph { let dot = assert.output(); // Snapshot is not deterministic - assert_eq!(dot.split('\n').count(), 450); + assert_eq!(dot.split('\n').count(), 448); } #[test] diff --git a/crates/cli/tests/docker_file_test.rs b/crates/cli/tests/docker_file_test.rs index ac905739e91..50f58b5c6ad 100644 --- a/crates/cli/tests/docker_file_test.rs +++ b/crates/cli/tests/docker_file_test.rs @@ -14,7 +14,7 @@ mod dockerfile { }); assert.inner.stderr(predicate::str::contains( - "No project has been configured with the name or alias missing.", + "No project has been configured with the identifier or alias missing.", )); } diff --git a/crates/cli/tests/snapshots/project_test__unknown_project.snap b/crates/cli/tests/snapshots/project_test__unknown_project.snap index f75447f9249..5c5d05668da 100644 --- a/crates/cli/tests/snapshots/project_test__unknown_project.snap +++ b/crates/cli/tests/snapshots/project_test__unknown_project.snap @@ -2,9 +2,9 @@ source: crates/cli/tests/project_test.rs expression: get_assert_stderr_output(&assert.inner) --- -Error: project_graph::unknown_project +Error: project_graph::unknown_id - × No project has been configured with the name or alias unknown. + × No project has been configured with the identifier or alias unknown. diff --git a/crates/cli/tests/snapshots/run_test__errors_for_unknown_project.snap b/crates/cli/tests/snapshots/run_test__errors_for_unknown_project.snap index d206f9886f4..2a65ad938f1 100644 --- a/crates/cli/tests/snapshots/run_test__errors_for_unknown_project.snap +++ b/crates/cli/tests/snapshots/run_test__errors_for_unknown_project.snap @@ -2,9 +2,9 @@ source: crates/cli/tests/run_test.rs expression: assert.output() --- -Error: project_graph::unknown_project +Error: project_graph::unknown_id - × No project has been configured with the name or alias unknown. + × No project has been configured with the identifier or alias unknown. diff --git a/crates/project-builder/src/project_builder.rs b/crates/project-builder/src/project_builder.rs index e94d5b6b59f..1e701df095b 100644 --- a/crates/project-builder/src/project_builder.rs +++ b/crates/project-builder/src/project_builder.rs @@ -48,7 +48,7 @@ impl<'app> ProjectBuilder<'app> { context: ProjectBuilderContext<'app>, ) -> miette::Result { trace!( - id = id.as_str(), + project_id = id.as_str(), source = source.as_str(), "Building project {} from source", color::id(id) @@ -87,7 +87,7 @@ impl<'app> ProjectBuilder<'app> { )?; trace!( - id = self.id.as_str(), + project_id = self.id.as_str(), lookup = ?global_config.order, "Inheriting global file groups and tasks", ); @@ -99,7 +99,7 @@ impl<'app> ProjectBuilder<'app> { /// Inherit the local config and then detect applicable language and platform fields. #[instrument(skip_all)] - pub async fn inherit_local_config(&mut self, config: ProjectConfig) -> miette::Result<()> { + pub async fn inherit_local_config(&mut self, config: &ProjectConfig) -> miette::Result<()> { // Use configured language or detect from environment self.language = if config.language == LanguageType::Unknown { let mut language = detect_project_language(&self.project_root); @@ -109,7 +109,7 @@ impl<'app> ProjectBuilder<'app> { } trace!( - id = self.id.as_str(), + project_id = self.id.as_str(), language = ?language, "Unknown project language, detecting from environment", ); @@ -128,7 +128,7 @@ impl<'app> ProjectBuilder<'app> { ); trace!( - id = self.id.as_str(), + project_id = self.id.as_str(), language = ?self.language, platform = ?self.platform, "Unknown tasks platform, inferring from language and toolchain", @@ -152,7 +152,7 @@ impl<'app> ProjectBuilder<'app> { } } - self.local_config = Some(config); + self.local_config = Some(config.to_owned()); Ok(()) } @@ -161,7 +161,7 @@ impl<'app> ProjectBuilder<'app> { #[instrument(skip_all)] pub async fn load_local_config(&mut self) -> miette::Result<()> { debug!( - id = self.id.as_str(), + project_id = self.id.as_str(), "Attempting to load {} (optional)", color::file( self.source @@ -174,7 +174,7 @@ impl<'app> ProjectBuilder<'app> { .config_loader .load_project_config(&self.project_root)?; - self.inherit_local_config(config).await?; + self.inherit_local_config(&config).await?; Ok(()) } @@ -278,7 +278,7 @@ impl<'app> ProjectBuilder<'app> { } trace!( - id = self.id.as_str(), + project_id = self.id.as_str(), dep = dep_id.as_str(), task = task_config.target.as_str(), "Marking arbitrary project as an implicit dependency because of a task dependency" @@ -303,7 +303,7 @@ impl<'app> ProjectBuilder<'app> { if !deps.is_empty() { trace!( - id = self.id.as_str(), + project_id = self.id.as_str(), deps = ?deps.keys().map(|k| k.as_str()).collect::>(), "Depends on {} projects", deps.len(), @@ -323,7 +323,7 @@ impl<'app> ProjectBuilder<'app> { // Inherit global first if let Some(global) = &self.global_config { trace!( - id = self.id.as_str(), + project_id = self.id.as_str(), groups = ?global.config.file_groups.keys().map(|k| k.as_str()).collect::>(), "Inheriting global file groups", ); @@ -336,7 +336,7 @@ impl<'app> ProjectBuilder<'app> { // Override with local second if let Some(local) = &self.local_config { trace!( - id = self.id.as_str(), + project_id = self.id.as_str(), groups = ?local.file_groups.keys().map(|k| k.as_str()).collect::>(), "Using local file groups", ); diff --git a/crates/project-expander/src/project_expander.rs b/crates/project-expander/src/project_expander.rs index 9edc3d2c245..4e7e17ce10d 100644 --- a/crates/project-expander/src/project_expander.rs +++ b/crates/project-expander/src/project_expander.rs @@ -23,7 +23,7 @@ impl<'graph, 'query> ProjectExpander<'graph, 'query> { let mut project = self.context.project.to_owned(); debug!( - id = project.id.as_str(), + project_id = project.id.as_str(), "Expanding project {}", color::id(&project.id) ); diff --git a/crates/project-graph/Cargo.toml b/crates/project-graph/Cargo.toml index 8d94fd295b8..4bad9b7b3ed 100644 --- a/crates/project-graph/Cargo.toml +++ b/crates/project-graph/Cargo.toml @@ -9,29 +9,26 @@ repository = "https://github.com/moonrepo/moon" publish = false [dependencies] -moon_cache = { path = "../cache" } moon_common = { path = "../common" } moon_config = { path = "../config" } -moon_hash = { path = "../hash" } moon_project = { path = "../project" } -moon_project_builder = { path = "../project-builder" } -moon_project_constraints = { path = "../project-constraints" } moon_project_expander = { path = "../project-expander" } moon_query = { path = "../query" } moon_task = { path = "../task" } -moon_vcs = { path = "../vcs" } miette = { workspace = true } petgraph = { workspace = true } rustc-hash = { workspace = true } scc = { workspace = true } serde = { workspace = true, features = ["rc"] } -starbase_events = { workspace = true } -starbase_utils = { workspace = true, features = ["glob", "json"] } +starbase_utils = { workspace = true, features = ["json"] } thiserror = { workspace = true } tracing = { workspace = true } [dev-dependencies] +moon_cache = { path = "../cache" } moon_test_utils2 = { path = "../test-utils" } +moon_workspace = { path = "../workspace" } +starbase_events = { workspace = true } starbase_sandbox = { workspace = true } tokio = { workspace = true } diff --git a/crates/project-graph/src/lib.rs b/crates/project-graph/src/lib.rs index a7d6047246c..62ca884fba2 100644 --- a/crates/project-graph/src/lib.rs +++ b/crates/project-graph/src/lib.rs @@ -1,16 +1,7 @@ -mod project_events; mod project_graph; -mod project_graph_builder; -mod project_graph_cache; mod project_graph_error; -mod project_graph_hash; mod project_matcher; -mod projects_locator; -pub use project_events::*; pub use project_graph::*; -pub use project_graph_builder::*; -pub use project_graph_cache::*; pub use project_graph_error::*; pub use project_matcher::*; -pub use projects_locator::*; diff --git a/crates/project-graph/src/project_graph.rs b/crates/project-graph/src/project_graph.rs index bd150fc8bf6..a00515106cd 100644 --- a/crates/project-graph/src/project_graph.rs +++ b/crates/project-graph/src/project_graph.rs @@ -19,12 +19,12 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; use tracing::{debug, instrument}; -pub type GraphType = DiGraph; +pub type ProjectGraphType = DiGraph; pub type ProjectsCache = FxHashMap>; #[derive(Serialize)] pub struct ProjectGraphCache<'graph> { - graph: &'graph GraphType, + graph: &'graph ProjectGraphType, projects: &'graph ProjectsCache, } @@ -51,7 +51,7 @@ pub struct ProjectGraph { fs_cache: HashMap>, /// Directed-acyclic graph (DAG) of non-expanded projects and their dependencies. - graph: GraphType, + graph: ProjectGraphType, /// Graph node information, mapped by project ID. nodes: FxHashMap, @@ -70,7 +70,11 @@ pub struct ProjectGraph { } impl ProjectGraph { - pub fn new(graph: GraphType, nodes: FxHashMap, workspace_root: &Path) -> Self { + pub fn new( + graph: ProjectGraphType, + nodes: FxHashMap, + workspace_root: &Path, + ) -> Self { debug!("Creating project graph"); Self { @@ -123,13 +127,13 @@ impl ProjectGraph { /// Return a project with the provided name or alias from the graph. /// If the project does not exist or has been misconfigured, return an error. #[instrument(name = "get_project", skip(self))] - pub fn get(&self, project_locator: &str) -> miette::Result> { - self.internal_get(project_locator) + pub fn get(&self, id_or_alias: &str) -> miette::Result> { + self.internal_get(id_or_alias) } /// Return an unexpanded project with the provided name or alias from the graph. - pub fn get_unexpanded(&self, project_locator: &str) -> miette::Result<&Project> { - let id = self.resolve_id(project_locator); + pub fn get_unexpanded(&self, id_or_alias: &str) -> miette::Result<&Project> { + let id = self.resolve_id(id_or_alias); let node = self .nodes @@ -216,12 +220,8 @@ impl ProjectGraph { .collect() } - pub fn into_focused( - &self, - project_locator: &Id, - with_dependents: bool, - ) -> miette::Result { - let project = self.get(project_locator)?; + pub fn into_focused(&self, id_or_alias: &Id, with_dependents: bool) -> miette::Result { + let project = self.get(id_or_alias)?; let upstream = self.dependencies_of(&project)?; let downstream = self.dependents_of(&project)?; let mut nodes = FxHashMap::default(); @@ -438,23 +438,23 @@ impl ProjectGraph { Ok(id) } - fn resolve_id(&self, alias_or_id: &str) -> Id { - Id::raw(if self.nodes.contains_key(alias_or_id) { - alias_or_id + fn resolve_id(&self, id_or_alias: &str) -> Id { + Id::raw(if self.nodes.contains_key(id_or_alias) { + id_or_alias } else { self.nodes .iter() .find(|(_, node)| { node.alias .as_ref() - .is_some_and(|alias| alias == alias_or_id) + .is_some_and(|alias| alias == id_or_alias) || node .original_id .as_ref() - .is_some_and(|id| id == alias_or_id) + .is_some_and(|id| id == id_or_alias) }) .map(|(id, _)| id.as_str()) - .unwrap_or(alias_or_id) + .unwrap_or(id_or_alias) }) } diff --git a/crates/project-graph/src/project_graph_builder.rs b/crates/project-graph/src/project_graph_builder.rs deleted file mode 100644 index 01c2eb92573..00000000000 --- a/crates/project-graph/src/project_graph_builder.rs +++ /dev/null @@ -1,665 +0,0 @@ -use crate::project_events::{ExtendProjectEvent, ExtendProjectGraphEvent}; -use crate::project_graph::{GraphType, ProjectGraph, ProjectNode}; -use crate::project_graph_cache::ProjectsCacheState; -use crate::project_graph_error::ProjectGraphError; -use crate::project_graph_hash::ProjectGraphHash; -use crate::projects_locator::locate_projects_with_globs; -use moon_cache::CacheEngine; -use moon_common::path::{is_root_level_source, to_virtual_string, WorkspaceRelativePathBuf}; -use moon_common::{color, consts, Id}; -use moon_config::{ - ConfigLoader, DependencyScope, InheritedTasksManager, ProjectConfig, ProjectsSourcesList, - ToolchainConfig, WorkspaceConfig, WorkspaceProjects, -}; -use moon_project::Project; -use moon_project_builder::{ProjectBuilder, ProjectBuilderContext}; -use moon_project_constraints::{enforce_project_type_relationships, enforce_tag_relationships}; -use moon_vcs::BoxedVcs; -use petgraph::graph::DiGraph; -use petgraph::prelude::NodeIndex; -use petgraph::visit::IntoNodeReferences; -use petgraph::Direction; -use rustc_hash::{FxHashMap, FxHashSet}; -use serde::{Deserialize, Serialize}; -use starbase_events::Emitter; -use starbase_utils::{glob, json}; -use std::collections::BTreeMap; -use std::path::Path; -use std::sync::Arc; -use tracing::{debug, instrument, trace}; - -pub struct ProjectGraphBuilderContext<'app> { - pub config_loader: &'app ConfigLoader, - pub extend_project: Emitter, - pub extend_project_graph: Emitter, - pub inherited_tasks: &'app InheritedTasksManager, - pub toolchain_config: &'app ToolchainConfig, - pub vcs: Option>, - pub working_dir: &'app Path, - pub workspace_config: &'app WorkspaceConfig, - pub workspace_root: &'app Path, -} - -#[derive(Deserialize, Serialize)] -pub struct ProjectGraphBuilder<'app> { - #[serde(skip)] - context: Option>>, - - /// Mapping of project IDs to project aliases. - aliases: FxHashMap, - - /// Loaded project configuration files. - #[serde(skip)] - configs: FxHashMap, - - /// The DAG instance. - graph: GraphType, - - /// Is this a monorepo or polyrepo. - monorepo: bool, - - /// Nodes (projects) inserted into the graph. - nodes: FxHashMap, - - /// Projects that have explicitly renamed themselves. - /// Maps original ID to renamed ID. - renamed_ids: FxHashMap, - - /// The root project ID (only if a monorepo). - root_id: Option, - - /// Mapping of project IDs to file system sources, - /// derived from the `workspace.projects` setting. - sources: FxHashMap, -} - -impl<'app> ProjectGraphBuilder<'app> { - /// Create a new project graph instance without reading from the - /// cache, and preloading all project sources and aliases. - pub async fn new( - context: ProjectGraphBuilderContext<'app>, - ) -> miette::Result> { - debug!("Building project graph"); - - let mut graph = ProjectGraphBuilder { - context: Some(Arc::new(context)), - configs: FxHashMap::default(), - aliases: FxHashMap::default(), - graph: DiGraph::new(), - monorepo: false, - nodes: FxHashMap::default(), - renamed_ids: FxHashMap::default(), - root_id: None, - sources: FxHashMap::default(), - }; - - graph.preload().await?; - - Ok(graph) - } - - /// Create a project graph with all projects inserted as nodes, - /// and read from the file system cache when applicable. - #[instrument(name = "generate_project_graph", skip_all)] - pub async fn generate( - context: ProjectGraphBuilderContext<'app>, - cache_engine: &CacheEngine, - ) -> miette::Result> { - let is_vcs_enabled = context - .vcs - .as_ref() - .expect("VCS is required for project graph caching!") - .is_enabled(); - let mut graph = Self::new(context).await?; - - // No VCS to hash with, so abort caching - if !is_vcs_enabled { - graph.load_all().await?; - - return Ok(graph); - } - - // Hash the project graph based on the preloaded state - let mut graph_contents = ProjectGraphHash::new(); - graph_contents.add_sources(&graph.sources); - graph_contents.add_aliases(&graph.aliases); - graph_contents.add_configs(graph.hash_required_configs().await?); - graph_contents.gather_env(); - - let hash = cache_engine - .hash - .save_manifest_without_hasher("Project graph", &graph_contents)?; - - debug!(hash, "Generated hash for project graph"); - - // Check the current state and cache - let mut state = cache_engine - .state - .load_state::("projects.json")?; - let cache_path = cache_engine.state.resolve_path("partialProjectGraph.json"); - - if hash == state.data.last_hash && cache_path.exists() { - debug!( - cache = ?cache_path, - "Loading project graph with {} projects from cache", - graph.sources.len(), - ); - - let mut cache: ProjectGraphBuilder = json::read_file(cache_path)?; - cache.configs = graph.configs; - cache.context = graph.context; - - return Ok(cache); - } - - // Build the graph, update the state, and save the cache - debug!( - "Generating project graph with {} projects", - graph.sources.len(), - ); - - graph.load_all().await?; - - state.data.last_hash = hash; - state.data.projects = graph.sources.clone(); - state.save()?; - - json::write_file(cache_path, &graph, false)?; - - Ok(graph) - } - - /// Build the project graph and return a new structure. - #[instrument(name = "build_project_graph", skip_all)] - pub async fn build(mut self) -> miette::Result { - self.enforce_constraints()?; - - let context = self.context.take().unwrap(); - let mut nodes = FxHashMap::default(); - - for (id, index) in self.nodes { - let Some(source) = self.sources.remove(&id) else { - continue; - }; - - nodes.insert( - id, - ProjectNode { - index, - source, - ..ProjectNode::default() - }, - ); - } - - for (id, alias) in self.aliases { - nodes.entry(id).and_modify(|node| { - node.alias = Some(alias); - }); - } - - for (original_id, id) in self.renamed_ids { - nodes.entry(id).and_modify(|node| { - node.original_id = Some(original_id); - }); - } - - let mut graph = ProjectGraph::new(self.graph, nodes, context.workspace_root); - graph.working_dir = context.working_dir.to_owned(); - - Ok(graph) - } - - /// Load a single project by name or alias into the graph. - pub async fn load(&mut self, project_locator: &str) -> miette::Result<()> { - self.internal_load(project_locator, &mut FxHashSet::default()) - .await?; - - Ok(()) - } - - /// Load all projects into the graph, as configured in the workspace. - pub async fn load_all(&mut self) -> miette::Result<()> { - let ids = self.sources.keys().cloned().collect::>(); - - for id in ids { - self.internal_load(&id, &mut FxHashSet::default()).await?; - } - - Ok(()) - } - - #[instrument(name = "load", skip(self, cycle))] - async fn internal_load( - &mut self, - project_locator: &str, - cycle: &mut FxHashSet, - ) -> miette::Result<(Id, NodeIndex)> { - let id = self.resolve_id(project_locator); - - // Already loaded, exit early with existing index - if let Some(index) = self.nodes.get(&id) { - trace!( - id = id.as_str(), - "Project already exists in the project graph, skipping load", - ); - - return Ok((id, *index)); - } - - // Check that the project ID is configured - trace!( - id = id.as_str(), - "Project does not exist in the project graph, attempting to load", - ); - - let Some(source) = self.sources.get(&id).map(|s| s.to_owned()) else { - return Err(ProjectGraphError::UnconfiguredID(id).into()); - }; - - // Create the project - let mut project = self.build_project(id, source).await?; - let id = project.id.clone(); - - cycle.insert(id.clone()); - - // Create dependent projects - let mut edges = vec![]; - - for dep_config in &mut project.dependencies { - let loaded_dep_id = if cycle.contains(&dep_config.id) { - debug!( - id = id.as_str(), - dependency_id = dep_config.id.as_str(), - "Encountered a dependency cycle (from project); will disconnect nodes to avoid recursion", - ); - - continue; - - // Don't link the root project to any project, but still load it - } else if dep_config.is_root_scope() { - Box::pin(self.internal_load(&dep_config.id, cycle)).await?.0 - - // Otherwise link projects - } else { - let dep = Box::pin(self.internal_load(&dep_config.id, cycle)).await?; - edges.push((dep.1, dep_config.scope)); - dep.0 - }; - - if loaded_dep_id != dep_config.id { - dep_config.id = loaded_dep_id; - } - } - - // Add to the graph - let index = self.graph.add_node(project); - - self.nodes.insert(id.clone(), index); - - for edge in edges { - self.graph.add_edge(index, edge.0, edge.1); - } - - cycle.clear(); - - Ok((id, index)) - } - - /// Create and build the project with the provided ID and source. - #[instrument(skip(self))] - async fn build_project( - &mut self, - id: Id, - source: WorkspaceRelativePathBuf, - ) -> miette::Result { - debug!(id = id.as_str(), "Building project {}", color::id(&id)); - - let context = self.context(); - - if !source.to_path(context.workspace_root).exists() { - return Err(ProjectGraphError::MissingAtSource(source.to_string()).into()); - } - - let mut builder = ProjectBuilder::new( - &id, - &source, - ProjectBuilderContext { - config_loader: context.config_loader, - monorepo: self.monorepo, - root_project_id: self.root_id.as_ref(), - toolchain_config: context.toolchain_config, - workspace_root: context.workspace_root, - }, - )?; - - if let Some(config) = self.configs.remove(&id) { - builder.inherit_local_config(config).await?; - } else { - builder.load_local_config().await?; - } - - builder.inherit_global_config(context.inherited_tasks)?; - - let extended_data = context - .extend_project - .emit(ExtendProjectEvent { - project_id: id.to_owned(), - project_source: source.to_owned(), - workspace_root: context.workspace_root.to_owned(), - }) - .await?; - - // Inherit implicit dependencies - for dep_config in extended_data.dependencies { - builder.extend_with_dependency(dep_config); - } - - // Inherit inferred tasks - for (task_id, task_config) in extended_data.tasks { - builder.extend_with_task(task_id, task_config); - } - - // Inherit alias before building in case the project - // references itself in tasks or dependencies - if let Some(alias) = self.aliases.get(&id) { - builder.set_alias(alias); - } - - let project = builder.build().await?; - - Ok(project) - } - - /// Enforce project constraints and boundaries after all nodes have been inserted. - #[instrument(skip_all)] - fn enforce_constraints(&self) -> miette::Result<()> { - debug!("Enforcing project constraints"); - - let context = self.context(); - let type_relationships = context - .workspace_config - .constraints - .enforce_project_type_relationships; - let tag_relationships = &context.workspace_config.constraints.tag_relationships; - - if !type_relationships && tag_relationships.is_empty() { - return Ok(()); - } - - let default_scope = DependencyScope::Build; - - for (project_index, project) in self.graph.node_references() { - let deps: Vec<_> = self - .graph - .neighbors_directed(project_index, Direction::Outgoing) - .flat_map(|dep_index| { - self.graph.node_weight(dep_index).map(|dep| { - ( - dep, - // Is this safe? - self.graph - .find_edge(project_index, dep_index) - .and_then(|ei| self.graph.edge_weight(ei)) - .unwrap_or(&default_scope), - ) - }) - }) - .collect(); - - for (dep, dep_scope) in deps { - if type_relationships { - enforce_project_type_relationships(project, dep, dep_scope)?; - } - - for (source_tag, required_tags) in tag_relationships { - enforce_tag_relationships(project, source_tag, dep, required_tags)?; - } - } - } - - Ok(()) - } - - /// When caching the project graph, we must hash all project and workspace - /// config files that are required to invalidate the cache. - async fn hash_required_configs( - &self, - ) -> miette::Result> { - let context = self.context(); - let config_names = context.config_loader.get_project_file_names(); - let mut configs = vec![]; - - // Hash all project-level config files - for source in self.sources.values() { - for name in &config_names { - configs.push(source.join(name).as_str().to_owned()); - } - } - - // Hash all workspace-level config files - for file in glob::walk( - context.workspace_root.join(consts::CONFIG_DIRNAME), - ["*.pkl", "tasks/**/*.pkl", "*.yml", "tasks/**/*.yml"], - )? { - configs.push(to_virtual_string( - file.strip_prefix(context.workspace_root).unwrap(), - )?); - } - - context - .vcs - .as_ref() - .unwrap() - .get_file_hashes(&configs, true, 500) - .await - } - - /// Preload the graph with project sources from the workspace configuration. - /// If globs are provided, walk the file system and gather sources. - /// Then extend the graph with aliases, derived from all event subscribers. - async fn preload(&mut self) -> miette::Result<()> { - let context = self.context(); - let mut globs = vec![]; - let mut sources = vec![]; - - // Locate all project sources - let mut add_sources = |map: &FxHashMap| { - for (id, source) in map { - sources.push((id.to_owned(), WorkspaceRelativePathBuf::from(source))); - } - }; - - match &context.workspace_config.projects { - WorkspaceProjects::Sources(map) => { - add_sources(map); - } - WorkspaceProjects::Globs(list) => { - globs.extend(list); - } - WorkspaceProjects::Both(cfg) => { - globs.extend(&cfg.globs); - add_sources(&cfg.sources); - } - }; - - if !sources.is_empty() { - debug!( - sources = ?sources, - "Using configured project sources", - ); - } - - if !globs.is_empty() { - debug!( - globs = ?globs, - "Locating projects with globs", - ); - - locate_projects_with_globs(&context, &globs, &mut sources)?; - } - - // Load all config files first so that ID renaming occurs - self.preload_configs(&mut sources)?; - - // Extend graph from subscribers - debug!("Extending project graph from subscribers"); - - let aliases = context - .extend_project_graph - .emit(ExtendProjectGraphEvent { - sources: sources.clone(), - workspace_root: context.workspace_root.to_owned(), - }) - .await? - .aliases; - - // Determine if a polyrepo or monorepo - let polyrepo = sources.len() == 1 - && sources - .first() - .map(|(_, source)| is_root_level_source(source)) - .unwrap_or_default(); - - self.monorepo = !polyrepo; - - // Find the root project - self.root_id = if self.monorepo { - sources.iter().find_map(|(id, source)| { - if is_root_level_source(source) { - Some(id.to_owned()) - } else { - None - } - }) - } else { - None - }; - - // Set our data and warn/error against problems - for (id, source) in sources { - if let Some(existing_source) = self.sources.get(&id) { - if existing_source == &source { - continue; - } - - return Err(ProjectGraphError::DuplicateId { - id: id.clone(), - old_source: existing_source.to_string(), - new_source: source.to_string(), - } - .into()); - } else { - self.sources.insert(id, source); - } - } - - let mut dupe_aliases = FxHashMap::::default(); - - for (id, alias) in aliases { - let id = match self.renamed_ids.get(&id) { - Some(new_id) => new_id, - None => &id, - }; - - // Skip aliases that match its own ID - if id == &alias { - continue; - } - - // Skip aliases that would override an ID - if self.sources.contains_key(alias.as_str()) { - debug!( - "Skipping alias {} (for project {}) as it conflicts with the project {}", - color::label(&alias), - color::id(id), - color::id(&alias), - ); - - continue; - } - - if let Some(existing_id) = dupe_aliases.get(&alias) { - // Skip if the existing ID is already for this ID. - // This scenario is possible when multiple platforms - // extract the same aliases (Bun vs Node, etc). - if existing_id == id { - continue; - } - - return Err(ProjectGraphError::DuplicateAlias { - alias: alias.clone(), - old_id: existing_id.to_owned(), - new_id: id.clone(), - } - .into()); - } - - dupe_aliases.insert(alias.clone(), id.to_owned()); - self.aliases.insert(id.to_owned(), alias); - } - - Ok(()) - } - - fn preload_configs(&mut self, sources: &mut ProjectsSourcesList) -> miette::Result<()> { - let context = self.context(); - let mut configs = FxHashMap::default(); - let mut renamed_ids = FxHashMap::default(); - - for (id, source) in sources { - debug!( - id = id.as_str(), - "Attempting to load {} (optional)", - color::file(source.join(context.config_loader.get_debug_label("moon", false))) - ); - - let config = context - .config_loader - .load_project_config_from_source(context.workspace_root, source)?; - - // Track ID renames - if let Some(new_id) = &config.id { - if new_id != id { - renamed_ids.insert(id.to_owned(), new_id.to_owned()); - *id = new_id.to_owned(); - } - } - - configs.insert(config.id.clone().unwrap_or(id.to_owned()), config); - } - - debug!("Loaded {} project configs", configs.len()); - - self.configs.extend(configs); - self.renamed_ids.extend(renamed_ids); - - Ok(()) - } - - fn context(&self) -> Arc> { - Arc::clone(self.context.as_ref().unwrap()) - } - - fn resolve_id(&self, project_locator: &str) -> Id { - let id = if self.sources.contains_key(project_locator) { - Id::raw(project_locator) - } else { - match self.aliases.iter().find_map(|(id, alias)| { - if alias == project_locator { - Some(id) - } else { - None - } - }) { - Some(project_id) => project_id.to_owned(), - None => Id::raw(project_locator), - } - }; - - match self.renamed_ids.get(&id) { - Some(new_id) => new_id.to_owned(), - None => id, - } - } -} diff --git a/crates/project-graph/src/project_graph_cache.rs b/crates/project-graph/src/project_graph_cache.rs deleted file mode 100644 index f1eba0572c9..00000000000 --- a/crates/project-graph/src/project_graph_cache.rs +++ /dev/null @@ -1,11 +0,0 @@ -use moon_cache::cache_item; -use moon_common::path::WorkspaceRelativePathBuf; -use moon_common::Id; -use rustc_hash::FxHashMap; - -cache_item!( - pub struct ProjectsCacheState { - pub last_hash: String, - pub projects: FxHashMap, - } -); diff --git a/crates/project-graph/src/project_graph_error.rs b/crates/project-graph/src/project_graph_error.rs index 7c274951460..0af71fea63e 100644 --- a/crates/project-graph/src/project_graph_error.rs +++ b/crates/project-graph/src/project_graph_error.rs @@ -5,43 +5,11 @@ use thiserror::Error; #[derive(Error, Debug, Diagnostic)] pub enum ProjectGraphError { - #[diagnostic(code(project_graph::duplicate_alias))] - #[error( - "Project {} is already using the alias {}, unable to set the alias for project {}.\nTry changing the alias to something unique to move forward.", - .old_id.style(Style::Id), - .alias.style(Style::Label), - .new_id.style(Style::Id), - )] - DuplicateAlias { - alias: String, - old_id: Id, - new_id: Id, - }, - - #[diagnostic(code(project_graph::duplicate_id))] - #[error( - "A project already exists with the name {} (existing source {}, new source {}).\nTry renaming the project folder to make it unique, or configure the {} setting in {}.", - .id.style(Style::Id), - .old_source.style(Style::File), - .new_source.style(Style::File), - "id".style(Style::Property), - "moon.yml".style(Style::File) - )] - DuplicateId { - id: Id, - old_source: String, - new_source: String, - }, - - #[diagnostic(code(project_graph::missing_source))] - #[error("No project exists at source path {}.", .0.style(Style::File))] - MissingAtSource(String), - #[diagnostic(code(project_graph::missing_from_path))] #[error("No project could be located starting from path {}.", .0.style(Style::Path))] MissingFromPath(PathBuf), - #[diagnostic(code(project_graph::unknown_project))] - #[error("No project has been configured with the name or alias {}.", .0.style(Style::Id))] + #[diagnostic(code(project_graph::unknown_id))] + #[error("No project has been configured with the identifier or alias {}.", .0.style(Style::Id))] UnconfiguredID(Id), } diff --git a/crates/project-graph/tests/project_graph_test.rs b/crates/project-graph/tests/project_graph_test.rs index 7e4f1ec3c7f..5c0a0ee16ae 100644 --- a/crates/project-graph/tests/project_graph_test.rs +++ b/crates/project-graph/tests/project_graph_test.rs @@ -4,13 +4,15 @@ use moon_config::{ WorkspaceProjectsConfig, }; use moon_project::{FileGroup, Project}; -use moon_project_graph::{ - ExtendProjectData, ExtendProjectEvent, ExtendProjectGraphData, ExtendProjectGraphEvent, - ProjectGraph, ProjectGraphBuilder, -}; +use moon_project_graph::ProjectGraph; use moon_query::build_query; use moon_task::Target; use moon_test_utils2::*; +use moon_workspace::{ + ExtendProjectData, ExtendProjectEvent, ExtendProjectGraphData, ExtendProjectGraphEvent, + WorkspaceProjectsCacheState, +}; +use petgraph::prelude::*; use rustc_hash::{FxHashMap, FxHashSet}; use starbase_events::EventState; use starbase_sandbox::{assert_snapshot, create_sandbox, Sandbox}; @@ -51,7 +53,7 @@ mod project_graph { } #[tokio::test] - #[should_panic(expected = "No project has been configured with the name or alias z")] + #[should_panic(expected = "No project has been configured with the identifier or alias z")] async fn errors_unknown_id() { let graph = generate_project_graph("dependencies").await; @@ -84,7 +86,7 @@ mod project_graph { } #[tokio::test] - #[should_panic(expected = "A project already exists with the name id")] + #[should_panic(expected = "A project already exists with the identifier id")] async fn errors_duplicate_ids() { generate_project_graph("dupe-folder-conflict").await; } @@ -110,12 +112,11 @@ mod project_graph { // Move files so that we can infer a compatible root project name fs::copy_dir_all(sandbox.path(), sandbox.path(), &root).unwrap(); - let mut container = ProjectGraphContainer::new(&root); + let mut mock = create_project_graph_mocker(&root); - container.workspace_config.projects = WorkspaceProjects::Globs(string_vec!["*", "."]); + mock.workspace_config.projects = WorkspaceProjects::Globs(string_vec!["*", "."]); - let context = container.create_context(); - let graph = container.build_graph(context).await; + let graph = mock.build_project_graph().await; assert_eq!( get_ids_from_projects(graph.get_all().unwrap()), @@ -126,13 +127,11 @@ mod project_graph { #[tokio::test] async fn globs_with_config() { let sandbox = create_sandbox("locate-configs"); - let mut container = ProjectGraphContainer::new(sandbox.path()); + let mut mock = create_project_graph_mocker(sandbox.path()); - container.workspace_config.projects = - WorkspaceProjects::Globs(string_vec!["*/moon.yml"]); + mock.workspace_config.projects = WorkspaceProjects::Globs(string_vec!["*/moon.yml"]); - let context = container.create_context(); - let graph = container.build_graph(context).await; + let graph = mock.build_project_graph().await; assert_eq!(get_ids_from_projects(graph.get_all().unwrap()), ["a", "c"]); } @@ -140,16 +139,14 @@ mod project_graph { #[tokio::test] async fn paths() { let sandbox = create_sandbox("dependencies"); - let mut container = ProjectGraphContainer::new(sandbox.path()); + let mut mock = create_project_graph_mocker(sandbox.path()); - container.workspace_config.projects = - WorkspaceProjects::Sources(FxHashMap::from_iter([ - (Id::raw("c"), "c".into()), - (Id::raw("b"), "b".into()), - ])); + mock.workspace_config.projects = WorkspaceProjects::Sources(FxHashMap::from_iter([ + (Id::raw("c"), "c".into()), + (Id::raw("b"), "b".into()), + ])); - let context = container.create_context(); - let graph = container.build_graph(context).await; + let graph = mock.build_project_graph().await; assert_eq!(get_ids_from_projects(graph.get_all().unwrap()), ["b", "c"]); } @@ -157,19 +154,17 @@ mod project_graph { #[tokio::test] async fn paths_and_globs() { let sandbox = create_sandbox("dependencies"); - let mut container = ProjectGraphContainer::new(sandbox.path()); + let mut mock = create_project_graph_mocker(sandbox.path()); - container.workspace_config.projects = - WorkspaceProjects::Both(WorkspaceProjectsConfig { - globs: string_vec!["{a,c}"], - sources: FxHashMap::from_iter([ - (Id::raw("b"), "b".into()), - (Id::raw("root"), ".".into()), - ]), - }); + mock.workspace_config.projects = WorkspaceProjects::Both(WorkspaceProjectsConfig { + globs: string_vec!["{a,c}"], + sources: FxHashMap::from_iter([ + (Id::raw("b"), "b".into()), + (Id::raw("root"), ".".into()), + ]), + }); - let context = container.create_context(); - let graph = container.build_graph(context).await; + let graph = mock.build_project_graph().await; assert_eq!( get_ids_from_projects(graph.get_all().unwrap()), @@ -212,9 +207,10 @@ mod project_graph { sandbox.enable_git(); sandbox.create_file(".gitignore", "*-other"); - let container = ProjectGraphContainer::with_vcs(sandbox.path()); - let context = container.create_context(); - let graph = container.build_graph(context).await; + let mut mock = create_project_graph_mocker(sandbox.path()); + mock.with_vcs(); + + let graph = mock.build_project_graph().await; assert_eq!( get_ids_from_projects(graph.get_all().unwrap()), @@ -243,24 +239,22 @@ mod project_graph { mod cache { use super::*; use moon_cache::CacheEngine; - use moon_project_graph::ProjectsCacheState; + use moon_workspace::ProjectBuildData; - const CACHE_PATH: &str = ".moon/cache/states/partialProjectGraph.json"; - const STATE_PATH: &str = ".moon/cache/states/projects.json"; + const CACHE_PATH: &str = ".moon/cache/states/workspaceGraph.json"; + const STATE_PATH: &str = ".moon/cache/states/projectsBuildData.json"; async fn do_generate(root: &Path) -> ProjectGraph { let cache_engine = CacheEngine::new(root).unwrap(); - let container = ProjectGraphContainer::with_vcs(root); - let context = container.create_context(); - let mut builder = ProjectGraphBuilder::generate(context, &cache_engine) - .await - .unwrap(); - builder.load_all().await.unwrap(); + let mut mock = create_project_graph_mocker(root); + mock.with_vcs(); - let graph = builder.build().await.unwrap(); - graph.get_all().unwrap(); - graph + mock.build_project_graph_with_options(ProjectGraphMockOptions { + cache: Some(cache_engine), + ..Default::default() + }) + .await } async fn generate_cached_project_graph( @@ -310,16 +304,44 @@ mod project_graph { }) .await; - let state: ProjectsCacheState = + let state: WorkspaceProjectsCacheState = json::read_file(sandbox.path().join(STATE_PATH)).unwrap(); assert_eq!( state.projects, FxHashMap::from_iter([ - (Id::raw("a"), "a".into()), - (Id::raw("b"), "b".into()), - (Id::raw("c"), "c".into()), - (Id::raw("d"), "d".into()), + ( + Id::raw("a"), + ProjectBuildData { + node_index: Some(NodeIndex::from(2)), + source: "a".into(), + ..Default::default() + } + ), + ( + Id::raw("b"), + ProjectBuildData { + node_index: Some(NodeIndex::from(1)), + source: "b".into(), + ..Default::default() + } + ), + ( + Id::raw("c"), + ProjectBuildData { + node_index: Some(NodeIndex::from(0)), + source: "c".into(), + ..Default::default() + } + ), + ( + Id::raw("d"), + ProjectBuildData { + node_index: Some(NodeIndex::from(3)), + source: "d".into(), + ..Default::default() + } + ), ]) ); @@ -339,13 +361,13 @@ mod project_graph { }) .await; - let state1: ProjectsCacheState = + let state1: WorkspaceProjectsCacheState = json::read_file(sandbox.path().join(STATE_PATH)).unwrap(); func(&sandbox); do_generate(sandbox.path()).await; - let state2: ProjectsCacheState = + let state2: WorkspaceProjectsCacheState = json::read_file(sandbox.path().join(STATE_PATH)).unwrap(); assert_ne!(state1.last_hash, state2.last_hash); @@ -440,14 +462,14 @@ mod project_graph { async fn generate_inheritance_project_graph(fixture: &str) -> ProjectGraph { let sandbox = create_sandbox(fixture); + let mut mock = create_project_graph_mocker(sandbox.path()); - generate_project_graph_with_changes(sandbox.path(), |container| { - container.inherited_tasks = container - .config_loader - .load_tasks_manager_from(sandbox.path(), sandbox.path().join(".moon")) - .unwrap(); - }) - .await + mock.inherited_tasks = mock + .config_loader + .load_tasks_manager_from(sandbox.path(), sandbox.path().join(".moon")) + .unwrap(); + + mock.build_project_graph().await } #[tokio::test] @@ -710,9 +732,9 @@ mod project_graph { #[tokio::test] async fn no_depends_on() { let sandbox = create_sandbox("dependency-types"); - let container = ProjectGraphContainer::new(sandbox.path()); - let context = container.create_context(); - let graph = container.build_graph_for(context, &["no-depends-on"]).await; + let mock = create_project_graph_mocker(sandbox.path()); + + let graph = mock.build_project_graph_for(&["no-depends-on"]).await; assert_eq!(map_ids(graph.ids()), ["no-depends-on"]); } @@ -720,11 +742,9 @@ mod project_graph { #[tokio::test] async fn some_depends_on() { let sandbox = create_sandbox("dependency-types"); - let container = ProjectGraphContainer::new(sandbox.path()); - let context = container.create_context(); - let graph = container - .build_graph_for(context, &["some-depends-on"]) - .await; + let mock = create_project_graph_mocker(sandbox.path()); + + let graph = mock.build_project_graph_for(&["some-depends-on"]).await; assert_eq!(map_ids(graph.ids()), ["a", "c", "some-depends-on"]); } @@ -732,11 +752,9 @@ mod project_graph { #[tokio::test] async fn from_task_deps() { let sandbox = create_sandbox("dependency-types"); - let container = ProjectGraphContainer::new(sandbox.path()); - let context = container.create_context(); - let graph = container - .build_graph_for(context, &["from-task-deps"]) - .await; + let mock = create_project_graph_mocker(sandbox.path()); + + let graph = mock.build_project_graph_for(&["from-task-deps"]).await; assert_eq!(map_ids(graph.ids()), ["b", "c", "from-task-deps"]); @@ -749,11 +767,9 @@ mod project_graph { #[tokio::test] async fn from_root_task_deps() { let sandbox = create_sandbox("dependency-types"); - let container = ProjectGraphContainer::new(sandbox.path()); - let context = container.create_context(); - let graph = container - .build_graph_for(context, &["from-root-task-deps"]) - .await; + let mock = create_project_graph_mocker(sandbox.path()); + + let graph = mock.build_project_graph_for(&["from-root-task-deps"]).await; assert_eq!(map_ids(graph.ids()), ["root", "from-root-task-deps"]); @@ -765,11 +781,9 @@ mod project_graph { #[tokio::test] async fn self_task_deps() { let sandbox = create_sandbox("dependency-types"); - let container = ProjectGraphContainer::new(sandbox.path()); - let context = container.create_context(); - let graph = container - .build_graph_for(context, &["self-task-deps"]) - .await; + let mock = create_project_graph_mocker(sandbox.path()); + + let graph = mock.build_project_graph_for(&["self-task-deps"]).await; assert_eq!(map_ids(graph.ids()), ["self-task-deps"]); } @@ -785,8 +799,8 @@ mod project_graph { async fn generate_aliases_project_graph_for_fixture(fixture: &str) -> ProjectGraph { let sandbox = create_sandbox(fixture); - let container = ProjectGraphContainer::new(sandbox.path()); - let context = container.create_context(); + let mock = create_project_graph_mocker(sandbox.path()); + let context = mock.create_context(); // Set aliases for projects context @@ -841,7 +855,11 @@ mod project_graph { ) .await; - container.build_graph(context).await + mock.build_project_graph_with_options(ProjectGraphMockOptions { + context: Some(context), + ..Default::default() + }) + .await } #[tokio::test] @@ -970,7 +988,7 @@ mod project_graph { } #[tokio::test] - #[should_panic(expected = "Project one is already using the alias @test")] + #[should_panic(expected = "Project two is already using the alias @test")] async fn errors_duplicate_aliases() { generate_aliases_project_graph_for_fixture("aliases-conflict").await; } @@ -978,8 +996,8 @@ mod project_graph { #[tokio::test] async fn ignores_duplicate_aliases_if_ids_match() { let sandbox = create_sandbox("aliases-conflict"); - let container = ProjectGraphContainer::new(sandbox.path()); - let context = container.create_context(); + let mock = create_project_graph_mocker(sandbox.path()); + let context = mock.create_context(); context .extend_project_graph @@ -999,7 +1017,12 @@ mod project_graph { ) .await; - let graph = container.build_graph(context).await; + let graph = mock + .build_project_graph_with_options(ProjectGraphMockOptions { + context: Some(context), + ..Default::default() + }) + .await; assert!(graph.get("@one").is_ok()); assert!(graph.get("@two").is_ok()); @@ -1016,16 +1039,13 @@ mod project_graph { func(&sandbox); - let mut container = ProjectGraphContainer::new(sandbox.path()); + let mut mock = create_project_graph_mocker(sandbox.path()); - container - .workspace_config + mock.workspace_config .constraints .enforce_project_type_relationships = true; - let context = container.create_context(); - - container.build_graph(context).await + mock.build_project_graph().await } #[tokio::test] @@ -1152,29 +1172,19 @@ mod project_graph { func(&sandbox); - let mut container = ProjectGraphContainer::new(sandbox.path()); + let mut mock = create_project_graph_mocker(sandbox.path()); - container - .workspace_config - .constraints - .tag_relationships - .insert( - Id::raw("warrior"), - vec![Id::raw("barbarian"), Id::raw("paladin"), Id::raw("druid")], - ); - - container - .workspace_config - .constraints - .tag_relationships - .insert( - Id::raw("mage"), - vec![Id::raw("wizard"), Id::raw("sorcerer"), Id::raw("druid")], - ); + mock.workspace_config.constraints.tag_relationships.insert( + Id::raw("warrior"), + vec![Id::raw("barbarian"), Id::raw("paladin"), Id::raw("druid")], + ); - let context = container.create_context(); + mock.workspace_config.constraints.tag_relationships.insert( + Id::raw("mage"), + vec![Id::raw("wizard"), Id::raw("sorcerer"), Id::raw("druid")], + ); - container.build_graph(context).await + mock.build_project_graph().await } #[tokio::test] @@ -1428,9 +1438,9 @@ mod project_graph { #[tokio::test] async fn renders_partial() { let sandbox = create_sandbox("dependencies"); - let container = ProjectGraphContainer::new(sandbox.path()); - let context = container.create_context(); - let graph = container.build_graph_for(context, &["b"]).await; + let mock = create_project_graph_mocker(sandbox.path()); + + let graph = mock.build_project_graph_for(&["b"]).await; assert_snapshot!(graph.to_dot()); } @@ -1475,7 +1485,7 @@ mod project_graph { } #[tokio::test] - #[should_panic(expected = "A project already exists with the name foo")] + #[should_panic(expected = "A project already exists with the identifier foo")] async fn errors_duplicate_ids_from_rename() { generate_project_graph("custom-id-conflict").await; } diff --git a/crates/task-builder/src/tasks_builder.rs b/crates/task-builder/src/tasks_builder.rs index f27f01531d1..77a5a88e612 100644 --- a/crates/task-builder/src/tasks_builder.rs +++ b/crates/task-builder/src/tasks_builder.rs @@ -135,7 +135,7 @@ impl<'proj> TasksBuilder<'proj> { } trace!( - id = self.project_id, + project_id = self.project_id, tasks = ?global_config.tasks.keys().map(|k| k.as_str()).collect::>(), "Filtering global tasks", ); @@ -214,7 +214,7 @@ impl<'proj> TasksBuilder<'proj> { } trace!( - id = self.project_id, + project_id = self.project_id, tasks = ?local_config.tasks.keys().map(|k| k.as_str()).collect::>(), "Loading local tasks", ); diff --git a/crates/task-hasher/tests/task_hasher_test.rs b/crates/task-hasher/tests/task_hasher_test.rs index c6e4557fcdc..a341fc18f25 100644 --- a/crates/task-hasher/tests/task_hasher_test.rs +++ b/crates/task-hasher/tests/task_hasher_test.rs @@ -1,7 +1,7 @@ use moon_config::{GlobPath, HasherConfig, HasherWalkStrategy, PortablePath}; use moon_project::Project; use moon_task_hasher::{TaskHash, TaskHasher}; -use moon_test_utils2::{ProjectGraph, ProjectGraphContainer}; +use moon_test_utils2::{create_project_graph_mocker, ProjectGraph}; use moon_vcs::BoxedVcs; use starbase_sandbox::create_sandbox; use std::fs; @@ -32,13 +32,13 @@ fn create_hasher_configs() -> (HasherConfig, HasherConfig) { } async fn generate_project_graph(workspace_root: &Path) -> (ProjectGraph, Arc) { - let mut graph_builder = ProjectGraphContainer::with_vcs(workspace_root); - let context = graph_builder.create_context(); + let mut mock = create_project_graph_mocker(workspace_root); + mock.with_vcs(); create_out_files(workspace_root); - let graph = graph_builder.build_graph(context).await; - let vcs = graph_builder.vcs.take().unwrap(); + let graph = mock.build_project_graph().await; + let vcs = mock.vcs.take().unwrap(); (graph, vcs) } diff --git a/crates/test-utils/Cargo.toml b/crates/test-utils/Cargo.toml index 60ebb16a5a7..09b8cce8012 100644 --- a/crates/test-utils/Cargo.toml +++ b/crates/test-utils/Cargo.toml @@ -15,6 +15,7 @@ moon_config = { path = "../config" } moon_console = { path = "../console" } moon_project_graph = { path = "../project-graph" } moon_vcs = { path = "../vcs" } +moon_workspace = { path = "../workspace" } proto_core = { workspace = true } starbase_events = { workspace = true } starbase_sandbox = { workspace = true } diff --git a/crates/test-utils/src/lib.rs b/crates/test-utils/src/lib.rs index 326322ab64b..17f392f4482 100644 --- a/crates/test-utils/src/lib.rs +++ b/crates/test-utils/src/lib.rs @@ -2,9 +2,11 @@ mod app_context; mod platform_manager; mod project_graph; mod sandbox; +mod workspace_mocker; pub use app_context::*; pub use platform_manager::*; pub use project_graph::*; pub use sandbox::*; pub use starbase_sandbox::{predicates, pretty_assertions}; +pub use workspace_mocker::*; diff --git a/crates/test-utils/src/project_graph.rs b/crates/test-utils/src/project_graph.rs index ff8d86fd1cc..94174b76510 100644 --- a/crates/test-utils/src/project_graph.rs +++ b/crates/test-utils/src/project_graph.rs @@ -1,139 +1,18 @@ -use moon_config::{ - ConfigLoader, InheritedTasksEntry, InheritedTasksManager, NodeConfig, - PartialInheritedTasksConfig, PartialTaskConfig, ToolchainConfig, WorkspaceConfig, - WorkspaceProjects, WorkspaceProjectsConfig, -}; -use moon_project_graph::{ - ExtendProjectEvent, ExtendProjectGraphEvent, ProjectGraphBuilder, ProjectGraphBuilderContext, -}; -use moon_vcs::{BoxedVcs, Git}; -use proto_core::ProtoConfig; -use starbase_events::Emitter; +use crate::workspace_mocker::*; use starbase_sandbox::create_sandbox; -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::path::Path; pub use moon_project_graph::ProjectGraph; -#[derive(Default)] -pub struct ProjectGraphContainer { - pub config_loader: ConfigLoader, - pub inherited_tasks: InheritedTasksManager, - pub toolchain_config: ToolchainConfig, - pub workspace_config: WorkspaceConfig, - pub workspace_root: PathBuf, - pub vcs: Option>, -} - -impl ProjectGraphContainer { - pub fn new(root: &Path) -> Self { - let proto_config = ProtoConfig::default(); - let config_loader = ConfigLoader::default(); - let mut graph = Self { - inherited_tasks: config_loader.load_tasks_manager(root).unwrap(), - toolchain_config: config_loader - .load_toolchain_config(root, &proto_config) - .unwrap(), - workspace_root: root.to_path_buf(), - config_loader, - ..Default::default() - }; - - // Add a global task to all projects - graph.inherited_tasks.configs.insert( - "*".into(), - InheritedTasksEntry { - input: ".moon/tasks.yml".into(), - config: PartialInheritedTasksConfig { - tasks: Some(BTreeMap::from_iter([( - "global".try_into().unwrap(), - PartialTaskConfig::default(), - )])), - ..PartialInheritedTasksConfig::default() - }, - }, - ); - - // Always use the node platform - if graph.toolchain_config.node.is_none() { - graph.toolchain_config.node = Some(NodeConfig::default()); - } - - // Use folders as project names - if root.join(".moon/workspace.yml").exists() { - graph.workspace_config = graph.config_loader.load_workspace_config(root).unwrap(); - } else { - let mut projects = WorkspaceProjectsConfig { - globs: vec![ - "*".into(), - "!.home".into(), - "!.moon".into(), - "!.proto".into(), - ], - ..WorkspaceProjectsConfig::default() - }; - - if root.join("moon.yml").exists() { - projects - .sources - .insert("root".try_into().unwrap(), ".".into()); - } - - graph.workspace_config.projects = WorkspaceProjects::Both(projects); - } - - graph - } - - pub fn with_vcs(root: &Path) -> Self { - let mut container = Self::new(root); - container.vcs = Some(Arc::new(Box::new(Git::load(root, "master", &[]).unwrap()))); - container - } - - pub fn create_context(&self) -> ProjectGraphBuilderContext { - ProjectGraphBuilderContext { - config_loader: &self.config_loader, - extend_project: Emitter::::new(), - extend_project_graph: Emitter::::new(), - inherited_tasks: &self.inherited_tasks, - toolchain_config: &self.toolchain_config, - vcs: self.vcs.clone(), - working_dir: &self.workspace_root, - workspace_config: &self.workspace_config, - workspace_root: &self.workspace_root, - } - } +pub fn create_project_graph_mocker(root: &Path) -> WorkspaceMocker { + let mut mock = WorkspaceMocker::new(root); - pub async fn build_graph<'l>(&self, context: ProjectGraphBuilderContext<'l>) -> ProjectGraph { - let mut builder = ProjectGraphBuilder::new(context).await.unwrap(); - builder.load_all().await.unwrap(); + mock.with_default_configs() + .with_default_projects() + .with_default_toolchain() + .with_global_tasks(); - let graph = builder.build().await.unwrap(); - graph.get_all().unwrap(); - graph - } - - pub async fn build_graph_for<'l>( - &self, - context: ProjectGraphBuilderContext<'l>, - ids: &[&str], - ) -> ProjectGraph { - let mut builder = ProjectGraphBuilder::new(context).await.unwrap(); - - for id in ids { - builder.load(id).await.unwrap(); - } - - let graph = builder.build().await.unwrap(); - - for id in ids { - graph.get(id).unwrap(); - } - - graph - } + mock } pub async fn generate_project_graph(fixture: &str) -> ProjectGraph { @@ -141,18 +20,7 @@ pub async fn generate_project_graph(fixture: &str) -> ProjectGraph { } pub async fn generate_project_graph_from_sandbox(root: &Path) -> ProjectGraph { - generate_project_graph_with_changes(root, |_| {}).await -} - -pub async fn generate_project_graph_with_changes(root: &Path, mut op: F) -> ProjectGraph -where - F: FnMut(&mut ProjectGraphContainer), -{ - let mut container = ProjectGraphContainer::new(root); - - op(&mut container); - - let context = container.create_context(); - - container.build_graph(context).await + create_project_graph_mocker(root) + .build_project_graph() + .await } diff --git a/crates/test-utils/src/workspace_mocker.rs b/crates/test-utils/src/workspace_mocker.rs new file mode 100644 index 00000000000..e62a950feac --- /dev/null +++ b/crates/test-utils/src/workspace_mocker.rs @@ -0,0 +1,175 @@ +use moon_cache::CacheEngine; +use moon_config::*; +use moon_project_graph::ProjectGraph; +use moon_vcs::{BoxedVcs, Git}; +use moon_workspace::*; +use proto_core::ProtoConfig; +use starbase_events::Emitter; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +#[derive(Default)] +pub struct WorkspaceMocker { + pub config_loader: ConfigLoader, + pub inherited_tasks: InheritedTasksManager, + pub toolchain_config: ToolchainConfig, + pub workspace_config: WorkspaceConfig, + pub workspace_root: PathBuf, + pub vcs: Option>, +} + +impl WorkspaceMocker { + pub fn new(root: impl AsRef) -> Self { + Self { + workspace_root: root.as_ref().to_path_buf(), + ..Default::default() + } + } + + pub fn with_default_configs(&mut self) -> &mut Self { + let root = &self.workspace_root; + + self.inherited_tasks = self.config_loader.load_tasks_manager(root).unwrap(); + + self.toolchain_config = self + .config_loader + .load_toolchain_config(root, &ProtoConfig::default()) + .unwrap(); + + self.workspace_config = self.config_loader.load_workspace_config(root).unwrap(); + + self + } + + pub fn with_default_projects(&mut self) -> &mut Self { + if !self.workspace_root.join(".moon/workspace.yml").exists() { + // Use folders as project names + let mut projects = WorkspaceProjectsConfig { + globs: vec![ + "*".into(), + "!.home".into(), + "!.moon".into(), + "!.proto".into(), + ], + ..WorkspaceProjectsConfig::default() + }; + + // Include a root project conditionally + if self.workspace_root.join("moon.yml").exists() { + projects + .sources + .insert("root".try_into().unwrap(), ".".into()); + } + + self.workspace_config.projects = WorkspaceProjects::Both(projects); + } + + self + } + + pub fn with_default_toolchain(&mut self) -> &mut Self { + if self.toolchain_config.node.is_none() { + self.toolchain_config.node = Some(NodeConfig::default()); + } + + self + } + + pub fn with_global_tasks(&mut self) -> &mut Self { + self.inherited_tasks.configs.insert( + "*".into(), + InheritedTasksEntry { + input: ".moon/tasks.yml".into(), + config: PartialInheritedTasksConfig { + tasks: Some(BTreeMap::from_iter([( + "global".try_into().unwrap(), + PartialTaskConfig::default(), + )])), + ..PartialInheritedTasksConfig::default() + }, + }, + ); + + self + } + + pub fn with_vcs(&mut self) -> &mut Self { + self.vcs = Some(Arc::new(Box::new( + Git::load(&self.workspace_root, "master", &[]).unwrap(), + ))); + + self + } + + pub fn create_context(&self) -> WorkspaceBuilderContext { + WorkspaceBuilderContext { + config_loader: &self.config_loader, + extend_project: Emitter::::new(), + extend_project_graph: Emitter::::new(), + inherited_tasks: &self.inherited_tasks, + toolchain_config: &self.toolchain_config, + vcs: self.vcs.clone(), + working_dir: &self.workspace_root, + workspace_config: &self.workspace_config, + workspace_root: &self.workspace_root, + } + } + + pub async fn build_project_graph(&self) -> ProjectGraph { + self.build_project_graph_with_options(ProjectGraphMockOptions::default()) + .await + } + + pub async fn build_project_graph_for(&self, ids: &[&str]) -> ProjectGraph { + self.build_project_graph_with_options(ProjectGraphMockOptions { + ids: Vec::from_iter(ids.iter().map(|id| id.to_string())), + ..Default::default() + }) + .await + } + + pub async fn build_project_graph_with_options<'l>( + &self, + mut options: ProjectGraphMockOptions<'l>, + ) -> ProjectGraph { + let context = options + .context + .take() + .unwrap_or_else(|| self.create_context()); + + let mut builder = match &options.cache { + Some(engine) => WorkspaceBuilder::new_with_cache(context, engine) + .await + .unwrap(), + None => WorkspaceBuilder::new(context).await.unwrap(), + }; + + if options.ids.is_empty() { + builder.load_projects().await.unwrap(); + } else { + for id in &options.ids { + builder.load_project(id).await.unwrap(); + } + } + + let project_graph = builder.build().await.unwrap().project_graph; + + if options.ids.is_empty() { + project_graph.get_all().unwrap(); + } else { + for id in &options.ids { + project_graph.get(id).unwrap(); + } + } + + project_graph + } +} + +#[derive(Default)] +pub struct ProjectGraphMockOptions<'l> { + pub cache: Option, + pub context: Option>, + pub ids: Vec, +} diff --git a/crates/workspace/Cargo.toml b/crates/workspace/Cargo.toml new file mode 100644 index 00000000000..4ea75dadb98 --- /dev/null +++ b/crates/workspace/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "moon_workspace" +version = "0.0.1" +edition = "2021" +license = "MIT" +description = "Workspace utilities." +homepage = "https://moonrepo.dev/moon" +repository = "https://github.com/moonrepo/moon" +publish = false + +[dependencies] +moon_cache = { path = "../cache" } +moon_common = { path = "../common" } +moon_config = { path = "../config" } +moon_hash = { path = "../hash" } +moon_project = { path = "../project" } +moon_project_builder = { path = "../project-builder" } +moon_project_constraints = { path = "../project-constraints" } +moon_project_graph = { path = "../project-graph" } +moon_vcs = { path = "../vcs" } +miette = { workspace = true } +petgraph = { workspace = true } +rustc-hash = { workspace = true } +serde = { workspace = true } +starbase_events = { workspace = true } +starbase_utils = { workspace = true, features = ["glob", "json"] } +thiserror = { workspace = true } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/crates/workspace/src/lib.rs b/crates/workspace/src/lib.rs new file mode 100644 index 00000000000..84af662dc70 --- /dev/null +++ b/crates/workspace/src/lib.rs @@ -0,0 +1,12 @@ +mod project_build_data; +mod projects_locator; +mod repo_type; +mod workspace_builder; +mod workspace_builder_error; +mod workspace_cache; + +pub use project_build_data::*; +pub use repo_type::*; +pub use workspace_builder::*; +pub use workspace_builder_error::*; +pub use workspace_cache::*; diff --git a/crates/project-graph/src/project_events.rs b/crates/workspace/src/project_build_data.rs similarity index 59% rename from crates/project-graph/src/project_events.rs rename to crates/workspace/src/project_build_data.rs index a1979dc43ff..acb37b64337 100644 --- a/crates/project-graph/src/project_events.rs +++ b/crates/workspace/src/project_build_data.rs @@ -1,10 +1,32 @@ use moon_common::path::WorkspaceRelativePathBuf; use moon_common::Id; -use moon_config::{DependencyConfig, ProjectsAliasesList, ProjectsSourcesList, TaskConfig}; +use moon_config::{ + DependencyConfig, ProjectConfig, ProjectsAliasesList, ProjectsSourcesList, TaskConfig, +}; +use petgraph::graph::NodeIndex; use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; use starbase_events::Event; use std::path::PathBuf; +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[serde(default)] +pub struct ProjectBuildData { + #[serde(skip_serializing_if = "Option::is_none")] + pub alias: Option, + + #[serde(skip)] + pub config: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub node_index: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub original_id: Option, + + pub source: WorkspaceRelativePathBuf, +} + // Extend the project graph with additional information. #[derive(Debug)] diff --git a/crates/project-graph/src/projects_locator.rs b/crates/workspace/src/projects_locator.rs similarity index 84% rename from crates/project-graph/src/projects_locator.rs rename to crates/workspace/src/projects_locator.rs index 969ea59519a..049bea202ea 100644 --- a/crates/project-graph/src/projects_locator.rs +++ b/crates/workspace/src/projects_locator.rs @@ -1,24 +1,13 @@ -use crate::project_graph_builder::ProjectGraphBuilderContext; +use crate::workspace_builder::WorkspaceBuilderContext; use moon_common::path::{is_root_level_source, to_virtual_string, WorkspaceRelativePathBuf}; use moon_common::{color, consts, Id}; use moon_config::{ProjectSourceEntry, ProjectsSourcesList}; use starbase_utils::{fs, glob}; -use std::path::Path; use tracing::{debug, instrument, warn}; /// Infer a project name from a source path, by using the name of /// the project folder. -fn infer_project_id_and_source( - path: &str, - workspace_root: &Path, -) -> miette::Result { - if path.is_empty() { - return Ok(( - Id::clean(fs::file_name(workspace_root))?, - WorkspaceRelativePathBuf::from("."), - )); - } - +fn infer_project_id_and_source(path: &str) -> miette::Result { let (id, source) = if path.contains('/') { (path.split('/').last().unwrap().to_owned(), path) } else { @@ -32,7 +21,7 @@ fn infer_project_id_and_source( /// for potential projects, and infer their name and source. #[instrument(skip_all)] pub fn locate_projects_with_globs<'glob, I, V>( - context: &ProjectGraphBuilderContext, + context: &WorkspaceBuilderContext, globs: I, sources: &mut ProjectsSourcesList, ) -> miette::Result<()> @@ -55,7 +44,10 @@ where } has_root_level = true; - sources.push(infer_project_id_and_source("", context.workspace_root)?); + sources.push(( + Id::clean(fs::file_name(context.workspace_root))?, + WorkspaceRelativePathBuf::from("."), + )); } else { locate_globs.push(glob); } @@ -80,7 +72,8 @@ where // Don't warn on dotfiles if project_root .file_name() - .map(|name| !name.to_string_lossy().starts_with('.')) + .and_then(|name| name.to_str()) + .map(|name| !name.starts_with('.')) .unwrap_or_default() { warn!( @@ -115,8 +108,7 @@ where } } - let (id, source) = - infer_project_id_and_source(&project_source, context.workspace_root)?; + let (id, source) = infer_project_id_and_source(&project_source)?; if id.starts_with(".") { debug!( diff --git a/crates/workspace/src/repo_type.rs b/crates/workspace/src/repo_type.rs new file mode 100644 index 00000000000..9b51c6bb2b6 --- /dev/null +++ b/crates/workspace/src/repo_type.rs @@ -0,0 +1,17 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Default, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum RepoType { + #[default] + Unknown, + Monorepo, + MonorepoWithRoot, + Polyrepo, +} + +impl RepoType { + pub fn is_monorepo(&self) -> bool { + matches!(self, Self::Monorepo | Self::MonorepoWithRoot) + } +} diff --git a/crates/workspace/src/workspace_builder.rs b/crates/workspace/src/workspace_builder.rs new file mode 100644 index 00000000000..c99f503b256 --- /dev/null +++ b/crates/workspace/src/workspace_builder.rs @@ -0,0 +1,681 @@ +use crate::project_build_data::*; +use crate::projects_locator::locate_projects_with_globs; +use crate::repo_type::RepoType; +use crate::workspace_builder_error::WorkspaceBuilderError; +use crate::workspace_cache::*; +use moon_cache::CacheEngine; +use moon_common::{ + color, consts, + path::{is_root_level_source, to_virtual_string, WorkspaceRelativePathBuf}, + Id, +}; +use moon_config::{ + ConfigLoader, DependencyScope, InheritedTasksManager, ProjectsSourcesList, ToolchainConfig, + WorkspaceConfig, WorkspaceProjects, +}; +use moon_project::Project; +use moon_project_builder::{ProjectBuilder, ProjectBuilderContext}; +use moon_project_constraints::{enforce_project_type_relationships, enforce_tag_relationships}; +use moon_project_graph::{ProjectGraph, ProjectGraphError, ProjectGraphType, ProjectNode}; +use moon_vcs::BoxedVcs; +use petgraph::prelude::*; +use petgraph::visit::IntoNodeReferences; +use rustc_hash::{FxHashMap, FxHashSet}; +use serde::{Deserialize, Serialize}; +use starbase_events::Emitter; +use starbase_utils::{glob, json}; +use std::sync::Arc; +use std::{collections::BTreeMap, path::Path}; +use tracing::{debug, instrument, trace}; + +pub struct WorkspaceBuilderContext<'app> { + pub config_loader: &'app ConfigLoader, + pub extend_project: Emitter, + pub extend_project_graph: Emitter, + pub inherited_tasks: &'app InheritedTasksManager, + pub toolchain_config: &'app ToolchainConfig, + pub vcs: Option>, + pub working_dir: &'app Path, + pub workspace_config: &'app WorkspaceConfig, + pub workspace_root: &'app Path, +} + +pub struct WorkspaceBuildResult { + pub project_graph: ProjectGraph, +} + +#[derive(Deserialize, Serialize)] +pub struct WorkspaceBuilder<'app> { + #[serde(skip)] + context: Option>>, + + /// Mapping of project IDs to associated data required for building + /// the project itself. Currently we track the following: + /// - The alias, derived from manifests (`package.json`). + /// - Their `moon.yml` in the project root. + /// - Their file source location, relative from the workspace root. + project_data: FxHashMap, + + /// The project DAG. + project_graph: ProjectGraphType, + + /// Projects that have explicitly renamed themselves with the `id` setting. + /// Maps original ID to renamed ID. + renamed_project_ids: FxHashMap, + + /// The type of repository: monorepo or polyrepo. + repo_type: RepoType, + + /// The root project ID (only if a monorepo). + root_project_id: Option, +} + +impl<'app> WorkspaceBuilder<'app> { + #[instrument(skip_all)] + pub async fn new( + context: WorkspaceBuilderContext<'app>, + ) -> miette::Result> { + debug!("Building workspace graph (project and task graphs)"); + + let mut graph = WorkspaceBuilder { + context: Some(Arc::new(context)), + project_data: FxHashMap::default(), + project_graph: ProjectGraphType::default(), + renamed_project_ids: FxHashMap::default(), + repo_type: RepoType::Unknown, + root_project_id: None, + }; + + graph.preload_build_data().await?; + graph.determine_repo_type()?; + + Ok(graph) + } + + #[instrument(skip_all)] + pub async fn new_with_cache( + context: WorkspaceBuilderContext<'app>, + cache_engine: &CacheEngine, + ) -> miette::Result> { + let is_vcs_enabled = context + .vcs + .as_ref() + .expect("VCS is required for project graph caching!") + .is_enabled(); + let mut graph = Self::new(context).await?; + + // No VCS to hash with, so abort caching + if !is_vcs_enabled { + graph.load_projects().await?; + + return Ok(graph); + } + + // Hash the project graph based on the preloaded state + let mut graph_contents = WorkspaceGraphHash::default(); + graph_contents.add_projects(&graph.project_data); + graph_contents.add_configs(graph.hash_required_configs().await?); + graph_contents.gather_env(); + + let hash = cache_engine + .hash + .save_manifest_without_hasher("Workspace graph", &graph_contents)?; + + debug!(hash, "Generated hash for workspace graph"); + + // Check the current state and cache + let mut state = cache_engine + .state + .load_state::("projectsBuildData.json")?; + let cache_path = cache_engine.state.resolve_path("workspaceGraph.json"); + + if hash == state.data.last_hash && cache_path.exists() { + debug!( + cache = ?cache_path, + "Loading workspace graph with {} projects from cache", + graph.project_data.len(), + ); + + let mut cache: WorkspaceBuilder = json::read_file(cache_path)?; + cache.context = graph.context; + + return Ok(cache); + } + + // Build the graph, update the state, and save the cache + debug!( + "Preparing workspace graph with {} projects", + graph.project_data.len(), + ); + + graph.load_projects().await?; + + state.data.last_hash = hash; + state.data.projects = graph.project_data.clone(); + state.save()?; + + json::write_file(cache_path, &graph, false)?; + + Ok(graph) + } + + /// Build the project graph and return a new structure. + #[instrument(name = "build_workspace_graph", skip_all)] + pub async fn build(mut self) -> miette::Result { + self.enforce_constraints()?; + + let context = self.context.take().unwrap(); + + let project_nodes = self + .project_data + .into_iter() + .map(|(id, data)| { + ( + id, + ProjectNode { + alias: data.alias, + index: data.node_index.unwrap_or_default(), + original_id: data.original_id, + source: data.source, + }, + ) + }) + .collect::>(); + + let mut project_graph = + ProjectGraph::new(self.project_graph, project_nodes, context.workspace_root); + + project_graph.working_dir = context.working_dir.to_owned(); + + Ok(WorkspaceBuildResult { project_graph }) + } + + /// Load a single project by ID or alias into the graph. + pub async fn load_project(&mut self, id_or_alias: &str) -> miette::Result<()> { + self.internal_load_project(id_or_alias, &mut FxHashSet::default()) + .await?; + + Ok(()) + } + + /// Load all projects into the graph, as configured in the workspace. + pub async fn load_projects(&mut self) -> miette::Result<()> { + let ids = self.project_data.keys().cloned().collect::>(); + + for id in ids { + self.internal_load_project(&id, &mut FxHashSet::default()) + .await?; + } + + Ok(()) + } + + #[instrument(name = "load_project", skip(self, cycle))] + async fn internal_load_project( + &mut self, + id_or_alias: &str, + cycle: &mut FxHashSet, + ) -> miette::Result<(Id, NodeIndex)> { + let id = self.resolve_project_id(id_or_alias); + + { + let Some(build_data) = self.project_data.get(&id) else { + return Err(ProjectGraphError::UnconfiguredID(id).into()); + }; + + // Already loaded, exit early with existing index + if let Some(index) = &build_data.node_index { + trace!( + project_id = id.as_str(), + "Project already exists in the project graph, skipping load", + ); + + return Ok((id, *index)); + } + } + + // Not loaded, build the project + trace!( + project_id = id.as_str(), + "Project does not exist in the project graph, attempting to load", + ); + + let mut project = self.build_project(&id).await?; + + cycle.insert(id.clone()); + + // Then build dependency projects + let mut edges = vec![]; + + for dep_config in &mut project.dependencies { + if cycle.contains(&dep_config.id) { + debug!( + project_id = id.as_str(), + dependency_id = dep_config.id.as_str(), + "Encountered a dependency cycle (from project); will disconnect nodes to avoid recursion", + ); + + continue; + } + + let dep = Box::pin(self.internal_load_project(&dep_config.id, cycle)).await?; + let dep_id = dep.0; + + // Don't link the root project to any project, but still load it + if !dep_config.is_root_scope() { + edges.push((dep.1, dep_config.scope)); + } + + // TODO is this needed? + if dep_id != dep_config.id { + dep_config.id = dep_id; + } + } + + // And finally add to the graph + let index = self.project_graph.add_node(project); + + self.project_data.get_mut(&id).unwrap().node_index = Some(index); + + for edge in edges { + self.project_graph.add_edge(index, edge.0, edge.1); + } + + cycle.clear(); + + Ok((id, index)) + } + + /// Create and build the project with the provided ID and source. + #[instrument(skip(self))] + async fn build_project(&mut self, id: &Id) -> miette::Result { + debug!( + project_id = id.as_str(), + "Building project {}", + color::id(id) + ); + + let context = self.context(); + let build_data = self.project_data.get(id).unwrap(); + + if !build_data.source.to_path(context.workspace_root).exists() { + return Err(WorkspaceBuilderError::MissingProjectAtSource( + build_data.source.to_string(), + ) + .into()); + } + + let mut builder = ProjectBuilder::new( + id, + &build_data.source, + ProjectBuilderContext { + config_loader: context.config_loader, + monorepo: self.repo_type.is_monorepo(), + root_project_id: self.root_project_id.as_ref(), + toolchain_config: context.toolchain_config, + workspace_root: context.workspace_root, + }, + )?; + + if let Some(config) = &build_data.config { + builder.inherit_local_config(config).await?; + } else { + builder.load_local_config().await?; + } + + builder.inherit_global_config(context.inherited_tasks)?; + + let extended_data = context + .extend_project + .emit(ExtendProjectEvent { + project_id: id.to_owned(), + project_source: build_data.source.to_owned(), + workspace_root: context.workspace_root.to_owned(), + }) + .await?; + + // Inherit implicit dependencies + for dep_config in extended_data.dependencies { + builder.extend_with_dependency(dep_config); + } + + // Inherit inferred tasks + for (task_id, task_config) in extended_data.tasks { + builder.extend_with_task(task_id, task_config); + } + + // Inherit alias before building in case the project + // references itself in tasks or dependencies + if let Some(alias) = &build_data.alias { + builder.set_alias(alias); + } + + let project = builder.build().await?; + + Ok(project) + } + + /// Determine the repository type/structure based on the number of project + /// sources, and where the point to. + fn determine_repo_type(&mut self) -> miette::Result<()> { + let single_project = self.project_data.len() == 1; + let mut has_root_project = false; + let mut root_project_id = None; + + for (id, build_data) in &self.project_data { + if is_root_level_source(&build_data.source) { + has_root_project = true; + root_project_id = Some(id.to_owned()); + break; + } + } + + self.repo_type = match (single_project, has_root_project) { + (true, true) => RepoType::Polyrepo, + (false, true) => RepoType::MonorepoWithRoot, + (false, false) | (true, false) => RepoType::Monorepo, + }; + + if self.repo_type == RepoType::MonorepoWithRoot { + self.root_project_id = root_project_id; + } + + Ok(()) + } + + /// Enforce project constraints and boundaries after all nodes have been inserted. + #[instrument(skip_all)] + fn enforce_constraints(&self) -> miette::Result<()> { + debug!("Enforcing project constraints"); + + let context = self.context(); + let type_relationships = context + .workspace_config + .constraints + .enforce_project_type_relationships; + let tag_relationships = &context.workspace_config.constraints.tag_relationships; + + if !type_relationships && tag_relationships.is_empty() { + return Ok(()); + } + + let default_scope = DependencyScope::Build; + + for (project_index, project) in self.project_graph.node_references() { + let deps: Vec<_> = self + .project_graph + .neighbors_directed(project_index, Direction::Outgoing) + .flat_map(|dep_index| { + self.project_graph.node_weight(dep_index).map(|dep| { + ( + dep, + // Is this safe? + self.project_graph + .find_edge(project_index, dep_index) + .and_then(|ei| self.project_graph.edge_weight(ei)) + .unwrap_or(&default_scope), + ) + }) + }) + .collect(); + + for (dep, dep_scope) in deps { + if type_relationships { + enforce_project_type_relationships(project, dep, dep_scope)?; + } + + for (source_tag, required_tags) in tag_relationships { + enforce_tag_relationships(project, source_tag, dep, required_tags)?; + } + } + } + + Ok(()) + } + + /// When caching the graph, we must hash all project and workspace + /// config files that are required to invalidate the cache. + async fn hash_required_configs( + &self, + ) -> miette::Result> { + let context = self.context(); + let config_names = context.config_loader.get_project_file_names(); + let mut configs = vec![]; + + // Hash all project-level config files + for build_data in self.project_data.values() { + for name in &config_names { + configs.push(build_data.source.join(name).to_string()); + } + } + + // Hash all workspace-level config files + for file in glob::walk( + context.workspace_root.join(consts::CONFIG_DIRNAME), + ["*.pkl", "tasks/**/*.pkl", "*.yml", "tasks/**/*.yml"], + )? { + configs.push(to_virtual_string( + file.strip_prefix(context.workspace_root).unwrap(), + )?); + } + + context + .vcs + .as_ref() + .expect("VCS required!") + .get_file_hashes(&configs, true, 500) + .await + } + + /// Preload the graph with project sources from the workspace configuration. + /// If globs are provided, walk the file system and gather sources. + /// Then extend the graph with aliases, derived from all event subscribers. + async fn preload_build_data(&mut self) -> miette::Result<()> { + let context = self.context(); + let mut globs = vec![]; + let mut sources = vec![]; + + // Gather all project sources + let mut add_sources = |map: &FxHashMap| { + for (id, source) in map { + sources.push((id.to_owned(), WorkspaceRelativePathBuf::from(source))); + } + }; + + match &context.workspace_config.projects { + WorkspaceProjects::Sources(map) => { + add_sources(map); + } + WorkspaceProjects::Globs(list) => { + globs.extend(list); + } + WorkspaceProjects::Both(cfg) => { + globs.extend(&cfg.globs); + add_sources(&cfg.sources); + } + }; + + if !sources.is_empty() { + debug!( + sources = ?sources, + "Using configured project sources", + ); + } + + if !globs.is_empty() { + debug!( + globs = ?globs, + "Locating projects with globs", + ); + + locate_projects_with_globs(&context, &globs, &mut sources)?; + } + + // Load projects and configs first + self.load_project_build_data(sources)?; + + // Then load aliases and extend projects + self.load_project_aliases().await?; + + Ok(()) + } + + async fn load_project_aliases(&mut self) -> miette::Result<()> { + let context = self.context(); + + debug!("Extending project graph with aliases"); + + let aliases = context + .extend_project_graph + .emit(ExtendProjectGraphEvent { + sources: self + .project_data + .iter() + .map(|(id, build_data)| (id.to_owned(), build_data.source.to_owned())) + .collect(), + workspace_root: context.workspace_root.to_owned(), + }) + .await? + .aliases; + + let mut dupe_aliases = FxHashMap::::default(); + + for (id, alias) in aliases { + let id = self.renamed_project_ids.get(&id).unwrap_or(&id); + + // Skip aliases that match its own ID + if id == &alias { + continue; + } + + // Skip aliases that would override an ID + if self.project_data.contains_key(alias.as_str()) { + debug!( + "Skipping alias {} for project {} as it conflicts with the existing project {}", + color::label(&alias), + color::id(id), + color::id(&alias), + ); + + continue; + } + + if let Some(existing_id) = dupe_aliases.get(&alias) { + // Skip if the existing ID is already for this ID. + // This scenario is possible when multiple platforms + // extract the same aliases (Bun vs Node, etc). + if existing_id == id { + continue; + } + + return Err(WorkspaceBuilderError::DuplicateProjectAlias { + alias: alias.clone(), + old_id: existing_id.to_owned(), + new_id: id.clone(), + } + .into()); + } + + dupe_aliases.insert(alias.clone(), id.to_owned()); + + self.project_data + .get_mut(id) + .expect("Project build data not found!") + .alias = Some(alias); + } + + Ok(()) + } + + fn load_project_build_data(&mut self, sources: ProjectsSourcesList) -> miette::Result<()> { + let context = self.context(); + let config_label = context.config_loader.get_debug_label("moon", false); + let mut project_data: FxHashMap = FxHashMap::default(); + let mut renamed_ids = FxHashMap::default(); + + debug!("Loading projects"); + + for (mut id, source) in sources { + trace!( + project_id = id.as_str(), + "Attempting to load {} (optional)", + color::file(source.join(&config_label)) + ); + + let config = context + .config_loader + .load_project_config_from_source(context.workspace_root, &source)?; + + let mut build_data = ProjectBuildData { + source, + ..Default::default() + }; + + // Track ID renames + if let Some(new_id) = &config.id { + if new_id != &id { + build_data.original_id = Some(id.clone()); + renamed_ids.insert(id.clone(), new_id.to_owned()); + id = new_id.to_owned(); + } + } + + // Check for duplicate IDs + if let Some(existing_data) = project_data.get(&id) { + if existing_data.source != build_data.source { + return Err(WorkspaceBuilderError::DuplicateProjectId { + id: id.clone(), + old_source: existing_data.source.to_string(), + new_source: build_data.source.to_string(), + } + .into()); + } + } + + // Otherwise persist the build data + build_data.config = Some(config); + project_data.insert(id, build_data); + } + + debug!("Loaded {} projects", project_data.len()); + + self.project_data.extend(project_data); + self.renamed_project_ids.extend(renamed_ids); + + Ok(()) + } + + fn resolve_project_id(&self, id_or_alias: &str) -> Id { + let id = if self.project_data.contains_key(id_or_alias) { + Id::raw(id_or_alias) + } else { + match self.project_data.iter().find_map(|(id, build_data)| { + if build_data + .alias + .as_ref() + .is_some_and(|alias| alias == id_or_alias) + { + Some(id) + } else { + None + } + }) { + Some(project_id) => project_id.to_owned(), + None => Id::raw(id_or_alias), + } + }; + + match self.renamed_project_ids.get(&id) { + Some(new_id) => new_id.to_owned(), + None => id, + } + } + + fn context(&self) -> Arc> { + Arc::clone( + self.context + .as_ref() + .expect("Missing workspace builder context!"), + ) + } +} diff --git a/crates/workspace/src/workspace_builder_error.rs b/crates/workspace/src/workspace_builder_error.rs new file mode 100644 index 00000000000..bab5aa163a6 --- /dev/null +++ b/crates/workspace/src/workspace_builder_error.rs @@ -0,0 +1,38 @@ +use miette::Diagnostic; +use moon_common::{Id, Style, Stylize}; +use thiserror::Error; + +#[derive(Error, Debug, Diagnostic)] +pub enum WorkspaceBuilderError { + #[diagnostic(code(project_graph::duplicate_alias))] + #[error( + "Project {} is already using the alias {}, unable to use the alias for project {}.\nTry changing the alias to something unique to move forward.", + .old_id.style(Style::Id), + .alias.style(Style::Label), + .new_id.style(Style::Id), + )] + DuplicateProjectAlias { + alias: String, + old_id: Id, + new_id: Id, + }, + + #[diagnostic(code(project_graph::duplicate_id))] + #[error( + "A project already exists with the identifier {} (existing source {}, new source {}).\nTry renaming the project folder to make it unique, or configure the {} setting in {}.", + .id.style(Style::Id), + .old_source.style(Style::File), + .new_source.style(Style::File), + "id".style(Style::Property), + "moon.yml".style(Style::File) + )] + DuplicateProjectId { + id: Id, + old_source: String, + new_source: String, + }, + + #[diagnostic(code(project_graph::missing_source))] + #[error("No project exists at source path {}.", .0.style(Style::File))] + MissingProjectAtSource(String), +} diff --git a/crates/project-graph/src/project_graph_hash.rs b/crates/workspace/src/workspace_cache.rs similarity index 66% rename from crates/project-graph/src/project_graph_hash.rs rename to crates/workspace/src/workspace_cache.rs index 9a67056eef2..2b34e81a4c1 100644 --- a/crates/project-graph/src/project_graph_hash.rs +++ b/crates/workspace/src/workspace_cache.rs @@ -1,3 +1,5 @@ +use crate::project_build_data::ProjectBuildData; +use moon_cache::cache_item; use moon_common::path::WorkspaceRelativePathBuf; use moon_common::{is_docker, Id}; use moon_hash::hash_content; @@ -5,11 +7,17 @@ use rustc_hash::FxHashMap; use std::collections::BTreeMap; use std::env; +cache_item!( + pub struct WorkspaceProjectsCacheState { + pub last_hash: String, + pub projects: FxHashMap, + } +); + hash_content!( - pub struct ProjectGraphHash<'graph> { - // Data derived from the project graph builder. - aliases: BTreeMap<&'graph Id, &'graph String>, - sources: BTreeMap<&'graph Id, &'graph WorkspaceRelativePathBuf>, + pub struct WorkspaceGraphHash<'graph> { + // Data derived from the workspace graph builder. + projects: BTreeMap<&'graph Id, &'graph ProjectBuildData>, // Project and workspace configs required for cache invalidation. configs: BTreeMap, @@ -17,7 +25,7 @@ hash_content!( // Environment variables required for cache invalidation. env: BTreeMap, - // The project graph stores absolute file paths, which breaks moon when + // The graph stores absolute file paths, which breaks moon when // running tasks inside and outside of a container at the same time. // This flag helps to continuously bust the cache. in_docker: bool, @@ -30,30 +38,27 @@ hash_content!( } ); -impl<'cfg> ProjectGraphHash<'cfg> { - pub fn new() -> Self { - ProjectGraphHash { - aliases: BTreeMap::default(), - sources: BTreeMap::default(), +impl<'graph> Default for WorkspaceGraphHash<'graph> { + fn default() -> Self { + WorkspaceGraphHash { + projects: BTreeMap::default(), configs: BTreeMap::default(), env: BTreeMap::default(), in_docker: is_docker(), version: env::var("MOON_VERSION").unwrap_or_default(), } } +} - pub fn add_aliases(&mut self, aliases: &'cfg FxHashMap) { - self.aliases.extend(aliases.iter()); +impl<'graph> WorkspaceGraphHash<'graph> { + pub fn add_projects(&mut self, projects: &'graph FxHashMap) { + self.projects.extend(projects.iter()); } pub fn add_configs(&mut self, configs: BTreeMap) { self.configs.extend(configs); } - pub fn add_sources(&mut self, sources: &'cfg FxHashMap) { - self.sources.extend(sources.iter()); - } - pub fn gather_env(&mut self) { for key in [ // Task options From b50cae4c1f94306b14c89737beebc3540fa8b24b Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:35:34 +0100 Subject: [PATCH 08/28] Add Tests and re-work based on discussions --- crates/cli/tests/run_python_test.rs | 63 ++++++++++ ...thon_test__runs_install_deps_via_args.snap | 12 ++ ...run_python_test__runs_standard_script.snap | 11 ++ crates/config/src/language_platform.rs | 1 + crates/config/src/toolchain/python_config.rs | 9 +- .../inheritance/files/tasks/python.yml | 3 + .../tests/inherited_tasks_config_test.rs | 33 ++++++ crates/config/tests/toolchain_config_test.rs | 1 + .../__fixtures__/builder/platforms/moon.yml | 4 + crates/toolchain/src/detect/task_platform.rs | 11 ++ legacy/core/test-utils/src/configs.rs | 42 +++++++ .../platform/src/actions/install_deps.rs | 101 ++++------------ legacy/python/platform/src/actions/mod.rs | 2 + .../python/platform/src/actions/setup_tool.rs | 50 ++++++++ legacy/python/platform/src/python_platform.rs | 25 ++-- legacy/python/tool/src/python_tool.rs | 112 +----------------- packages/types/src/toolchain-config.ts | 4 +- tests/fixtures/python/base/moon.yml | 12 ++ website/static/schemas/toolchain.json | 2 +- 19 files changed, 285 insertions(+), 213 deletions(-) create mode 100644 crates/cli/tests/run_python_test.rs create mode 100644 crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap create mode 100644 crates/cli/tests/snapshots/run_python_test__runs_standard_script.snap create mode 100644 crates/config/tests/__fixtures__/inheritance/files/tasks/python.yml create mode 100644 legacy/python/platform/src/actions/setup_tool.rs create mode 100644 tests/fixtures/python/base/moon.yml diff --git a/crates/cli/tests/run_python_test.rs b/crates/cli/tests/run_python_test.rs new file mode 100644 index 00000000000..ac82f32816c --- /dev/null +++ b/crates/cli/tests/run_python_test.rs @@ -0,0 +1,63 @@ +use moon_config::{PartialPipConfig, PartialPythonConfig}; +use moon_test_utils::{ + assert_snapshot, create_sandbox_with_config, get_python_fixture_configs, Sandbox, +}; +use proto_core::UnresolvedVersionSpec; + +fn python_sandbox(config: PartialPythonConfig) -> Sandbox { + python_sandbox_with_config(|_| {}, config) +} + +fn python_sandbox_with_config(callback: C, config: PartialPythonConfig) -> Sandbox +where + C: FnOnce(&mut PartialPythonConfig), +{ + let (workspace_config, mut toolchain_config, tasks_config) = get_python_fixture_configs(); + + toolchain_config.python = Some(config); + + if let Some(python_config) = &mut toolchain_config.python { + callback(python_config); + } + + let sandbox = create_sandbox_with_config( + "python", + Some(workspace_config), + Some(toolchain_config), + Some(tasks_config), + ); + + sandbox.enable_git(); + sandbox +} + +#[test] +fn runs_standard_script() { + let sandbox = python_sandbox(PartialPythonConfig { + version: Some(UnresolvedVersionSpec::parse("3.11.10").unwrap()), + ..PartialPythonConfig::default() + }); + let assert = sandbox.run_moon(|cmd| { + cmd.arg("run").arg("python:standard"); + }); + + assert_snapshot!(assert.output()); +} + +#[test] +fn runs_install_deps_via_args() { + let sandbox = python_sandbox(PartialPythonConfig { + version: Some(UnresolvedVersionSpec::parse("3.11.10").unwrap()), + pip: Some(PartialPipConfig { + version: Some(UnresolvedVersionSpec::parse("latest").unwrap()), + install_args: Some(vec!["poetry==1.8.4".to_string()]), + ..PartialPipConfig::default() + }), + ..PartialPythonConfig::default() + }); + let assert = sandbox.run_moon(|cmd| { + cmd.arg("run").arg("python:poetry"); + }); + + assert_snapshot!(assert.output()); +} diff --git a/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap b/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap new file mode 100644 index 00000000000..3f7391f5497 --- /dev/null +++ b/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap @@ -0,0 +1,12 @@ +--- +source: crates/cli/tests/run_python_test.rs +assertion_line: 62 +expression: assert.output() +--- +▪▪▪▪ Install pip dependencies from install args +▪▪▪▪ python:poetry +Poetry (version 1.8.4) +▪▪▪▪ python:poetry (100ms) + +Tasks: 1 completed + Time: 100ms diff --git a/crates/cli/tests/snapshots/run_python_test__runs_standard_script.snap b/crates/cli/tests/snapshots/run_python_test__runs_standard_script.snap new file mode 100644 index 00000000000..5873dd9d926 --- /dev/null +++ b/crates/cli/tests/snapshots/run_python_test__runs_standard_script.snap @@ -0,0 +1,11 @@ +--- +source: crates/cli/tests/run_python_test.rs +assertion_line: 38 +expression: assert.output() +--- +▪▪▪▪ python:standard +Python 3.11.10 +▪▪▪▪ python:standard (100ms) + +Tasks: 1 completed + Time: 100ms diff --git a/crates/config/src/language_platform.rs b/crates/config/src/language_platform.rs index 2c9377b7fd7..d291c8044a8 100644 --- a/crates/config/src/language_platform.rs +++ b/crates/config/src/language_platform.rs @@ -104,6 +104,7 @@ mod tests { assert_eq!(LanguageType::Go.to_string(), "go"); assert_eq!(LanguageType::JavaScript.to_string(), "javascript"); assert_eq!(LanguageType::Ruby.to_string(), "ruby"); + assert_eq!(LanguageType::Python.to_string(), "python"); assert_eq!(LanguageType::Unknown.to_string(), "unknown"); assert_eq!(LanguageType::Other(Id::raw("dotnet")).to_string(), "dotnet"); } diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs index 1dd8d8577f9..74eff708e6f 100644 --- a/crates/config/src/toolchain/python_config.rs +++ b/crates/config/src/toolchain/python_config.rs @@ -9,7 +9,7 @@ pub struct PipConfig { /// List of arguments to append to `pip install` commands. pub install_args: Option>, - /// The version of pip to download, install, and run `pip` tasks with. + /// The version of pip to download, install, and run `pip` tasks with. pub version: Option, } @@ -18,15 +18,10 @@ pub struct PythonConfig { /// Location of the WASM plugin to use for Python support. pub plugin: Option, - /// Options for pnpm, when used as a package manager. + /// Options for pip, when used as a package manager. #[setting(nested)] pub pip: Option, - /// The relative root of the virtual environment workspace. Default to moon's - /// workspace root - #[setting(default = ".", skip)] - pub venv_root: String, - #[setting(default = ".venv", skip)] pub venv_name: String, diff --git a/crates/config/tests/__fixtures__/inheritance/files/tasks/python.yml b/crates/config/tests/__fixtures__/inheritance/files/tasks/python.yml new file mode 100644 index 00000000000..02c17dc9145 --- /dev/null +++ b/crates/config/tests/__fixtures__/inheritance/files/tasks/python.yml @@ -0,0 +1,3 @@ +tasks: + python: + command: python diff --git a/crates/config/tests/inherited_tasks_config_test.rs b/crates/config/tests/inherited_tasks_config_test.rs index 90c37987536..d7aa6ce5dc4 100644 --- a/crates/config/tests/inherited_tasks_config_test.rs +++ b/crates/config/tests/inherited_tasks_config_test.rs @@ -524,6 +524,7 @@ mod task_manager { "javascript-tool", "kotlin", "node", + "python", "node-application", "node-library", "rust", @@ -792,6 +793,38 @@ mod task_manager { ); } + #[test] + fn creates_python_config() { + let sandbox = create_sandbox("inheritance/files"); + let manager = load_manager_from_root(sandbox.path(), sandbox.path()).unwrap(); + + let config = manager + .get_inherited_config( + &PlatformType::System, + &LanguageType::Python, + &StackType::Frontend, + &ProjectType::Library, + &[], + ) + .unwrap(); + + assert_eq!( + config.config.tasks, + BTreeMap::from_iter([ + ( + Id::raw("global"), + stub_task("global", PlatformType::Unknown) + ), + (Id::raw("python"), stub_task("python", PlatformType::System)), + ]), + ); + + assert_eq!( + config.layers.keys().collect::>(), + vec!["tasks.yml", "tasks/python.yml",] + ); + } + #[test] fn creates_js_config_via_bun() { let sandbox = create_sandbox("inheritance/files"); diff --git a/crates/config/tests/toolchain_config_test.rs b/crates/config/tests/toolchain_config_test.rs index 8ad3a30d673..c3c898908cd 100644 --- a/crates/config/tests/toolchain_config_test.rs +++ b/crates/config/tests/toolchain_config_test.rs @@ -48,6 +48,7 @@ mod toolchain_config { assert!(config.deno.is_none()); assert!(config.node.is_none()); + assert!(config.python.is_none()); assert!(config.rust.is_none()); assert!(config.typescript.is_none()); } diff --git a/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml b/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml index fb3ad2ba314..91935a27e17 100644 --- a/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml +++ b/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml @@ -9,6 +9,10 @@ tasks: command: bun deno-via-cmd: command: deno + python: + platform: python + python-via-cmd: + command: python node: platform: node node-via-cmd: diff --git a/crates/toolchain/src/detect/task_platform.rs b/crates/toolchain/src/detect/task_platform.rs index 30df9a79a6e..d294942854d 100644 --- a/crates/toolchain/src/detect/task_platform.rs +++ b/crates/toolchain/src/detect/task_platform.rs @@ -4,6 +4,7 @@ use std::sync::OnceLock; pub static BUN_COMMANDS: OnceLock = OnceLock::new(); pub static DENO_COMMANDS: OnceLock = OnceLock::new(); +pub static PYTHON_COMMANDS: OnceLock = OnceLock::new(); pub static RUST_COMMANDS: OnceLock = OnceLock::new(); pub static NODE_COMMANDS: OnceLock = OnceLock::new(); pub static UNIX_SYSTEM_COMMANDS: OnceLock = OnceLock::new(); @@ -17,6 +18,9 @@ fn use_platform_if_enabled( PlatformType::Bun if enabled_platforms.contains(&PlatformType::Bun) => return platform, PlatformType::Deno if enabled_platforms.contains(&PlatformType::Deno) => return platform, PlatformType::Node if enabled_platforms.contains(&PlatformType::Node) => return platform, + PlatformType::Python if enabled_platforms.contains(&PlatformType::Python) => { + return platform + } PlatformType::Rust if enabled_platforms.contains(&PlatformType::Rust) => return platform, _ => {} }; @@ -55,6 +59,13 @@ pub fn detect_task_platform(command: &str, enabled_platforms: &[PlatformType]) - return use_platform_if_enabled(PlatformType::Deno, enabled_platforms); } + if PYTHON_COMMANDS + .get_or_init(|| Regex::new("^(python|python3|pip|pip3)$").unwrap()) + .is_match(command) + { + return use_platform_if_enabled(PlatformType::Python, enabled_platforms); + } + if RUST_COMMANDS .get_or_init(|| Regex::new("^(rust-|rustc|rustdoc|rustfmt|rustup|cargo)").unwrap()) .is_match(command) diff --git a/legacy/core/test-utils/src/configs.rs b/legacy/core/test-utils/src/configs.rs index 7e5fe2ab892..1cb626f1612 100644 --- a/legacy/core/test-utils/src/configs.rs +++ b/legacy/core/test-utils/src/configs.rs @@ -484,6 +484,48 @@ pub fn get_node_fixture_configs() -> ( (workspace_config, toolchain_config, tasks_config) } +pub fn get_python_fixture_configs() -> ( + PartialWorkspaceConfig, + PartialToolchainConfig, + PartialInheritedTasksConfig, +) { + let workspace_config = PartialWorkspaceConfig { + projects: Some(PartialWorkspaceProjects::Sources(FxHashMap::from_iter([ + ("python".try_into().unwrap(), "base".to_owned()), + ]))), + ..PartialWorkspaceConfig::default() + }; + + let mut toolchain_config = get_default_toolchain(); + toolchain_config.python = Some(PartialPythonConfig { + version: Some(UnresolvedVersionSpec::parse("3.11.10").unwrap()), + ..PartialPythonConfig::default() + }); + + let tasks_config = PartialInheritedTasksConfig { + tasks: Some(BTreeMap::from_iter([ + ( + "version".try_into().unwrap(), + PartialTaskConfig { + command: Some(PartialTaskArgs::String("python".into())), + args: Some(PartialTaskArgs::String("--version".into())), + ..PartialTaskConfig::default() + }, + ), + ( + "noop".try_into().unwrap(), + PartialTaskConfig { + command: Some(PartialTaskArgs::String("noop".into())), + ..PartialTaskConfig::default() + }, + ), + ])), + ..PartialInheritedTasksConfig::default() + }; + + (workspace_config, toolchain_config, tasks_config) +} + pub fn get_node_depman_fixture_configs( depman: &str, ) -> ( diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs index 8c2c20129c5..180d0df5ac4 100644 --- a/legacy/python/platform/src/actions/install_deps.rs +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -1,4 +1,5 @@ use moon_action::Operation; +use moon_common::color; use moon_console::{Checkpoint, Console}; use moon_python_tool::PythonTool; use moon_utils::get_workspace_root; @@ -13,97 +14,43 @@ pub async fn install_deps( ) -> miette::Result> { let mut operations = vec![]; - // python.exec_python(args, working_dir) - if let Some(pip_config) = &python.config.pip { - // Very first step: Activate virtual environment - console - .out - .print_checkpoint(Checkpoint::Setup, format!("activate virtual environment"))?; - let virtual_environment = &get_workspace_root().join(python.config.venv_name.clone()); - - if !virtual_environment.exists() { - let args = vec![ - "-m", - "venv", - virtual_environment.as_os_str().to_str().unwrap(), - ]; - operations.push( - Operation::task_execution(format!("python {} ", args.join(" "))) - .track_async(|| python.exec_python(args, working_dir)) - .await?, - ); - } - - if let Some(pip_version) = &pip_config.version { - console - .out - .print_checkpoint(Checkpoint::Setup, format!("install pip {pip_version}"))?; - - let p_version: String = if pip_version.is_latest() { - format!("pip") - } else { - format!( - "pip{}", - pip_version.to_owned().to_string().replace("~", "~=") - ) - }; - let args = vec!["-m", "pip", "install", "-U", &p_version]; - // #"--quiet", - operations.push( - Operation::task_execution(format!(" {} ", args.join(" "))) - .track_async(|| python.exec_python(args, working_dir)) - .await?, - ); - } - let requirements_path = find_requirements_txt(working_dir, &get_workspace_root()); if let Some(install_args) = &pip_config.install_args { - if install_args.iter().any(|x| !x.starts_with("-")) { - if let Some(_) = requirements_path { - console.out.print_checkpoint( - Checkpoint::Setup, - "Skip installation via install args, found requirements.txt additional.", - )?; - } else { - console.out.print_checkpoint( - Checkpoint::Setup, - "pip dependencies from install args", - )?; - - let mut args = vec!["-m", "pip", "install"]; - if pip_config.install_args.is_some() { - args.extend( - pip_config - .install_args - .as_ref() - .unwrap() - .iter() - .map(|c| c.as_str()), - ); - } - - operations.push( - Operation::task_execution(format!(" {}", args.join(" "))) - .track_async(|| python.exec_python(args, working_dir)) - .await?, + if install_args.iter().any(|x| !x.starts_with("-")) && requirements_path.is_none() { + console.out.print_checkpoint( + Checkpoint::Setup, + "Install pip dependencies from install args", + )?; + + let mut args = vec!["-m", "pip", "install", "--quiet"]; + if pip_config.install_args.is_some() { + args.extend( + pip_config + .install_args + .as_ref() + .unwrap() + .iter() + .map(|c| c.as_str()), ); } + + operations.push( + Operation::task_execution(format!(" {}", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); } } if let Some(req) = requirements_path { console.out.print_checkpoint( Checkpoint::Setup, - format!( - "pip dependencies from {}", - req.as_os_str().to_str().unwrap() - ), + format!("Install pip dependencies from {}", color::path(&req)), )?; - let mut args = vec!["-m", "pip", "install"]; - // #, "--quiet" + let mut args = vec!["-m", "pip", "install", "--quiet"]; if pip_config.install_args.is_some() { args.extend( pip_config diff --git a/legacy/python/platform/src/actions/mod.rs b/legacy/python/platform/src/actions/mod.rs index 31a65cfdfdf..b4bae854747 100644 --- a/legacy/python/platform/src/actions/mod.rs +++ b/legacy/python/platform/src/actions/mod.rs @@ -1,3 +1,5 @@ mod install_deps; +mod setup_tool; pub use install_deps::*; +pub use setup_tool::*; diff --git a/legacy/python/platform/src/actions/setup_tool.rs b/legacy/python/platform/src/actions/setup_tool.rs new file mode 100644 index 00000000000..cdb4281dab4 --- /dev/null +++ b/legacy/python/platform/src/actions/setup_tool.rs @@ -0,0 +1,50 @@ +use moon_action::Operation; +use moon_python_tool::PythonTool; +use starbase_utils::fs; +use std::path::Path; + +pub async fn setup_tool(python: &PythonTool, workspace_root: &Path) -> miette::Result<()> { + let mut operations = vec![]; + + if let Some(pip_config) = &python.config.pip { + let virtual_environment = &workspace_root.join(python.config.venv_name.clone()); + + if !virtual_environment.exists() { + let args = vec![ + "-m", + "venv", + virtual_environment.as_os_str().to_str().unwrap(), + ]; + operations.push( + Operation::task_execution(format!("python {} ", args.join(" "))) + .track_async(|| python.exec_python(args, workspace_root)) + .await?, + ); + } + + if let Some(pip_version) = &pip_config.version { + let p_version: String = if pip_version.is_latest() { + format!("pip") + } else { + format!( + "pip{}", + pip_version.to_owned().to_string().replace("~", "~=") + ) + }; + let args = vec!["-m", "pip", "install", "--quiet", "-U", &p_version]; + operations.push( + Operation::task_execution(format!(" {} ", args.join(" "))) + .track_async(|| python.exec_python(args, workspace_root)) + .await?, + ); + } + } + + // Create version file + if let Some(python_version) = &python.config.version { + let rc_path = workspace_root.join(".python-version".to_string()); + fs::write_file(&rc_path, python_version.to_string())?; + } + + Ok(()) +} diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 45692908179..c752dc171fb 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -8,7 +8,7 @@ use moon_config::{ }; use moon_console::Console; use moon_hash::ContentHasher; -// use moon_logger::debug; +use moon_logger::info; use moon_platform::{Platform, Runtime, RuntimeReq}; use moon_process::Command; use moon_project::Project; @@ -26,8 +26,6 @@ use std::{ }; use tracing::instrument; -// const LOG_TARGET: &str = "moon:python-platform"; - pub struct PythonPlatform { pub config: PythonConfig, @@ -101,7 +99,7 @@ impl Platform for PythonPlatform { // PROJECT GRAPH fn is_project_in_dependency_workspace(&self, _project_source: &str) -> miette::Result { - // Single version policy / only a root package.json + // Single version policy / only a root requirements.txt Ok(true) } @@ -111,12 +109,7 @@ impl Platform for PythonPlatform { _projects_list: &ProjectsSourcesList, _aliases_list: &mut ProjectsAliasesList, ) -> miette::Result<()> { - // Extract the alias from the Cargo project relative to the lockfile - // for (id, source) in projects_list { - // let project_root = source.to_path(&self.workspace_root); - - // } - + // Not supported Ok(()) } @@ -168,11 +161,6 @@ impl Platform for PythonPlatform { self.toolchain.setup(&req, &mut last_versions).await?; - // info!( - // target: LOG_TARGET, - // "Setup toolchain" - // ); - Ok(()) } @@ -205,7 +193,12 @@ impl Platform for PythonPlatform { .await?, ); } - Ok(self.toolchain.setup(req, last_versions).await?) + + let installed = self.toolchain.setup(req, last_versions).await?; + + actions::setup_tool(self.toolchain.get_for_version(req)?, &self.workspace_root).await?; + + Ok(installed) } #[instrument(skip_all)] diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs index fc6ce537131..1a13ea556b7 100644 --- a/legacy/python/tool/src/python_tool.rs +++ b/legacy/python/tool/src/python_tool.rs @@ -1,13 +1,10 @@ use moon_config::PythonConfig; use moon_console::{Checkpoint, Console}; -use moon_python_lang::pip_requirements::load_lockfile_dependencies; -use moon_python_lang::LockfileDependencyVersions; -// use moon_logger::debug; use moon_logger::{debug, map_list}; use moon_process::Command; use moon_tool::{ - async_trait, get_proto_env_vars, get_proto_paths, get_proto_version_env, load_tool_plugin, - prepend_path_env_var, use_global_tool_on_path, DependencyManager, Tool, + async_trait, get_proto_env_vars, get_proto_paths, load_tool_plugin, prepend_path_env_var, + use_global_tool_on_path, Tool, }; use moon_toolchain::RuntimeReq; use moon_utils::get_workspace_root; @@ -15,8 +12,6 @@ use proto_core::flow::install::InstallOptions; use proto_core::{Id, ProtoEnvironment, Tool as ProtoTool, UnresolvedVersionSpec}; use rustc_hash::FxHashMap; use starbase_styles::color; -use starbase_utils::fs; -use std::env; use std::path::PathBuf; use std::sync::Arc; use std::{ffi::OsStr, path::Path}; @@ -174,106 +169,3 @@ impl Tool for PythonTool { Ok(()) } } - -#[async_trait] -impl DependencyManager for PythonTool { - fn create_command(&self, python: &PythonTool) -> miette::Result { - let mut cmd = Command::new("python"); - cmd.with_console(self.console.clone()); - cmd.envs(get_proto_env_vars()); - if !self.global { - cmd.env("PATH", prepend_path_env_var(get_python_tool_paths(&self))); - } - - if let Some(version) = get_proto_version_env(&self.tool) { - cmd.env("PROTO_PYTHON_VERSION", version); - } - - if let Some(version) = get_proto_version_env(&python.tool) { - cmd.env("PROTO_PYTHON_VERSION", version); - } - - Ok(cmd) - } - - #[instrument(skip_all)] - async fn dedupe_dependencies( - &self, - _python: &PythonTool, - _working_dir: &Path, - _log: bool, - ) -> miette::Result<()> { - // Not supported! - - Ok(()) - } - - fn get_lock_filename(&self) -> String { - String::from("requirements.txt") - } - - fn get_manifest_filename(&self) -> String { - String::from("requirements.txt") - } - - #[instrument(skip_all)] - async fn get_resolved_dependencies( - &self, - project_root: &Path, - ) -> miette::Result { - let Some(lockfile_path) = - fs::find_upwards_until("requirements.txt", project_root, get_workspace_root()) - else { - return Ok(FxHashMap::default()); - }; - - Ok(load_lockfile_dependencies(lockfile_path)?) - } - #[instrument(skip_all)] - async fn install_dependencies( - &self, - python: &PythonTool, - working_dir: &Path, - log: bool, - ) -> miette::Result<()> { - let mut cmd = self.create_command(python)?; - - if let Some(pip_config) = &self.config.pip { - cmd.args(["install"]) - .cwd(working_dir) - .set_print_command(log); - - //TODO: only read from root, but ready for sub virtual environments - if let Some(requirements_path) = fs::find_upwards_until( - "requirements.txt", - get_workspace_root(), - get_workspace_root(), - ) { - cmd.args(["-r", &requirements_path.as_os_str().to_str().unwrap()]); - } - if let Some(install_args) = &pip_config.install_args { - cmd.args(install_args); - } - - let mut cmd = cmd.create_async(); - if env::var("MOON_TEST_HIDE_INSTALL_OUTPUT").is_ok() { - cmd.exec_capture_output().await?; - } else { - cmd.exec_stream_output().await?; - } - } - - Ok(()) - } - - #[instrument(skip_all)] - async fn install_focused_dependencies( - &self, - _python: &PythonTool, - _packages: &[String], - _production_only: bool, - ) -> miette::Result<()> { - // TODO: Implement for docker purposes - Ok(()) - } -} diff --git a/packages/types/src/toolchain-config.ts b/packages/types/src/toolchain-config.ts index b4002fb7f37..9fc148e8255 100644 --- a/packages/types/src/toolchain-config.ts +++ b/packages/types/src/toolchain-config.ts @@ -258,7 +258,7 @@ export interface PipConfig { } export interface PythonConfig { - /** Options for pnpm, when used as a package manager. */ + /** Options for pip, when used as a package manager. */ pip: PipConfig | null; /** Location of the WASM plugin to use for Python support. */ plugin: PluginLocator | null; @@ -637,7 +637,7 @@ export interface PartialPipConfig { } export interface PartialPythonConfig { - /** Options for pnpm, when used as a package manager. */ + /** Options for pip, when used as a package manager. */ pip?: PartialPipConfig | null; /** Location of the WASM plugin to use for Python support. */ plugin?: PluginLocator | null; diff --git a/tests/fixtures/python/base/moon.yml b/tests/fixtures/python/base/moon.yml new file mode 100644 index 00000000000..d9bcbc7a3a3 --- /dev/null +++ b/tests/fixtures/python/base/moon.yml @@ -0,0 +1,12 @@ +language: python + +tasks: + standard: + command: python + args: + - --version + + poetry: + command: poetry + args: + - --version \ No newline at end of file diff --git a/website/static/schemas/toolchain.json b/website/static/schemas/toolchain.json index 2f56f231ab5..8686ed8dcf1 100644 --- a/website/static/schemas/toolchain.json +++ b/website/static/schemas/toolchain.json @@ -665,7 +665,7 @@ "properties": { "pip": { "title": "pip", - "description": "Options for pnpm, when used as a package manager.", + "description": "Options for pip, when used as a package manager.", "anyOf": [ { "$ref": "#/definitions/PipConfig" From 07cda58ef8d3d5041ff374be88748ebf1e9a224c Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:35:18 +0100 Subject: [PATCH 09/28] Add documentation --- CHANGELOG.md | 9 +++ legacy/core/test-utils/src/configs.rs | 7 +- .../setup-toolchain/python/tier2.mdx | 10 ++- .../setup-toolchain/python/tier3.mdx | 12 ++-- website/docs/config/toolchain.mdx | 69 +++++++++++++++++++ 5 files changed, 92 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9096f26e697..71d6e5296af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +- Added Python tier 3 support. + - Will download and install Python into the toolchain when a `version` is configured. + - Will parse the `requirements.txt` to resolve and install dependencies. + - Added a `python.version` setting to `.moon/toolchain.yml`. + - Added a `toolchain.python` setting to `moon.yml`. + - Updated `moon bin` and `moon docker` commands to support Python. + ## 1.29.1 #### 🚀 Updates diff --git a/legacy/core/test-utils/src/configs.rs b/legacy/core/test-utils/src/configs.rs index 1cb626f1612..e06d71aa219 100644 --- a/legacy/core/test-utils/src/configs.rs +++ b/legacy/core/test-utils/src/configs.rs @@ -490,9 +490,10 @@ pub fn get_python_fixture_configs() -> ( PartialInheritedTasksConfig, ) { let workspace_config = PartialWorkspaceConfig { - projects: Some(PartialWorkspaceProjects::Sources(FxHashMap::from_iter([ - ("python".try_into().unwrap(), "base".to_owned()), - ]))), + projects: Some(PartialWorkspaceProjects::Sources(FxHashMap::from_iter([( + "python".try_into().unwrap(), + "base".to_owned(), + )]))), ..PartialWorkspaceConfig::default() }; diff --git a/website/docs/__partials__/setup-toolchain/python/tier2.mdx b/website/docs/__partials__/setup-toolchain/python/tier2.mdx index 21ef77ca7f2..622d4ce24b7 100644 --- a/website/docs/__partials__/setup-toolchain/python/tier2.mdx +++ b/website/docs/__partials__/setup-toolchain/python/tier2.mdx @@ -1,6 +1,4 @@ -:::warning - -Python does not implement tier 2 platform support. However, Python based tasks can still be executed -using the default system platform. - -::: +```yaml title=".moon/toolchain.yml" +python: + pip: {} +``` \ No newline at end of file diff --git a/website/docs/__partials__/setup-toolchain/python/tier3.mdx b/website/docs/__partials__/setup-toolchain/python/tier3.mdx index f95b497d0d6..902b7adb8ea 100644 --- a/website/docs/__partials__/setup-toolchain/python/tier3.mdx +++ b/website/docs/__partials__/setup-toolchain/python/tier3.mdx @@ -1,6 +1,6 @@ -:::warning - -Python does not implement tier 3 tool support. The required `python` and related binaries must exist -on `PATH`. - -::: +```yaml title=".moon/toolchain.yml" +python: + version: '3.11.10' + pip: + version: 'latest' +``` diff --git a/website/docs/config/toolchain.mdx b/website/docs/config/toolchain.mdx index 2324de2734b..0efb7ccae7b 100644 --- a/website/docs/config/toolchain.mdx +++ b/website/docs/config/toolchain.mdx @@ -721,6 +721,75 @@ Both imports can optionally be nested within a `src` directory. > This setting runs _after_ [`syncProjectReferences`](#syncprojectreferences) and will inherit any > synced references from that setting. +## Python + +## `python` + + + +Enables and configures Python. + +### `version` + + + +Defines the explicit Python toolchain +If this field is _not defined_, the global `python` binary will be used. + +```yaml title=".moon/toolchain.yml" {2} +python: + version: '3.11.10' +``` + +> Version can also be defined with [`.prototools`](../proto/config). + +### `venv_name` + + + +Defines the virtual environment name which will be created on workspace root, +project dependencies will be installed into this. Defaults to `.venv` + +```yaml title=".moon/toolchain.yml" {2} +python: + venv_name: '.my-custom-venv' +``` + +### `pip` + + + +#### `version` + + + +The `version` setting defines the explicit pip +[version specification](../concepts/toolchain#version-specification) to use. If this field is _not +defined_, the built-in library will be used. + +A list of plugins that will automatically be imported using `yarn plugin import` (Yarn 2+ only). For +performance reasons, plugins will only be imported when the Yarn version changes. + +```yaml title=".moon/toolchain.yml" {4} +python: + version: '3.11.10' + pip: + version: 'latest' +``` + +### `install_args` + + + +Customize the arguments that will be passed to the pip install command, when the +`InstallDeps` action is triggered in the pipeline. These arguments are used both locally and in CI. + +```yaml title=".moon/toolchain.yml" {3} +python: + pip: + installArgs: ['--trusted-host company.repo.com', '-i https://company.repo.com/simple'] +``` + ## Rust ## `rust` From 290a42a9c96c4ffdbe209355dc59421f0d9f9101 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:37:25 +0100 Subject: [PATCH 10/28] Resolve conflict --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71d6e5296af..468274ce719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,21 @@ - Added a `toolchain.python` setting to `moon.yml`. - Updated `moon bin` and `moon docker` commands to support Python. +## 1.29.2 + +#### 🚀 Updates + +- Removed the warning around `.env` files not existing in certain environments. + +#### 🐞 Fixes + +- Fixed an issue where the task option `timeout` would apply to the overall run, and not for each + attempt when using the `retryCount` option. + +#### ⚙️ Internal + +- Updated Rust to v1.82. + ## 1.29.1 #### 🚀 Updates From d5b59d200d2e5728c8702d79be6640a5770d62d6 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Sat, 12 Oct 2024 16:53:13 +0200 Subject: [PATCH 11/28] Save my work --- Cargo.lock | 71 ++++ crates/app/Cargo.toml | 3 + crates/app/src/systems/analyze.rs | 13 + crates/config/src/language_platform.rs | 1 + crates/config/src/project/overrides_config.rs | 4 + crates/config/src/toolchain/mod.rs | 2 + crates/config/src/toolchain/python_config.rs | 35 ++ crates/config/src/toolchain_config.rs | 18 + .../toolchain/src/detect/project_platform.rs | 7 + legacy/python/lang/Cargo.toml | 23 ++ legacy/python/lang/src/lib.rs | 5 + legacy/python/lang/src/pip_requirements.rs | 15 + legacy/python/platform/Cargo.toml | 36 ++ .../platform/src/actions/install_deps.rs | 78 +++++ legacy/python/platform/src/actions/mod.rs | 3 + legacy/python/platform/src/lib.rs | 16 + legacy/python/platform/src/manifest_hash.rs | 10 + legacy/python/platform/src/python_platform.rs | 320 ++++++++++++++++++ legacy/python/platform/src/target_hash.rs | 18 + legacy/python/platform/src/toolchain_hash.rs | 21 ++ legacy/python/tool/Cargo.toml | 25 ++ legacy/python/tool/src/lib.rs | 3 + legacy/python/tool/src/python_tool.rs | 292 ++++++++++++++++ packages/types/src/project-config.ts | 28 +- packages/types/src/tasks-config.ts | 15 +- packages/types/src/template-config.ts | 12 +- packages/types/src/toolchain-config.ts | 55 ++- packages/types/src/workspace-config.ts | 5 +- website/static/schemas/project.json | 14 + website/static/schemas/tasks.json | 1 + website/static/schemas/toolchain.json | 90 +++++ 31 files changed, 1182 insertions(+), 57 deletions(-) create mode 100644 crates/config/src/toolchain/python_config.rs create mode 100644 legacy/python/lang/Cargo.toml create mode 100644 legacy/python/lang/src/lib.rs create mode 100644 legacy/python/lang/src/pip_requirements.rs create mode 100644 legacy/python/platform/Cargo.toml create mode 100644 legacy/python/platform/src/actions/install_deps.rs create mode 100644 legacy/python/platform/src/actions/mod.rs create mode 100644 legacy/python/platform/src/lib.rs create mode 100644 legacy/python/platform/src/manifest_hash.rs create mode 100644 legacy/python/platform/src/python_platform.rs create mode 100644 legacy/python/platform/src/target_hash.rs create mode 100644 legacy/python/platform/src/toolchain_hash.rs create mode 100644 legacy/python/tool/Cargo.toml create mode 100644 legacy/python/tool/src/lib.rs create mode 100644 legacy/python/tool/src/python_tool.rs diff --git a/Cargo.lock b/Cargo.lock index 4e77999a743..adb503c192f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3016,6 +3016,9 @@ dependencies = [ "moon_plugin", "moon_project", "moon_project_graph", + "moon_python_lang", + "moon_python_platform", + "moon_python_tool", "moon_query", "moon_rust_lang", "moon_rust_platform", @@ -3753,6 +3756,74 @@ dependencies = [ "tracing", ] +[[package]] +name = "moon_python_lang" +version = "0.0.1" +dependencies = [ + "cached", + "cargo-lock", + "cargo_toml", + "miette", + "moon_lang", + "moon_logger", + "moon_test_utils", + "rustc-hash 2.0.0", + "serde", + "starbase_styles", + "starbase_utils", +] + +[[package]] +name = "moon_python_platform" +version = "0.0.1" +dependencies = [ + "miette", + "moon_action", + "moon_action_context", + "moon_common", + "moon_config", + "moon_console", + "moon_hash", + "moon_logger", + "moon_platform", + "moon_process", + "moon_project", + "moon_python_lang", + "moon_python_tool", + "moon_task", + "moon_test_utils", + "moon_tool", + "moon_utils", + "proto_core", + "rustc-hash 2.0.0", + "serde", + "starbase_styles", + "starbase_utils", + "tokio", + "tracing", +] + +[[package]] +name = "moon_python_tool" +version = "0.0.1" +dependencies = [ + "miette", + "moon_common", + "moon_config", + "moon_console", + "moon_logger", + "moon_process", + "moon_python_lang", + "moon_tool", + "moon_toolchain", + "moon_utils", + "proto_core", + "rustc-hash 2.0.0", + "starbase_styles", + "starbase_utils", + "tracing", +] + [[package]] name = "moon_query" version = "0.0.1" diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index e4095646c8e..7eafafdbc12 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -87,6 +87,9 @@ moon_deno_platform = { path = "../../legacy/deno/platform" } moon_node_lang = { path = "../../legacy/node/lang" } moon_node_tool = { path = "../../legacy/node/tool" } moon_node_platform = { path = "../../legacy/node/platform" } +moon_python_lang = { path = "../../legacy/python/lang" } +moon_python_tool = { path = "../../legacy/python/tool" } +moon_python_platform = { path = "../../legacy/python/platform" } moon_rust_lang = { path = "../../legacy/rust/lang" } moon_rust_tool = { path = "../../legacy/rust/tool" } moon_rust_platform = { path = "../../legacy/rust/platform" } diff --git a/crates/app/src/systems/analyze.rs b/crates/app/src/systems/analyze.rs index 391105d4faa..aafc7d8eb00 100644 --- a/crates/app/src/systems/analyze.rs +++ b/crates/app/src/systems/analyze.rs @@ -8,6 +8,7 @@ use moon_console::{Checkpoint, Console}; use moon_deno_platform::DenoPlatform; use moon_node_platform::NodePlatform; use moon_platform::PlatformManager; +use moon_python_platform::PythonPlatform; use moon_rust_platform::RustPlatform; use moon_system_platform::SystemPlatform; use moon_toolchain_plugin::ToolchainRegistry; @@ -172,6 +173,18 @@ pub async fn register_platforms( ); } + if let Some(python_config) = &toolchain_config.python { + registry.register( + PlatformType::Python, + Box::new(PythonPlatform::new( + python_config, + workspace_root, + Arc::clone(proto_env), + Arc::clone(&console), + )), + ); + } + if let Some(rust_config) = &toolchain_config.rust { registry.register( PlatformType::Rust, diff --git a/crates/config/src/language_platform.rs b/crates/config/src/language_platform.rs index c7179e3f572..2c9377b7fd7 100644 --- a/crates/config/src/language_platform.rs +++ b/crates/config/src/language_platform.rs @@ -70,6 +70,7 @@ derive_enum!( Bun, Deno, Node, + Python, Rust, System, #[default] diff --git a/crates/config/src/project/overrides_config.rs b/crates/config/src/project/overrides_config.rs index 0c87bcda69c..ea8982aa8d8 100644 --- a/crates/config/src/project/overrides_config.rs +++ b/crates/config/src/project/overrides_config.rs @@ -48,6 +48,10 @@ cacheable!( /// Overrides `deno` settings. #[setting(nested)] pub deno: Option, + + /// Overrides `python` settings. + #[setting(nested)] + pub python: Option, /// Overrides `node` settings. #[setting(nested)] diff --git a/crates/config/src/toolchain/mod.rs b/crates/config/src/toolchain/mod.rs index 49836e27e9d..33ad220b361 100644 --- a/crates/config/src/toolchain/mod.rs +++ b/crates/config/src/toolchain/mod.rs @@ -3,6 +3,7 @@ mod bun_config; mod deno_config; mod moon_config; mod node_config; +mod python_config; mod rust_config; mod typescript_config; @@ -11,6 +12,7 @@ pub use bun_config::*; pub use deno_config::*; pub use moon_config::*; pub use node_config::*; +pub use python_config::*; pub use rust_config::*; pub use typescript_config::*; diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs new file mode 100644 index 00000000000..5214f52cca1 --- /dev/null +++ b/crates/config/src/toolchain/python_config.rs @@ -0,0 +1,35 @@ +// use super::bin_config::BinEntry; +use schematic::Config; +use serde::Serialize; +use version_spec::UnresolvedVersionSpec; +use warpgate_api::PluginLocator; + + +#[derive(Clone, Config, Debug, PartialEq, Serialize)] +pub struct PipConfig { + /// List of arguments to append to `pip install` commands. + pub install_args: Option>, + + /// The version of pip to download, install, and run `pip` tasks with. + pub version: Option, +} + + +#[derive(Clone, Config, Debug, PartialEq)] +pub struct PythonConfig { + /// Location of the WASM plugin to use for Python support. + pub plugin: Option, + + /// Options for pnpm, when used as a package manager. + #[setting(nested)] + pub pip: Option, + + /// The relative root of the virtual environment workspace. Default to moon's + /// workspace root + #[setting(default = ".", skip)] + pub venv_root: String, + + /// The version of Python to download, install, and run `python` tasks with. + #[setting(env = "MOON_PYTHON_VERSION")] + pub version: Option, +} diff --git a/crates/config/src/toolchain_config.rs b/crates/config/src/toolchain_config.rs index 6ec57d5ade7..42fab4166c1 100644 --- a/crates/config/src/toolchain_config.rs +++ b/crates/config/src/toolchain_config.rs @@ -57,6 +57,10 @@ pub struct ToolchainConfig { #[setting(nested)] pub node: Option, + /// Configures and enables the Python platform. + #[setting(nested)] + pub python: Option, + /// Configures and enables the Rust platform. #[setting(nested)] pub rust: Option, @@ -86,6 +90,10 @@ impl ToolchainConfig { tools.push(PlatformType::Node); } + if self.python.is_some() { + tools.push(PlatformType::Python) + } + if self.rust.is_some() { tools.push(PlatformType::Rust); } @@ -141,6 +149,12 @@ impl ToolchainConfig { } } + if let Some(python_config) = &self.python { + if let Some(version) = &python_config.version { + inject("PROTO_PYTHON_VERSION", version); + } + } + // We don't include Rust since it's a special case! env @@ -155,6 +169,8 @@ impl ToolchainConfig { inherit_tool!(NodeConfig, node, "node", inherit_proto_node); + inherit_tool!(PythonConfig, python, "python", inherit_proto_python); + inherit_tool!(RustConfig, rust, "rust", inherit_proto_rust); inherit_tool_without_version!( @@ -171,6 +187,7 @@ impl ToolchainConfig { is_using_tool_version!(self, node, bun); is_using_tool_version!(self, node, pnpm); is_using_tool_version!(self, node, yarn); + is_using_tool_version!(self, python); is_using_tool_version!(self, rust); // Special case @@ -189,6 +206,7 @@ impl ToolchainConfig { self.inherit_proto_bun(proto_config)?; self.inherit_proto_deno(proto_config)?; self.inherit_proto_node(proto_config)?; + self.inherit_proto_python(proto_config)?; self.inherit_proto_rust(proto_config)?; self.inherit_proto_typescript(proto_config)?; diff --git a/crates/toolchain/src/detect/project_platform.rs b/crates/toolchain/src/detect/project_platform.rs index 163e4ccfcc8..54e1c6cb1b0 100644 --- a/crates/toolchain/src/detect/project_platform.rs +++ b/crates/toolchain/src/detect/project_platform.rs @@ -28,6 +28,13 @@ pub fn detect_project_platform( PlatformType::System } } + LanguageType::Python => { + if enabled_platforms.contains(&PlatformType::Python) { + PlatformType::Python + } else { + PlatformType::System + } + } LanguageType::Rust => { if enabled_platforms.contains(&PlatformType::Rust) { PlatformType::Rust diff --git a/legacy/python/lang/Cargo.toml b/legacy/python/lang/Cargo.toml new file mode 100644 index 00000000000..6074c3866eb --- /dev/null +++ b/legacy/python/lang/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "moon_python_lang" +version = "0.0.1" +edition = "2021" +publish = false + +[dependencies] +moon_lang = { path = "../../core/lang" } +moon_logger = { path = "../../core/logger" } +cached = { workspace = true } +cargo-lock = "9.0.0" +cargo_toml = "0.20.4" +miette = { workspace = true } +rustc-hash = { workspace = true } +serde = { workspace = true } +starbase_styles = { workspace = true } +starbase_utils = { workspace = true, features = ["glob", "toml"] } + +[dev-dependencies] +moon_test_utils = { path = "../../core/test-utils" } + +[lints] +workspace = true diff --git a/legacy/python/lang/src/lib.rs b/legacy/python/lang/src/lib.rs new file mode 100644 index 00000000000..d55284cc8f7 --- /dev/null +++ b/legacy/python/lang/src/lib.rs @@ -0,0 +1,5 @@ +pub mod pip_requirements; + + +pub use pip_requirements::*; +pub use moon_lang::LockfileDependencyVersions; diff --git a/legacy/python/lang/src/pip_requirements.rs b/legacy/python/lang/src/pip_requirements.rs new file mode 100644 index 00000000000..d4c7aa3ea25 --- /dev/null +++ b/legacy/python/lang/src/pip_requirements.rs @@ -0,0 +1,15 @@ +use cached::proc_macro::cached; +use moon_lang::LockfileDependencyVersions; +use rustc_hash::FxHashMap; +use std::path::PathBuf; +use std::fs; + +#[cached(result)] +pub fn load_lockfile_dependencies(path: PathBuf) -> miette::Result { + let mut deps: LockfileDependencyVersions = FxHashMap::default(); + + let lockfile = fs::read_to_string(path.as_path()).expect("Unable to read file"); + let dep = deps.entry(path.display().to_string()).or_default(); + dep.push(lockfile); + Ok(deps) +} diff --git a/legacy/python/platform/Cargo.toml b/legacy/python/platform/Cargo.toml new file mode 100644 index 00000000000..c797eaa1e1b --- /dev/null +++ b/legacy/python/platform/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "moon_python_platform" +version = "0.0.1" +edition = "2021" +publish = false + +[dependencies] +moon_action = { path = "../../../crates/action" } +moon_action_context = { path = "../../../crates/action-context" } +moon_common = { path = "../../../crates/common" } +moon_config = { path = "../../../crates/config" } +moon_console = { path = "../../../crates/console" } +moon_hash = { path = "../../../crates/hash" } +moon_logger = { path = "../../core/logger" } +moon_platform = { path = "../../core/platform" } +moon_process = { path = "../../../crates/process" } +moon_project = { path = "../../../crates/project" } +moon_python_lang = { path = "../lang" } +moon_python_tool = { path = "../tool" } +moon_task = { path = "../../../crates/task" } +moon_tool = { path = "../../core/tool" } +moon_utils = { path = "../../core/utils" } +miette = { workspace = true } +proto_core = { workspace = true } +rustc-hash = { workspace = true } +serde = { workspace = true } +starbase_styles = { workspace = true } +starbase_utils = { workspace = true, features = ["glob"] } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +moon_test_utils = { path = "../../core/test-utils" } + +[lints] +workspace = true diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs new file mode 100644 index 00000000000..a7ed87a496e --- /dev/null +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -0,0 +1,78 @@ +use moon_action::Operation; +use moon_console::{Checkpoint, Console}; +use moon_python_tool::PythonTool; +use moon_utils::get_workspace_root; +use std::path::Path; + +use crate::find_requirements_txt; + +pub async fn install_deps( + python: &PythonTool, + working_dir: &Path, + console: &Console, +) -> miette::Result> { + let mut operations = vec![]; + + if let Some(pip_config) = &python.config.pip { + + + + if let Some(pip_version) = &pip_config.version { + console + .out + .print_checkpoint(Checkpoint::Setup, format!("install pip {pip_version}"))?; + + let p_version: String = if pip_version.is_latest() { + format!("pip") + } else { + format!("pip{}", pip_version.to_owned().to_string().replace("~", "~=")) + }; + let args = vec!["-m", "pip", "install","--quiet", "-U", &p_version]; + + operations.push( + Operation::task_execution(format!("python {} ", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); + } + + console + .out + .print_checkpoint(Checkpoint::Setup, format!("activate virtual environment"))?; + let virtual_environment = &get_workspace_root().join(".venv"); + + if !virtual_environment.exists() { + let args = vec!["-m", "venv", virtual_environment.as_os_str().to_str().unwrap()]; + operations.push( + Operation::task_execution(format!("python {} ", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); + } + + + + + if let Some(req) = find_requirements_txt(working_dir, &get_workspace_root()) { + console + .out + .print_checkpoint(Checkpoint::Setup, format!("pip dependencies from {}", req.as_os_str().to_str().unwrap()))?; + + let mut args = vec!["-m", "pip", "install", "--quiet"]; + + if pip_config.install_args.is_some() { + args.extend(pip_config.install_args.as_ref().unwrap().iter().map(|c| c.as_str())); + } + + args.extend(["-r", req.as_os_str().to_str().unwrap()]); + + operations.push( + Operation::task_execution(format!("python {}", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); + } + } + + Ok(operations) +} \ No newline at end of file diff --git a/legacy/python/platform/src/actions/mod.rs b/legacy/python/platform/src/actions/mod.rs new file mode 100644 index 00000000000..b7b5cf56da6 --- /dev/null +++ b/legacy/python/platform/src/actions/mod.rs @@ -0,0 +1,3 @@ +mod install_deps; + +pub use install_deps::*; \ No newline at end of file diff --git a/legacy/python/platform/src/lib.rs b/legacy/python/platform/src/lib.rs new file mode 100644 index 00000000000..bb487c34931 --- /dev/null +++ b/legacy/python/platform/src/lib.rs @@ -0,0 +1,16 @@ +pub mod actions; +mod manifest_hash; +mod python_platform; +mod target_hash; +mod toolchain_hash; + + +pub use python_platform::*; + +use starbase_utils::fs; +use std::path::{Path, PathBuf}; + +fn find_requirements_txt(starting_dir: &Path, workspace_root: &Path) -> Option { + fs::find_upwards_until("requirements.txt", starting_dir, workspace_root) +} + diff --git a/legacy/python/platform/src/manifest_hash.rs b/legacy/python/platform/src/manifest_hash.rs new file mode 100644 index 00000000000..89acff80801 --- /dev/null +++ b/legacy/python/platform/src/manifest_hash.rs @@ -0,0 +1,10 @@ +use moon_hash::hash_content; +// use moon_python_lang::pipfile::DependencyDetail; +// use std::collections::BTreeMap; + +hash_content!( + pub struct PythonManifestHash { + // pub dependencies: BTreeMap, + pub name: String, + } +); diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs new file mode 100644 index 00000000000..1ef0f635198 --- /dev/null +++ b/legacy/python/platform/src/python_platform.rs @@ -0,0 +1,320 @@ +use crate::{ + actions, find_requirements_txt, target_hash::PythonTargetHash, toolchain_hash::PythonToolchainHash +}; +use moon_action::Operation; +use moon_action_context::ActionContext; +use moon_common::Id; +use moon_config::{ + HasherConfig, PlatformType, ProjectConfig, ProjectsAliasesList, ProjectsSourcesList, + PythonConfig, UnresolvedVersionSpec, +}; +use moon_console::{Checkpoint, Console}; +use moon_hash::ContentHasher; +use moon_logger::debug; +use moon_platform::{Platform, Runtime, RuntimeReq}; +use moon_process::Command; +use moon_project::Project; +use moon_python_lang::pip_requirements::load_lockfile_dependencies; +use moon_python_tool::{get_python_tool_paths, PythonTool}; +use moon_task::Task; +use moon_tool::{get_proto_version_env, prepend_path_env_var, Tool, ToolManager}; +use moon_utils::{async_trait, get_workspace_root}; +use proto_core::ProtoEnvironment; +use rustc_hash::FxHashMap; +use std::{ + collections::BTreeMap, path::{Path, PathBuf}, sync::Arc +}; +use tracing::instrument; + +const LOG_TARGET: &str = "moon:python-platform"; + +pub struct PythonPlatform { + pub config: PythonConfig, + + console: Arc, + + proto_env: Arc, + + toolchain: ToolManager, + + #[allow(dead_code)] + pub workspace_root: PathBuf, +} + +impl PythonPlatform { + pub fn new( + config: &PythonConfig, + workspace_root: &Path, + proto_env: Arc, + console: Arc, + ) -> Self { + PythonPlatform { + config: config.to_owned(), + proto_env, + toolchain: ToolManager::new(Runtime::new(PlatformType::Python, RuntimeReq::Global)), + workspace_root: workspace_root.to_path_buf(), + console, + } + } +} + +#[async_trait] +impl Platform for PythonPlatform { + fn get_type(&self) -> PlatformType { + PlatformType::Python + } + + fn get_runtime_from_config(&self, project_config: Option<&ProjectConfig>) -> Runtime { + if let Some(config) = &project_config { + if let Some(python_config) = &config.toolchain.python { + if let Some(version) = &python_config.version { + return Runtime::new_override( + PlatformType::Python, + RuntimeReq::Toolchain(version.to_owned()), + ); + } + } + } + + if let Some(version) = &self.config.version { + return Runtime::new( + PlatformType::Python, + RuntimeReq::Toolchain(version.to_owned()), + ); + } + + Runtime::new(PlatformType::Python, RuntimeReq::Global) + } + + fn matches(&self, platform: &PlatformType, runtime: Option<&Runtime>) -> bool { + if matches!(platform, PlatformType::Python) { + return true; + } + + if let Some(runtime) = &runtime { + return matches!(runtime.platform, PlatformType::Python); + } + + false + } + + // PROJECT GRAPH + + fn is_project_in_dependency_workspace(&self, _project_source: &str) -> miette::Result { + + Ok(false) + } + + #[instrument(skip_all)] + fn load_project_graph_aliases( + &mut self, + _projects_list: &ProjectsSourcesList, + _aliases_list: &mut ProjectsAliasesList, + ) -> miette::Result<()> { + // Extract the alias from the Cargo project relative to the lockfile + // for (id, source) in projects_list { + // let project_root = source.to_path(&self.workspace_root); + + // } + + Ok(()) + } + + // TOOLCHAIN + + fn is_toolchain_enabled(&self) -> miette::Result { + Ok(self.config.version.is_some()) + } + + fn get_tool(&self) -> miette::Result> { + let tool = self.toolchain.get()?; + + Ok(Box::new(tool)) + } + + fn get_tool_for_version(&self, req: RuntimeReq) -> miette::Result> { + let tool = self.toolchain.get_for_version(&req)?; + + Ok(Box::new(tool)) + } + + fn get_dependency_configs(&self) -> miette::Result> { + Ok(Some(("requirements.txt".to_owned(), "requirements.txt".to_owned()))) + } + + async fn setup_toolchain(&mut self) -> miette::Result<()> { + let req = match &self.config.version { + Some(v) => RuntimeReq::Toolchain(v.to_owned()), + None => RuntimeReq::Global, + }; + + let mut last_versions = FxHashMap::default(); + + if !self.toolchain.has(&req) { + self.toolchain.register( + &req, + PythonTool::new( + Arc::clone(&self.proto_env), + Arc::clone(&self.console), + &self.config, + &req, + ) + .await?, + ); + } + + + + self.toolchain.setup(&req, &mut last_versions).await?; + + // info!( + // target: LOG_TARGET, + // "Setup toolchain" + // ); + + Ok(()) + } + + async fn teardown_toolchain(&mut self) -> miette::Result<()> { + self.toolchain.teardown_all().await?; + + Ok(()) + } + + // ACTIONS + + #[instrument(skip_all)] + async fn setup_tool( + &mut self, + _context: &ActionContext, + runtime: &Runtime, + last_versions: &mut FxHashMap, + ) -> miette::Result { + let req = &runtime.requirement; + + + + + + if !self.toolchain.has(req) { + self.toolchain.register( + req, + PythonTool::new( + Arc::clone(&self.proto_env), + Arc::clone(&self.console), + &self.config, + req, + ) + .await?, + ); + } + Ok(self.toolchain.setup(req, last_versions).await?) + } + + #[instrument(skip_all)] + async fn install_deps( + &self, + _context: &ActionContext, + runtime: &Runtime, + _working_dir: &Path, + ) -> miette::Result> { + actions::install_deps( + self.toolchain.get_for_version(&runtime.requirement)?, + &get_workspace_root(), + &self.console, + ) + .await + } + + #[instrument(skip_all)] + async fn sync_project( + &self, + _context: &ActionContext, + _project: &Project, + _dependencies: &FxHashMap>, + ) -> miette::Result { + let mutated_files = false; + //TODO: Here we can modifiy something + + Ok(mutated_files) + } + + + // # Lockfile or manifests have not changed since last run, skipping dependency install + #[instrument(skip_all)] + async fn hash_manifest_deps( + &self, + _manifest_path: &Path, + hasher: &mut ContentHasher, + _hasher_config: &HasherConfig, + ) -> miette::Result<()> { + hasher.hash_content(PythonToolchainHash { + pip: Some(self.config.pip).expect("S"), + version: Some(self.config.version).expect("S"), + requirements_dependencies: "".into(), + })?; + + Ok(()) + } + + #[instrument(skip_all)] + async fn hash_run_target( + &self, + project: &Project, + _runtime: &Runtime, + hasher: &mut ContentHasher, + _hasher_config: &HasherConfig, + ) -> miette::Result<()> { + let lockfile_path = project.root.join("requirements.txt"); + + + debug!( + target: LOG_TARGET, + "{} does not exist, installing", + lockfile_path.display() + ); + + // Not running in the Cargo workspace root, not sure how to handle! + if !lockfile_path.exists() { + return Ok(()); + } + + let mut hash = PythonTargetHash::new(None); + + // Use the resolved dependencies from the lockfile directly, + // since it also takes into account features and workspace members. + hash.locked_dependencies = BTreeMap::from_iter(load_lockfile_dependencies(lockfile_path)?); + + hasher.hash_content(hash)?; + + Ok(()) + } + + #[instrument(skip_all)] + async fn create_run_target_command( + &self, + _context: &ActionContext, + _project: &Project, + task: &Task, + runtime: &Runtime, + _working_dir: &Path, + ) -> miette::Result { + + let mut command = Command::new(&task.command); + command.with_console(self.console.clone()); + command.args(&task.args); + command.envs(&task.env); + + if let Ok(python) = self.toolchain.get_for_version(&runtime.requirement) { + if let Some(version) = get_proto_version_env(&python.tool) { + command.env("PROTO_PYTHON_VERSION", version); + command.env( + "PATH", + prepend_path_env_var(get_python_tool_paths(&python)), + ); + } + } + + Ok(command) + } +} diff --git a/legacy/python/platform/src/target_hash.rs b/legacy/python/platform/src/target_hash.rs new file mode 100644 index 00000000000..e32a6b3f9a1 --- /dev/null +++ b/legacy/python/platform/src/target_hash.rs @@ -0,0 +1,18 @@ +use moon_hash::hash_content; +use std::collections::BTreeMap; + +hash_content!( + pub struct PythonTargetHash { + pub python_version: String, + pub locked_dependencies: BTreeMap>, + } +); + +impl PythonTargetHash { + pub fn new(python_version: Option) -> Self { + PythonTargetHash { + python_version: python_version.unwrap_or_else(|| "unknown".into()), + locked_dependencies: BTreeMap::new(), + } + } +} diff --git a/legacy/python/platform/src/toolchain_hash.rs b/legacy/python/platform/src/toolchain_hash.rs new file mode 100644 index 00000000000..540bb06c372 --- /dev/null +++ b/legacy/python/platform/src/toolchain_hash.rs @@ -0,0 +1,21 @@ +use moon_config::{PipConfig, UnresolvedVersionSpec}; +use moon_hash::hash_content; +use std::collections::BTreeMap; + +hash_content!( + pub struct PythonToolchainHash { + pub pip: Option, + pub version: Option, + pub requirements_dependencies: String, + } +); + +impl PythonToolchainHash { + pub fn new(python_version: Option, pip_config: Option) -> Self { + PythonToolchainHash { + version: python_version, + requirements_dependencies: "".into(), + pip: pip_config, + } + } +} \ No newline at end of file diff --git a/legacy/python/tool/Cargo.toml b/legacy/python/tool/Cargo.toml new file mode 100644 index 00000000000..8e9cba3f79b --- /dev/null +++ b/legacy/python/tool/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "moon_python_tool" +version = "0.0.1" +edition = "2021" +publish = false + +[dependencies] +moon_python_lang = { path = "../lang" } +moon_common = { path = "../../../crates/common" } +moon_config = { path = "../../../crates/config" } +moon_console = { path = "../../../crates/console" } +moon_logger = { path = "../../core/logger" } +moon_process = { path = "../../../crates/process" } +moon_tool = { path = "../../core/tool" } +moon_utils = { path = "../../core/utils" } +moon_toolchain = { path = "../../../crates/toolchain" } +starbase_styles = { workspace = true } +miette = { workspace = true } +proto_core = { workspace = true } +rustc-hash = { workspace = true } +starbase_utils = { workspace = true } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/legacy/python/tool/src/lib.rs b/legacy/python/tool/src/lib.rs new file mode 100644 index 00000000000..84807fe6f11 --- /dev/null +++ b/legacy/python/tool/src/lib.rs @@ -0,0 +1,3 @@ +mod python_tool; + +pub use python_tool::*; diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs new file mode 100644 index 00000000000..75bb82e7732 --- /dev/null +++ b/legacy/python/tool/src/python_tool.rs @@ -0,0 +1,292 @@ +use moon_config::PythonConfig; +use moon_python_lang::pip_requirements::{load_lockfile_dependencies}; +use moon_python_lang::{LockfileDependencyVersions}; +use moon_console::{Checkpoint, Console}; +// use moon_logger::debug; +use moon_utils::get_workspace_root; +use moon_process::Command; +use moon_tool::{ + async_trait, get_proto_paths, load_tool_plugin, prepend_path_env_var, use_global_tool_on_path, + Tool, get_proto_version_env, get_proto_env_vars, DependencyManager +}; +use moon_toolchain::RuntimeReq; +use proto_core::flow::install::InstallOptions; +use proto_core::{Id, ProtoEnvironment, Tool as ProtoTool, UnresolvedVersionSpec}; +use rustc_hash::FxHashMap; +use std::env; +use std::path::PathBuf; +use std::sync::Arc; +use std::{ffi::OsStr, path::Path}; +use starbase_utils::fs; +use tracing::instrument; +use starbase_styles::color; +use moon_logger::{debug, map_list}; + +const LOG_TARGET: &str = "moon:python-tool"; + + +pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { + // let mut paths = get_proto_paths(proto_env); + // let mut paths:Vec = []; + + let paths = python_tool.tool.get_globals_dirs() + .iter() + .cloned() + .collect::>(); + + paths +} + +pub fn get_python_env_paths(proto_env: &ProtoEnvironment) -> Vec { + let paths = get_proto_paths(proto_env); + + paths +} + +pub struct PythonTool { + pub config: PythonConfig, + + pub global: bool, + + pub tool: ProtoTool, + + console: Arc, + + proto_env: Arc, +} + +impl PythonTool { + pub async fn new( + proto_env: Arc, + console: Arc, + config: &PythonConfig, + req: &RuntimeReq, + ) -> miette::Result { + let mut python = PythonTool { + config: config.to_owned(), + global: false, + tool: load_tool_plugin( + &Id::raw("python"), + &proto_env, + config.plugin.as_ref().unwrap(), + ) + .await?, + proto_env, + console, + }; + + if use_global_tool_on_path("python") || req.is_global() { + python.global = true; + python.config.version = None; + } else { + python.config.version = req.to_spec(); + }; + + Ok(python) + } + + #[instrument(skip_all)] + pub async fn exec_python(&self, args: I, working_dir: &Path) -> miette::Result<()> + where + I: IntoIterator, + S: AsRef, + { + Command::new("python") + .args(args) + .envs(get_proto_env_vars()) + .env( + "PATH", + prepend_path_env_var(get_python_env_paths(&self.proto_env)), + ) + .cwd(working_dir) + .with_console(self.console.clone()) + .create_async() + .exec_stream_output() + .await?; + + Ok(()) + + + // let mut cmd = Command::new("python"); + // cmd.with_console(self.console.clone()); + // cmd.envs(get_proto_env_vars()); + // cmd.envs(get_proto_env_vars()); + // if !self.global { + // cmd.env( + // "PATH", + // prepend_path_env_var(get_python_env_paths(&self.proto_env)), + // ); + // } + } + +} + +#[async_trait] +impl Tool for PythonTool { + fn as_any(&self) -> &(dyn std::any::Any + Send + Sync) { + self + } + + #[instrument(skip_all)] + async fn setup( + &mut self, + last_versions: &mut FxHashMap, + ) -> miette::Result { + let mut installed = 0; + + let Some(version) = &self.config.version else { + return Ok(installed); + }; + + if self.global { + debug!("Using global binary in PATH"); + } else if self.tool.is_setup(version).await? { + debug!("Python has already been setup"); + + // When offline and the tool doesn't exist, fallback to the global binary + } else if proto_core::is_offline() { + debug!( + "No internet connection and Python has not been setup, falling back to global binary in PATH" + ); + + self.global = true; + + // Otherwise try and install the tool + } else { + let setup = match last_versions.get("python") { + Some(last) => version != last, + None => true, + }; + + if setup || !self.tool.get_product_dir().exists() { + self.console + .out + .print_checkpoint(Checkpoint::Setup, format!("installing python {version}"))?; + + if self.tool.setup(version, InstallOptions::default()).await? { + last_versions.insert("python".into(), version.to_owned()); + installed += 1; + } + } + } + self.tool.locate_globals_dirs().await?; + + + + + + Ok(installed) + } + + async fn teardown(&mut self) -> miette::Result<()> { + self.tool.teardown().await?; + + Ok(()) + } +} + + +#[async_trait] +impl DependencyManager for PythonTool { + fn create_command(&self, python: &PythonTool) -> miette::Result { + let mut cmd = Command::new("python"); + cmd.with_console(self.console.clone()); + cmd.envs(get_proto_env_vars()); + if !self.global { + cmd.env( + "PATH", + prepend_path_env_var(get_python_env_paths(&self.proto_env)), + ); + } + + if let Some(version) = get_proto_version_env(&self.tool) { + cmd.env("PROTO_PYTHON_VERSION", version); + } + + if let Some(version) = get_proto_version_env(&python.tool) { + cmd.env("PROTO_PYTHON_VERSION", version); + } + + Ok(cmd) + } + + #[instrument(skip_all)] + async fn dedupe_dependencies( + &self, + _python: &PythonTool, + _working_dir: &Path, + _log: bool, + ) -> miette::Result<()> { + // Not supported! + + Ok(()) + } + + fn get_lock_filename(&self) -> String { + String::from("requirements.txt") + } + + fn get_manifest_filename(&self) -> String { + String::from("requirements.txt") + } + + #[instrument(skip_all)] + async fn get_resolved_dependencies( + &self, + project_root: &Path, + ) -> miette::Result { + let Some(lockfile_path) = + fs::find_upwards_until("requirements.txt", project_root, get_workspace_root()) + else { + return Ok(FxHashMap::default()); + }; + + Ok(load_lockfile_dependencies(lockfile_path)?) + } + #[instrument(skip_all)] + async fn install_dependencies( + &self, + python: &PythonTool, + working_dir: &Path, + log: bool, + ) -> miette::Result<()> { + let mut cmd = self.create_command(python)?; + + // TODO: DONT KNOW IF CORRECT LOCATION, BECAUSE IT IS HANDLING THE TOOL INSTALLATION + if let Some(pip_config) = &self.config.pip { + + cmd.args(["install"]) + // .args(&args) + .cwd(working_dir) + .set_print_command(log); + + //TODO: only read from root, but ready for sub virtual environments + if let Some(requirements_path) = fs::find_upwards_until("requirements.txt", get_workspace_root(), get_workspace_root()) { + cmd.args(["-r", &requirements_path.as_os_str().to_str().unwrap()]); + } + if let Some(install_args) = &pip_config.install_args { + cmd.args(install_args); + } + + let mut cmd = cmd.create_async(); + if env::var("MOON_TEST_HIDE_INSTALL_OUTPUT").is_ok() { + cmd.exec_capture_output().await?; + } else { + cmd.exec_stream_output().await?; + } + } + + Ok(()) + } + + #[instrument(skip_all)] + async fn install_focused_dependencies( + &self, + _python: &PythonTool, + _packages: &[String], + _production_only: bool, + ) -> miette::Result<()> { + // TODO: Implement for docker purposes + Ok(()) + } +} diff --git a/packages/types/src/project-config.ts b/packages/types/src/project-config.ts index de23f2100c3..578432efbed 100644 --- a/packages/types/src/project-config.ts +++ b/packages/types/src/project-config.ts @@ -2,8 +2,8 @@ /* eslint-disable */ -import type { UnresolvedVersionSpec } from './toolchain-config'; import type { PartialTaskConfig, PlatformType, TaskConfig } from './tasks-config'; +import type { UnresolvedVersionSpec } from './toolchain-config'; /** The scope and or relationship of the dependency. */ export type DependencyScope = 'build' | 'development' | 'peer' | 'production' | 'root'; @@ -66,18 +66,7 @@ export interface ProjectDockerConfig { } /** Supported programming languages that each project can be written in. */ -export type LanguageType = - | 'bash' - | 'batch' - | 'go' - | 'javascript' - | 'php' - | 'python' - | 'ruby' - | 'rust' - | 'typescript' - | 'unknown' - | string; +export type LanguageType = 'bash' | 'batch' | 'go' | 'javascript' | 'php' | 'python' | 'ruby' | 'rust' | 'typescript' | 'unknown' | string; export type OwnersPaths = string[] | Record; @@ -164,6 +153,8 @@ export interface ProjectToolchainConfig { deno: ProjectToolchainCommonToolConfig | null; /** Overrides `node` settings. */ node: ProjectToolchainCommonToolConfig | null; + /** Overrides `python` settings. */ + python: ProjectToolchainCommonToolConfig | null; /** Overrides `rust` settings. */ rust: ProjectToolchainCommonToolConfig | null; /** Overrides `typescript` settings. */ @@ -171,14 +162,7 @@ export interface ProjectToolchainConfig { } /** The type of project, for categorizing. */ -export type ProjectType = - | 'application' - | 'automation' - | 'configuration' - | 'library' - | 'scaffolding' - | 'tool' - | 'unknown'; +export type ProjectType = 'application' | 'automation' | 'configuration' | 'library' | 'scaffolding' | 'tool' | 'unknown'; /** Controls how tasks are inherited. */ export interface ProjectWorkspaceInheritedTasksConfig { @@ -405,6 +389,8 @@ export interface PartialProjectToolchainConfig { deno?: PartialProjectToolchainCommonToolConfig | null; /** Overrides `node` settings. */ node?: PartialProjectToolchainCommonToolConfig | null; + /** Overrides `python` settings. */ + python?: PartialProjectToolchainCommonToolConfig | null; /** Overrides `rust` settings. */ rust?: PartialProjectToolchainCommonToolConfig | null; /** Overrides `typescript` settings. */ diff --git a/packages/types/src/tasks-config.ts b/packages/types/src/tasks-config.ts index cc4773fce9a..189bcabda8c 100644 --- a/packages/types/src/tasks-config.ts +++ b/packages/types/src/tasks-config.ts @@ -32,16 +32,7 @@ export type TaskOperatingSystem = 'linux' | 'macos' | 'windows'; export type TaskOutputStyle = 'buffer' | 'buffer-only-failure' | 'hash' | 'none' | 'stream'; /** A list of available shells on Unix. */ -export type TaskUnixShell = - | 'bash' - | 'elvish' - | 'fish' - | 'ion' - | 'murex' - | 'nu' - | 'pwsh' - | 'xonsh' - | 'zsh'; +export type TaskUnixShell = 'bash' | 'elvish' | 'fish' | 'ion' | 'murex' | 'nu' | 'pwsh' | 'xonsh' | 'zsh'; /** A list of available shells on Windows. */ export type TaskWindowsShell = 'bash' | 'elvish' | 'fish' | 'murex' | 'nu' | 'pwsh' | 'xonsh'; @@ -165,7 +156,7 @@ export interface TaskOptionsConfig { } /** Platforms that each programming language can belong to. */ -export type PlatformType = 'bun' | 'deno' | 'node' | 'rust' | 'system' | 'unknown'; +export type PlatformType = 'bun' | 'deno' | 'node' | 'python' | 'rust' | 'system' | 'unknown'; /** Preset options to inherit. */ export type TaskPreset = 'server' | 'watcher'; @@ -228,7 +219,7 @@ export interface TaskConfig { * be automatically detected. * * @default 'unknown' - * @type {'bun' | 'deno' | 'node' | 'rust' | 'system' | 'unknown'} + * @type {'bun' | 'deno' | 'node' | 'python' | 'rust' | 'system' | 'unknown'} */ platform: PlatformType; /** The preset to apply for the task. Will inherit default options. */ diff --git a/packages/types/src/template-config.ts b/packages/types/src/template-config.ts index db9ea77308c..9c1ffcb55cd 100644 --- a/packages/types/src/template-config.ts +++ b/packages/types/src/template-config.ts @@ -92,11 +92,7 @@ export interface TemplateVariableStringSetting { type: 'string'; } -export type TemplateVariable = - | TemplateVariableBoolSetting - | TemplateVariableEnumSetting - | TemplateVariableNumberSetting - | TemplateVariableStringSetting; +export type TemplateVariable = TemplateVariableBoolSetting | TemplateVariableEnumSetting | TemplateVariableNumberSetting | TemplateVariableStringSetting; /** * Configures a template and its files to be scaffolded. @@ -197,11 +193,7 @@ export interface PartialTemplateVariableStringSetting { type?: 'string' | null; } -export type PartialTemplateVariable = - | PartialTemplateVariableBoolSetting - | PartialTemplateVariableEnumSetting - | PartialTemplateVariableNumberSetting - | PartialTemplateVariableStringSetting; +export type PartialTemplateVariable = PartialTemplateVariableBoolSetting | PartialTemplateVariableEnumSetting | PartialTemplateVariableNumberSetting | PartialTemplateVariableStringSetting; /** * Configures a template and its files to be scaffolded. diff --git a/packages/types/src/toolchain-config.ts b/packages/types/src/toolchain-config.ts index 07f654a2aa2..b4002fb7f37 100644 --- a/packages/types/src/toolchain-config.ts +++ b/packages/types/src/toolchain-config.ts @@ -3,16 +3,7 @@ /* eslint-disable */ /** Formats that a `package.json` version dependency can be. */ -export type NodeVersionFormat = - | 'file' - | 'link' - | 'star' - | 'version' - | 'version-caret' - | 'version-tilde' - | 'workspace' - | 'workspace-caret' - | 'workspace-tilde'; +export type NodeVersionFormat = 'file' | 'link' | 'star' | 'version' | 'version-caret' | 'version-tilde' | 'workspace' | 'workspace-caret' | 'workspace-tilde'; export type PluginLocator = string; @@ -259,6 +250,26 @@ export interface NodeConfig { yarn: YarnConfig | null; } +export interface PipConfig { + /** List of arguments to append to `pip install` commands. */ + installArgs: string[] | null; + /** The version of pip to download, install, and run `pip` tasks with. */ + version: UnresolvedVersionSpec | null; +} + +export interface PythonConfig { + /** Options for pnpm, when used as a package manager. */ + pip: PipConfig | null; + /** Location of the WASM plugin to use for Python support. */ + plugin: PluginLocator | null; + /** + * The version of Python to download, install, and run `python` tasks with. + * + * @envvar MOON_PYTHON_VERSION + */ + version: UnresolvedVersionSpec | null; +} + /** * Configures and enables the Rust platform. * Docs: https://moonrepo.dev/docs/config/toolchain#rust @@ -376,6 +387,8 @@ export interface ToolchainConfig { moon: MoonConfig; /** Configures and enables the Node.js platform. */ node: NodeConfig | null; + /** Configures and enables the Python platform. */ + python: PythonConfig | null; /** Configures and enables the Rust platform. */ rust: RustConfig | null; /** All configured toolchains by unique ID. */ @@ -616,6 +629,26 @@ export interface PartialNodeConfig { yarn?: PartialYarnConfig | null; } +export interface PartialPipConfig { + /** List of arguments to append to `pip install` commands. */ + installArgs?: string[] | null; + /** The version of pip to download, install, and run `pip` tasks with. */ + version?: UnresolvedVersionSpec | null; +} + +export interface PartialPythonConfig { + /** Options for pnpm, when used as a package manager. */ + pip?: PartialPipConfig | null; + /** Location of the WASM plugin to use for Python support. */ + plugin?: PluginLocator | null; + /** + * The version of Python to download, install, and run `python` tasks with. + * + * @envvar MOON_PYTHON_VERSION + */ + version?: UnresolvedVersionSpec | null; +} + /** * Configures and enables the Rust platform. * Docs: https://moonrepo.dev/docs/config/toolchain#rust @@ -733,6 +766,8 @@ export interface PartialToolchainConfig { moon?: PartialMoonConfig | null; /** Configures and enables the Node.js platform. */ node?: PartialNodeConfig | null; + /** Configures and enables the Python platform. */ + python?: PartialPythonConfig | null; /** Configures and enables the Rust platform. */ rust?: PartialRustConfig | null; /** All configured toolchains by unique ID. */ diff --git a/packages/types/src/workspace-config.ts b/packages/types/src/workspace-config.ts index d03b98f62d3..bae3fe69deb 100644 --- a/packages/types/src/workspace-config.ts +++ b/packages/types/src/workspace-config.ts @@ -529,10 +529,7 @@ export interface PartialWorkspaceProjectsConfig { sources?: Record | null; } -export type PartialWorkspaceProjects = - | PartialWorkspaceProjectsConfig - | string[] - | Record; +export type PartialWorkspaceProjects = PartialWorkspaceProjectsConfig | string[] | Record; /** Configures aspects of the task runner (also known as the action pipeline). */ export interface PartialRunnerConfig { diff --git a/website/static/schemas/project.json b/website/static/schemas/project.json index e1f7e4830a1..d95b059945e 100644 --- a/website/static/schemas/project.json +++ b/website/static/schemas/project.json @@ -376,6 +376,7 @@ "bun", "deno", "node", + "python", "rust", "system", "unknown" @@ -616,6 +617,19 @@ ], "markdownDescription": "Overrides `node` settings." }, + "python": { + "title": "python", + "description": "Overrides python settings.", + "anyOf": [ + { + "$ref": "#/definitions/ProjectToolchainCommonToolConfig" + }, + { + "type": "null" + } + ], + "markdownDescription": "Overrides `python` settings." + }, "rust": { "title": "rust", "description": "Overrides rust settings.", diff --git a/website/static/schemas/tasks.json b/website/static/schemas/tasks.json index 0f9cbedbce9..e2e106b1538 100644 --- a/website/static/schemas/tasks.json +++ b/website/static/schemas/tasks.json @@ -83,6 +83,7 @@ "bun", "deno", "node", + "python", "rust", "system", "unknown" diff --git a/website/static/schemas/toolchain.json b/website/static/schemas/toolchain.json index 28751563819..2f56f231ab5 100644 --- a/website/static/schemas/toolchain.json +++ b/website/static/schemas/toolchain.json @@ -65,6 +65,18 @@ } ] }, + "python": { + "title": "python", + "description": "Configures and enables the Python platform.", + "anyOf": [ + { + "$ref": "#/definitions/PythonConfig" + }, + { + "type": "null" + } + ] + }, "rust": { "title": "rust", "description": "Configures and enables the Rust platform.", @@ -568,6 +580,41 @@ }, "additionalProperties": false }, + "PipConfig": { + "type": "object", + "properties": { + "installArgs": { + "title": "installArgs", + "description": "List of arguments to append to pip install commands.", + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ], + "markdownDescription": "List of arguments to append to `pip install` commands." + }, + "version": { + "title": "version", + "description": "The version of pip to download, install, and run pip tasks with.", + "anyOf": [ + { + "$ref": "#/definitions/UnresolvedVersionSpec" + }, + { + "type": "null" + } + ], + "markdownDescription": "The version of pip to download, install, and run `pip` tasks with." + } + }, + "additionalProperties": false + }, "PluginLocator": { "description": "Strategies and protocols for locating plugins.", "type": "string" @@ -613,6 +660,49 @@ }, "additionalProperties": false }, + "PythonConfig": { + "type": "object", + "properties": { + "pip": { + "title": "pip", + "description": "Options for pnpm, when used as a package manager.", + "anyOf": [ + { + "$ref": "#/definitions/PipConfig" + }, + { + "type": "null" + } + ] + }, + "plugin": { + "title": "plugin", + "description": "Location of the WASM plugin to use for Python support.", + "anyOf": [ + { + "$ref": "#/definitions/PluginLocator" + }, + { + "type": "null" + } + ] + }, + "version": { + "title": "version", + "description": "The version of Python to download, install, and run python tasks with.", + "anyOf": [ + { + "$ref": "#/definitions/UnresolvedVersionSpec" + }, + { + "type": "null" + } + ], + "markdownDescription": "The version of Python to download, install, and run `python` tasks with." + } + }, + "additionalProperties": false + }, "RustConfig": { "description": "Configures and enables the Rust platform. Docs: https://moonrepo.dev/docs/config/toolchain#rust", "type": "object", From e8b1ae5cdb193b6e5219497c0e4d4ee5683b5387 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:09:21 +0200 Subject: [PATCH 12/28] next iteration --- legacy/python/platform/src/python_platform.rs | 48 +++++++++---------- legacy/python/platform/src/toolchain_hash.rs | 12 ++--- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 1ef0f635198..850ae978169 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -14,7 +14,7 @@ use moon_logger::debug; use moon_platform::{Platform, Runtime, RuntimeReq}; use moon_process::Command; use moon_project::Project; -use moon_python_lang::pip_requirements::load_lockfile_dependencies; +use moon_python_lang::pip_requirements::{self, load_lockfile_dependencies}; use moon_python_tool::{get_python_tool_paths, PythonTool}; use moon_task::Task; use moon_tool::{get_proto_version_env, prepend_path_env_var, Tool, ToolManager}; @@ -248,11 +248,18 @@ impl Platform for PythonPlatform { hasher: &mut ContentHasher, _hasher_config: &HasherConfig, ) -> miette::Result<()> { - hasher.hash_content(PythonToolchainHash { - pip: Some(self.config.pip).expect("S"), - version: Some(self.config.version).expect("S"), - requirements_dependencies: "".into(), - })?; + + if let Some(python_version) = &self.config.version { + let mut deps = BTreeMap::new(); + if let Some(pip_requirements) = find_requirements_txt(&get_workspace_root(), &get_workspace_root()) { + deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); + } + + hasher.hash_content(PythonToolchainHash { + version: python_version.clone(), + dependencies: deps, + })?; + } Ok(()) } @@ -265,27 +272,20 @@ impl Platform for PythonPlatform { hasher: &mut ContentHasher, _hasher_config: &HasherConfig, ) -> miette::Result<()> { - let lockfile_path = project.root.join("requirements.txt"); - - - debug!( - target: LOG_TARGET, - "{} does not exist, installing", - lockfile_path.display() - ); - // Not running in the Cargo workspace root, not sure how to handle! - if !lockfile_path.exists() { - return Ok(()); + if let Some(python_version) = &self.config.version { + let mut deps = BTreeMap::new(); + if let Some(pip_requirements) = find_requirements_txt(&get_workspace_root(), &get_workspace_root()) { + deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); + } + + hasher.hash_content(PythonToolchainHash { + version: python_version.clone(), + dependencies: deps, + })?; } - let mut hash = PythonTargetHash::new(None); - - // Use the resolved dependencies from the lockfile directly, - // since it also takes into account features and workspace members. - hash.locked_dependencies = BTreeMap::from_iter(load_lockfile_dependencies(lockfile_path)?); - - hasher.hash_content(hash)?; + Ok(()) } diff --git a/legacy/python/platform/src/toolchain_hash.rs b/legacy/python/platform/src/toolchain_hash.rs index 540bb06c372..9ce77a8ea07 100644 --- a/legacy/python/platform/src/toolchain_hash.rs +++ b/legacy/python/platform/src/toolchain_hash.rs @@ -4,18 +4,18 @@ use std::collections::BTreeMap; hash_content!( pub struct PythonToolchainHash { - pub pip: Option, - pub version: Option, - pub requirements_dependencies: String, + // pub pip: Option, + pub version: UnresolvedVersionSpec, + pub dependencies: BTreeMap>, } ); impl PythonToolchainHash { - pub fn new(python_version: Option, pip_config: Option) -> Self { + pub fn new(python_version: UnresolvedVersionSpec, pip_config: Option) -> Self { PythonToolchainHash { version: python_version, - requirements_dependencies: "".into(), - pip: pip_config, + dependencies: BTreeMap::new(), + // pip: pip_config, } } } \ No newline at end of file From 59c55df51eceed2688286fb4c9e4a309f2952519 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:44:27 +0200 Subject: [PATCH 13/28] Latest version --- crates/config/src/toolchain/python_config.rs | 3 + .../platform/src/actions/install_deps.rs | 44 +++++++------- legacy/python/platform/src/python_platform.rs | 12 ++-- legacy/python/platform/src/target_hash.rs | 16 +++--- legacy/python/platform/src/toolchain_hash.rs | 22 ++++--- legacy/python/tool/src/python_tool.rs | 57 +++++++++++++++---- packages/types/src/project-config.ts | 2 +- 7 files changed, 98 insertions(+), 58 deletions(-) diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs index 5214f52cca1..b4dddf50659 100644 --- a/crates/config/src/toolchain/python_config.rs +++ b/crates/config/src/toolchain/python_config.rs @@ -29,6 +29,9 @@ pub struct PythonConfig { #[setting(default = ".", skip)] pub venv_root: String, + #[setting(default = ".venv", skip)] + pub venv_name: String, + /// The version of Python to download, install, and run `python` tasks with. #[setting(env = "MOON_PYTHON_VERSION")] pub version: Option, diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs index a7ed87a496e..b60aae71574 100644 --- a/legacy/python/platform/src/actions/install_deps.rs +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -13,8 +13,27 @@ pub async fn install_deps( ) -> miette::Result> { let mut operations = vec![]; + // python.exec_python(args, working_dir) + + if let Some(pip_config) = &python.config.pip { + // Very first step: Activate virtual environment + console + .out + .print_checkpoint(Checkpoint::Setup, format!("activate virtual environment"))?; + let virtual_environment = &get_workspace_root().join(python.config.venv_name.clone()); + + if !virtual_environment.exists() { + let args = vec!["-m", "venv", virtual_environment.as_os_str().to_str().unwrap()]; + operations.push( + Operation::task_execution(format!("python {} ", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); + } + + if let Some(pip_version) = &pip_config.version { @@ -27,29 +46,16 @@ pub async fn install_deps( } else { format!("pip{}", pip_version.to_owned().to_string().replace("~", "~=")) }; - let args = vec!["-m", "pip", "install","--quiet", "-U", &p_version]; - + let args = vec!["-m", "pip", "install", "-U", &p_version]; +// #"--quiet", operations.push( - Operation::task_execution(format!("python {} ", args.join(" "))) + Operation::task_execution(format!(" {} ", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, ); } - console - .out - .print_checkpoint(Checkpoint::Setup, format!("activate virtual environment"))?; - let virtual_environment = &get_workspace_root().join(".venv"); - if !virtual_environment.exists() { - let args = vec!["-m", "venv", virtual_environment.as_os_str().to_str().unwrap()]; - operations.push( - Operation::task_execution(format!("python {} ", args.join(" "))) - .track_async(|| python.exec_python(args, working_dir)) - .await?, - ); - } - @@ -58,8 +64,8 @@ pub async fn install_deps( .out .print_checkpoint(Checkpoint::Setup, format!("pip dependencies from {}", req.as_os_str().to_str().unwrap()))?; - let mut args = vec!["-m", "pip", "install", "--quiet"]; - + let mut args = vec!["-m", "pip", "install"]; + // #, "--quiet" if pip_config.install_args.is_some() { args.extend(pip_config.install_args.as_ref().unwrap().iter().map(|c| c.as_str())); } @@ -67,7 +73,7 @@ pub async fn install_deps( args.extend(["-r", req.as_os_str().to_str().unwrap()]); operations.push( - Operation::task_execution(format!("python {}", args.join(" "))) + Operation::task_execution(format!(" {}", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, ); diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 850ae978169..33a395cb007 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -1,5 +1,5 @@ use crate::{ - actions, find_requirements_txt, target_hash::PythonTargetHash, toolchain_hash::PythonToolchainHash + actions, find_requirements_txt, toolchain_hash::PythonToolchainHash }; use moon_action::Operation; use moon_action_context::ActionContext; @@ -8,13 +8,13 @@ use moon_config::{ HasherConfig, PlatformType, ProjectConfig, ProjectsAliasesList, ProjectsSourcesList, PythonConfig, UnresolvedVersionSpec, }; -use moon_console::{Checkpoint, Console}; +use moon_console::Console; use moon_hash::ContentHasher; -use moon_logger::debug; +// use moon_logger::debug; use moon_platform::{Platform, Runtime, RuntimeReq}; use moon_process::Command; use moon_project::Project; -use moon_python_lang::pip_requirements::{self, load_lockfile_dependencies}; +use moon_python_lang::pip_requirements::load_lockfile_dependencies; use moon_python_tool::{get_python_tool_paths, PythonTool}; use moon_task::Task; use moon_tool::{get_proto_version_env, prepend_path_env_var, Tool, ToolManager}; @@ -26,7 +26,7 @@ use std::{ }; use tracing::instrument; -const LOG_TARGET: &str = "moon:python-platform"; +// const LOG_TARGET: &str = "moon:python-platform"; pub struct PythonPlatform { pub config: PythonConfig, @@ -267,7 +267,7 @@ impl Platform for PythonPlatform { #[instrument(skip_all)] async fn hash_run_target( &self, - project: &Project, + _project: &Project, _runtime: &Runtime, hasher: &mut ContentHasher, _hasher_config: &HasherConfig, diff --git a/legacy/python/platform/src/target_hash.rs b/legacy/python/platform/src/target_hash.rs index e32a6b3f9a1..155f3e1d195 100644 --- a/legacy/python/platform/src/target_hash.rs +++ b/legacy/python/platform/src/target_hash.rs @@ -8,11 +8,11 @@ hash_content!( } ); -impl PythonTargetHash { - pub fn new(python_version: Option) -> Self { - PythonTargetHash { - python_version: python_version.unwrap_or_else(|| "unknown".into()), - locked_dependencies: BTreeMap::new(), - } - } -} +// impl PythonTargetHash { +// pub fn new(python_version: Option) -> Self { +// PythonTargetHash { +// python_version: python_version.unwrap_or_else(|| "unknown".into()), +// locked_dependencies: BTreeMap::new(), +// } +// } +// } diff --git a/legacy/python/platform/src/toolchain_hash.rs b/legacy/python/platform/src/toolchain_hash.rs index 9ce77a8ea07..22f0b8c376c 100644 --- a/legacy/python/platform/src/toolchain_hash.rs +++ b/legacy/python/platform/src/toolchain_hash.rs @@ -1,21 +1,19 @@ -use moon_config::{PipConfig, UnresolvedVersionSpec}; +use moon_config::UnresolvedVersionSpec; use moon_hash::hash_content; use std::collections::BTreeMap; hash_content!( - pub struct PythonToolchainHash { - // pub pip: Option, + pub struct PythonToolchainHash { pub version: UnresolvedVersionSpec, pub dependencies: BTreeMap>, } ); -impl PythonToolchainHash { - pub fn new(python_version: UnresolvedVersionSpec, pip_config: Option) -> Self { - PythonToolchainHash { - version: python_version, - dependencies: BTreeMap::new(), - // pip: pip_config, - } - } -} \ No newline at end of file +// impl PythonToolchainHash { +// pub fn new(python_version: UnresolvedVersionSpec) -> Self { +// PythonToolchainHash { +// version: python_version, +// dependencies: BTreeMap::new(), +// } +// } +// } \ No newline at end of file diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs index 75bb82e7732..98a102317e7 100644 --- a/legacy/python/tool/src/python_tool.rs +++ b/legacy/python/tool/src/python_tool.rs @@ -6,8 +6,7 @@ use moon_console::{Checkpoint, Console}; use moon_utils::get_workspace_root; use moon_process::Command; use moon_tool::{ - async_trait, get_proto_paths, load_tool_plugin, prepend_path_env_var, use_global_tool_on_path, - Tool, get_proto_version_env, get_proto_env_vars, DependencyManager + async_trait, get_proto_env_vars, get_proto_paths, get_proto_version_env, load_tool_plugin, prepend_path_env_var, use_global_tool_on_path, DependencyManager, Tool }; use moon_toolchain::RuntimeReq; use proto_core::flow::install::InstallOptions; @@ -29,20 +28,48 @@ pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { // let mut paths = get_proto_paths(proto_env); // let mut paths:Vec = []; - let paths = python_tool.tool.get_globals_dirs() - .iter() - .cloned() - .collect::>(); + // let mut python_command = "python"; - paths -} + let venv_python = &get_workspace_root().join(python_tool.config.venv_name.clone()); + + let paths; + + if venv_python.exists() { + paths = vec![venv_python.join("Scripts").clone()]; + } else { + // paths = python_tool.tool.get_globals_dirs() + // .iter() + // .cloned() + // .collect::>(); + paths = get_proto_paths(&python_tool.proto_env); + } -pub fn get_python_env_paths(proto_env: &ProtoEnvironment) -> Vec { - let paths = get_proto_paths(proto_env); + // for p in python_tool.tool.get_globals_dirs().iter() { + // debug!( + // target: LOG_TARGET, + // "Proto Env {} ", + // p.clone().display(), + // ); + // } + + + + + debug!( + target: LOG_TARGET, + "Proto Env {} ", + map_list(&paths, |c| color::label(c.display().to_string())), + ); paths } +// pub fn get_python_env_paths(proto_env: &ProtoEnvironment) -> Vec { +// let paths = get_proto_paths(proto_env); + +// paths +// } + pub struct PythonTool { pub config: PythonConfig, @@ -91,12 +118,18 @@ impl PythonTool { I: IntoIterator, S: AsRef, { + + // Check if venv is already created. + + + + Command::new("python") .args(args) .envs(get_proto_env_vars()) .env( "PATH", - prepend_path_env_var(get_python_env_paths(&self.proto_env)), + prepend_path_env_var(get_python_tool_paths(&self)), ) .cwd(working_dir) .with_console(self.console.clone()) @@ -195,7 +228,7 @@ impl DependencyManager for PythonTool { if !self.global { cmd.env( "PATH", - prepend_path_env_var(get_python_env_paths(&self.proto_env)), + prepend_path_env_var(get_python_tool_paths(&self)), ); } diff --git a/packages/types/src/project-config.ts b/packages/types/src/project-config.ts index 578432efbed..9be59614c27 100644 --- a/packages/types/src/project-config.ts +++ b/packages/types/src/project-config.ts @@ -2,8 +2,8 @@ /* eslint-disable */ -import type { PartialTaskConfig, PlatformType, TaskConfig } from './tasks-config'; import type { UnresolvedVersionSpec } from './toolchain-config'; +import type { PartialTaskConfig, PlatformType, TaskConfig } from './tasks-config'; /** The scope and or relationship of the dependency. */ export type DependencyScope = 'build' | 'development' | 'peer' | 'production' | 'root'; From ac8357135d37a41b1f7db982d7af006c997141df Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:18:09 +0200 Subject: [PATCH 14/28] Run linter --- crates/config/src/project/overrides_config.rs | 2 +- crates/config/src/toolchain/python_config.rs | 4 +- legacy/python/lang/src/lib.rs | 3 +- legacy/python/lang/src/pip_requirements.rs | 4 +- .../platform/src/actions/install_deps.rs | 59 ++++++----- legacy/python/platform/src/actions/mod.rs | 2 +- legacy/python/platform/src/lib.rs | 2 - legacy/python/platform/src/python_platform.rs | 51 ++++------ legacy/python/platform/src/toolchain_hash.rs | 6 +- legacy/python/tool/src/python_tool.rs | 99 +++++-------------- 10 files changed, 89 insertions(+), 143 deletions(-) diff --git a/crates/config/src/project/overrides_config.rs b/crates/config/src/project/overrides_config.rs index ea8982aa8d8..0430d22900a 100644 --- a/crates/config/src/project/overrides_config.rs +++ b/crates/config/src/project/overrides_config.rs @@ -48,7 +48,7 @@ cacheable!( /// Overrides `deno` settings. #[setting(nested)] pub deno: Option, - + /// Overrides `python` settings. #[setting(nested)] pub python: Option, diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs index b4dddf50659..1dd8d8577f9 100644 --- a/crates/config/src/toolchain/python_config.rs +++ b/crates/config/src/toolchain/python_config.rs @@ -4,7 +4,6 @@ use serde::Serialize; use version_spec::UnresolvedVersionSpec; use warpgate_api::PluginLocator; - #[derive(Clone, Config, Debug, PartialEq, Serialize)] pub struct PipConfig { /// List of arguments to append to `pip install` commands. @@ -14,7 +13,6 @@ pub struct PipConfig { pub version: Option, } - #[derive(Clone, Config, Debug, PartialEq)] pub struct PythonConfig { /// Location of the WASM plugin to use for Python support. @@ -23,7 +21,7 @@ pub struct PythonConfig { /// Options for pnpm, when used as a package manager. #[setting(nested)] pub pip: Option, - + /// The relative root of the virtual environment workspace. Default to moon's /// workspace root #[setting(default = ".", skip)] diff --git a/legacy/python/lang/src/lib.rs b/legacy/python/lang/src/lib.rs index d55284cc8f7..90a0e45a446 100644 --- a/legacy/python/lang/src/lib.rs +++ b/legacy/python/lang/src/lib.rs @@ -1,5 +1,4 @@ pub mod pip_requirements; - -pub use pip_requirements::*; pub use moon_lang::LockfileDependencyVersions; +pub use pip_requirements::*; diff --git a/legacy/python/lang/src/pip_requirements.rs b/legacy/python/lang/src/pip_requirements.rs index d4c7aa3ea25..c636b7dc347 100644 --- a/legacy/python/lang/src/pip_requirements.rs +++ b/legacy/python/lang/src/pip_requirements.rs @@ -1,13 +1,13 @@ use cached::proc_macro::cached; use moon_lang::LockfileDependencyVersions; use rustc_hash::FxHashMap; -use std::path::PathBuf; use std::fs; +use std::path::PathBuf; #[cached(result)] pub fn load_lockfile_dependencies(path: PathBuf) -> miette::Result { let mut deps: LockfileDependencyVersions = FxHashMap::default(); - + let lockfile = fs::read_to_string(path.as_path()).expect("Unable to read file"); let dep = deps.entry(path.display().to_string()).or_default(); dep.push(lockfile); diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs index b60aae71574..4bdb05fd8c3 100644 --- a/legacy/python/platform/src/actions/install_deps.rs +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -14,71 +14,80 @@ pub async fn install_deps( let mut operations = vec![]; // python.exec_python(args, working_dir) - if let Some(pip_config) = &python.config.pip { - // Very first step: Activate virtual environment console .out .print_checkpoint(Checkpoint::Setup, format!("activate virtual environment"))?; let virtual_environment = &get_workspace_root().join(python.config.venv_name.clone()); - - if !virtual_environment.exists() { - let args = vec!["-m", "venv", virtual_environment.as_os_str().to_str().unwrap()]; + + if !virtual_environment.exists() { + let args = vec![ + "-m", + "venv", + virtual_environment.as_os_str().to_str().unwrap(), + ]; operations.push( Operation::task_execution(format!("python {} ", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, - ); + ); } - - - if let Some(pip_version) = &pip_config.version { console .out .print_checkpoint(Checkpoint::Setup, format!("install pip {pip_version}"))?; - + let p_version: String = if pip_version.is_latest() { format!("pip") } else { - format!("pip{}", pip_version.to_owned().to_string().replace("~", "~=")) + format!( + "pip{}", + pip_version.to_owned().to_string().replace("~", "~=") + ) }; let args = vec!["-m", "pip", "install", "-U", &p_version]; -// #"--quiet", + // #"--quiet", operations.push( Operation::task_execution(format!(" {} ", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, - ); + ); } - - - - if let Some(req) = find_requirements_txt(working_dir, &get_workspace_root()) { - console - .out - .print_checkpoint(Checkpoint::Setup, format!("pip dependencies from {}", req.as_os_str().to_str().unwrap()))?; + console.out.print_checkpoint( + Checkpoint::Setup, + format!( + "pip dependencies from {}", + req.as_os_str().to_str().unwrap() + ), + )?; let mut args = vec!["-m", "pip", "install"]; // #, "--quiet" if pip_config.install_args.is_some() { - args.extend(pip_config.install_args.as_ref().unwrap().iter().map(|c| c.as_str())); + args.extend( + pip_config + .install_args + .as_ref() + .unwrap() + .iter() + .map(|c| c.as_str()), + ); } - + args.extend(["-r", req.as_os_str().to_str().unwrap()]); - + operations.push( Operation::task_execution(format!(" {}", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, - ); + ); } } Ok(operations) -} \ No newline at end of file +} diff --git a/legacy/python/platform/src/actions/mod.rs b/legacy/python/platform/src/actions/mod.rs index b7b5cf56da6..31a65cfdfdf 100644 --- a/legacy/python/platform/src/actions/mod.rs +++ b/legacy/python/platform/src/actions/mod.rs @@ -1,3 +1,3 @@ mod install_deps; -pub use install_deps::*; \ No newline at end of file +pub use install_deps::*; diff --git a/legacy/python/platform/src/lib.rs b/legacy/python/platform/src/lib.rs index bb487c34931..d08d7820a8c 100644 --- a/legacy/python/platform/src/lib.rs +++ b/legacy/python/platform/src/lib.rs @@ -4,7 +4,6 @@ mod python_platform; mod target_hash; mod toolchain_hash; - pub use python_platform::*; use starbase_utils::fs; @@ -13,4 +12,3 @@ use std::path::{Path, PathBuf}; fn find_requirements_txt(starting_dir: &Path, workspace_root: &Path) -> Option { fs::find_upwards_until("requirements.txt", starting_dir, workspace_root) } - diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 33a395cb007..96cf4871da1 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -1,6 +1,4 @@ -use crate::{ - actions, find_requirements_txt, toolchain_hash::PythonToolchainHash -}; +use crate::{actions, find_requirements_txt, toolchain_hash::PythonToolchainHash}; use moon_action::Operation; use moon_action_context::ActionContext; use moon_common::Id; @@ -22,7 +20,9 @@ use moon_utils::{async_trait, get_workspace_root}; use proto_core::ProtoEnvironment; use rustc_hash::FxHashMap; use std::{ - collections::BTreeMap, path::{Path, PathBuf}, sync::Arc + collections::BTreeMap, + path::{Path, PathBuf}, + sync::Arc, }; use tracing::instrument; @@ -101,7 +101,6 @@ impl Platform for PythonPlatform { // PROJECT GRAPH fn is_project_in_dependency_workspace(&self, _project_source: &str) -> miette::Result { - Ok(false) } @@ -139,7 +138,10 @@ impl Platform for PythonPlatform { } fn get_dependency_configs(&self) -> miette::Result> { - Ok(Some(("requirements.txt".to_owned(), "requirements.txt".to_owned()))) + Ok(Some(( + "requirements.txt".to_owned(), + "requirements.txt".to_owned(), + ))) } async fn setup_toolchain(&mut self) -> miette::Result<()> { @@ -163,8 +165,6 @@ impl Platform for PythonPlatform { ); } - - self.toolchain.setup(&req, &mut last_versions).await?; // info!( @@ -192,10 +192,6 @@ impl Platform for PythonPlatform { ) -> miette::Result { let req = &runtime.requirement; - - - - if !self.toolchain.has(req) { self.toolchain.register( req, @@ -207,7 +203,7 @@ impl Platform for PythonPlatform { ) .await?, ); - } + } Ok(self.toolchain.setup(req, last_versions).await?) } @@ -239,7 +235,6 @@ impl Platform for PythonPlatform { Ok(mutated_files) } - // # Lockfile or manifests have not changed since last run, skipping dependency install #[instrument(skip_all)] async fn hash_manifest_deps( @@ -248,14 +243,15 @@ impl Platform for PythonPlatform { hasher: &mut ContentHasher, _hasher_config: &HasherConfig, ) -> miette::Result<()> { - if let Some(python_version) = &self.config.version { let mut deps = BTreeMap::new(); - if let Some(pip_requirements) = find_requirements_txt(&get_workspace_root(), &get_workspace_root()) { + if let Some(pip_requirements) = + find_requirements_txt(&get_workspace_root(), &get_workspace_root()) + { deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); } - - hasher.hash_content(PythonToolchainHash { + + hasher.hash_content(PythonToolchainHash { version: python_version.clone(), dependencies: deps, })?; @@ -272,21 +268,20 @@ impl Platform for PythonPlatform { hasher: &mut ContentHasher, _hasher_config: &HasherConfig, ) -> miette::Result<()> { - if let Some(python_version) = &self.config.version { let mut deps = BTreeMap::new(); - if let Some(pip_requirements) = find_requirements_txt(&get_workspace_root(), &get_workspace_root()) { + if let Some(pip_requirements) = + find_requirements_txt(&get_workspace_root(), &get_workspace_root()) + { deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); } - - hasher.hash_content(PythonToolchainHash { + + hasher.hash_content(PythonToolchainHash { version: python_version.clone(), dependencies: deps, })?; } - - Ok(()) } @@ -299,19 +294,15 @@ impl Platform for PythonPlatform { runtime: &Runtime, _working_dir: &Path, ) -> miette::Result { - let mut command = Command::new(&task.command); command.with_console(self.console.clone()); command.args(&task.args); - command.envs(&task.env); + command.envs(&task.env); if let Ok(python) = self.toolchain.get_for_version(&runtime.requirement) { if let Some(version) = get_proto_version_env(&python.tool) { command.env("PROTO_PYTHON_VERSION", version); - command.env( - "PATH", - prepend_path_env_var(get_python_tool_paths(&python)), - ); + command.env("PATH", prepend_path_env_var(get_python_tool_paths(&python))); } } diff --git a/legacy/python/platform/src/toolchain_hash.rs b/legacy/python/platform/src/toolchain_hash.rs index 22f0b8c376c..63df731d848 100644 --- a/legacy/python/platform/src/toolchain_hash.rs +++ b/legacy/python/platform/src/toolchain_hash.rs @@ -3,7 +3,7 @@ use moon_hash::hash_content; use std::collections::BTreeMap; hash_content!( - pub struct PythonToolchainHash { + pub struct PythonToolchainHash { pub version: UnresolvedVersionSpec, pub dependencies: BTreeMap>, } @@ -13,7 +13,7 @@ hash_content!( // pub fn new(python_version: UnresolvedVersionSpec) -> Self { // PythonToolchainHash { // version: python_version, -// dependencies: BTreeMap::new(), +// dependencies: BTreeMap::new(), // } // } -// } \ No newline at end of file +// } diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs index 98a102317e7..e88caa8ee56 100644 --- a/legacy/python/tool/src/python_tool.rs +++ b/legacy/python/tool/src/python_tool.rs @@ -1,29 +1,29 @@ use moon_config::PythonConfig; -use moon_python_lang::pip_requirements::{load_lockfile_dependencies}; -use moon_python_lang::{LockfileDependencyVersions}; use moon_console::{Checkpoint, Console}; +use moon_python_lang::pip_requirements::load_lockfile_dependencies; +use moon_python_lang::LockfileDependencyVersions; // use moon_logger::debug; -use moon_utils::get_workspace_root; +use moon_logger::{debug, map_list}; use moon_process::Command; use moon_tool::{ - async_trait, get_proto_env_vars, get_proto_paths, get_proto_version_env, load_tool_plugin, prepend_path_env_var, use_global_tool_on_path, DependencyManager, Tool + async_trait, get_proto_env_vars, get_proto_paths, get_proto_version_env, load_tool_plugin, + prepend_path_env_var, use_global_tool_on_path, DependencyManager, Tool, }; use moon_toolchain::RuntimeReq; +use moon_utils::get_workspace_root; use proto_core::flow::install::InstallOptions; use proto_core::{Id, ProtoEnvironment, Tool as ProtoTool, UnresolvedVersionSpec}; use rustc_hash::FxHashMap; +use starbase_styles::color; +use starbase_utils::fs; use std::env; use std::path::PathBuf; use std::sync::Arc; use std::{ffi::OsStr, path::Path}; -use starbase_utils::fs; use tracing::instrument; -use starbase_styles::color; -use moon_logger::{debug, map_list}; const LOG_TARGET: &str = "moon:python-tool"; - pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { // let mut paths = get_proto_paths(proto_env); // let mut paths:Vec = []; @@ -31,7 +31,7 @@ pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { // let mut python_command = "python"; let venv_python = &get_workspace_root().join(python_tool.config.venv_name.clone()); - + let paths; if venv_python.exists() { @@ -39,37 +39,19 @@ pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { } else { // paths = python_tool.tool.get_globals_dirs() // .iter() - // .cloned() + // .cloned() // .collect::>(); paths = get_proto_paths(&python_tool.proto_env); } - // for p in python_tool.tool.get_globals_dirs().iter() { - // debug!( - // target: LOG_TARGET, - // "Proto Env {} ", - // p.clone().display(), - // ); - // } - - - - - debug!( target: LOG_TARGET, "Proto Env {} ", - map_list(&paths, |c| color::label(c.display().to_string())), + map_list(&paths, |c| color::label(c.display().to_string())), ); paths } -// pub fn get_python_env_paths(proto_env: &ProtoEnvironment) -> Vec { -// let paths = get_proto_paths(proto_env); - -// paths -// } - pub struct PythonTool { pub config: PythonConfig, @@ -118,19 +100,10 @@ impl PythonTool { I: IntoIterator, S: AsRef, { - - // Check if venv is already created. - - - - Command::new("python") .args(args) .envs(get_proto_env_vars()) - .env( - "PATH", - prepend_path_env_var(get_python_tool_paths(&self)), - ) + .env("PATH", prepend_path_env_var(get_python_tool_paths(&self))) .cwd(working_dir) .with_console(self.console.clone()) .create_async() @@ -138,20 +111,7 @@ impl PythonTool { .await?; Ok(()) - - - // let mut cmd = Command::new("python"); - // cmd.with_console(self.console.clone()); - // cmd.envs(get_proto_env_vars()); - // cmd.envs(get_proto_env_vars()); - // if !self.global { - // cmd.env( - // "PATH", - // prepend_path_env_var(get_python_env_paths(&self.proto_env)), - // ); - // } } - } #[async_trait] @@ -203,33 +163,23 @@ impl Tool for PythonTool { } } self.tool.locate_globals_dirs().await?; - - - - - Ok(installed) } async fn teardown(&mut self) -> miette::Result<()> { self.tool.teardown().await?; - Ok(()) } } - #[async_trait] impl DependencyManager for PythonTool { fn create_command(&self, python: &PythonTool) -> miette::Result { let mut cmd = Command::new("python"); cmd.with_console(self.console.clone()); - cmd.envs(get_proto_env_vars()); + cmd.envs(get_proto_env_vars()); if !self.global { - cmd.env( - "PATH", - prepend_path_env_var(get_python_tool_paths(&self)), - ); + cmd.env("PATH", prepend_path_env_var(get_python_tool_paths(&self))); } if let Some(version) = get_proto_version_env(&self.tool) { @@ -284,23 +234,24 @@ impl DependencyManager for PythonTool { log: bool, ) -> miette::Result<()> { let mut cmd = self.create_command(python)?; - - // TODO: DONT KNOW IF CORRECT LOCATION, BECAUSE IT IS HANDLING THE TOOL INSTALLATION - if let Some(pip_config) = &self.config.pip { + if let Some(pip_config) = &self.config.pip { cmd.args(["install"]) - // .args(&args) - .cwd(working_dir) - .set_print_command(log); + .cwd(working_dir) + .set_print_command(log); //TODO: only read from root, but ready for sub virtual environments - if let Some(requirements_path) = fs::find_upwards_until("requirements.txt", get_workspace_root(), get_workspace_root()) { - cmd.args(["-r", &requirements_path.as_os_str().to_str().unwrap()]); + if let Some(requirements_path) = fs::find_upwards_until( + "requirements.txt", + get_workspace_root(), + get_workspace_root(), + ) { + cmd.args(["-r", &requirements_path.as_os_str().to_str().unwrap()]); } if let Some(install_args) = &pip_config.install_args { - cmd.args(install_args); + cmd.args(install_args); } - + let mut cmd = cmd.create_async(); if env::var("MOON_TEST_HIDE_INSTALL_OUTPUT").is_ok() { cmd.exec_capture_output().await?; From 16d5038db53888ccce024f817cef6437c3a452a0 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:00:15 +0200 Subject: [PATCH 15/28] run pip with install args --- .../platform/src/actions/install_deps.rs | 38 ++++++++++++++++++- legacy/python/platform/src/python_platform.rs | 3 +- legacy/python/tool/src/python_tool.rs | 5 ++- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs index 4bdb05fd8c3..8c2c20129c5 100644 --- a/legacy/python/platform/src/actions/install_deps.rs +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -57,7 +57,43 @@ pub async fn install_deps( ); } - if let Some(req) = find_requirements_txt(working_dir, &get_workspace_root()) { + let requirements_path = find_requirements_txt(working_dir, &get_workspace_root()); + + if let Some(install_args) = &pip_config.install_args { + if install_args.iter().any(|x| !x.starts_with("-")) { + if let Some(_) = requirements_path { + console.out.print_checkpoint( + Checkpoint::Setup, + "Skip installation via install args, found requirements.txt additional.", + )?; + } else { + console.out.print_checkpoint( + Checkpoint::Setup, + "pip dependencies from install args", + )?; + + let mut args = vec!["-m", "pip", "install"]; + if pip_config.install_args.is_some() { + args.extend( + pip_config + .install_args + .as_ref() + .unwrap() + .iter() + .map(|c| c.as_str()), + ); + } + + operations.push( + Operation::task_execution(format!(" {}", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); + } + } + } + + if let Some(req) = requirements_path { console.out.print_checkpoint( Checkpoint::Setup, format!( diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 96cf4871da1..45692908179 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -101,7 +101,8 @@ impl Platform for PythonPlatform { // PROJECT GRAPH fn is_project_in_dependency_workspace(&self, _project_source: &str) -> miette::Result { - Ok(false) + // Single version policy / only a root package.json + Ok(true) } #[instrument(skip_all)] diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs index e88caa8ee56..fc6ce537131 100644 --- a/legacy/python/tool/src/python_tool.rs +++ b/legacy/python/tool/src/python_tool.rs @@ -35,7 +35,10 @@ pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { let paths; if venv_python.exists() { - paths = vec![venv_python.join("Scripts").clone()]; + paths = vec![ + venv_python.join("Scripts").clone(), + venv_python.join("bin").clone(), + ]; } else { // paths = python_tool.tool.get_globals_dirs() // .iter() From 4fe41c33d47505d5399b1796f47ea69f071d24e9 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:35:34 +0100 Subject: [PATCH 16/28] Add Tests and re-work based on discussions --- crates/cli/tests/run_python_test.rs | 63 ++++++++++ ...thon_test__runs_install_deps_via_args.snap | 12 ++ ...run_python_test__runs_standard_script.snap | 11 ++ crates/config/src/language_platform.rs | 1 + crates/config/src/toolchain/python_config.rs | 9 +- .../inheritance/files/tasks/python.yml | 3 + .../tests/inherited_tasks_config_test.rs | 33 ++++++ crates/config/tests/toolchain_config_test.rs | 1 + .../__fixtures__/builder/platforms/moon.yml | 4 + crates/toolchain/src/detect/task_platform.rs | 11 ++ legacy/core/test-utils/src/configs.rs | 42 +++++++ .../platform/src/actions/install_deps.rs | 101 ++++------------ legacy/python/platform/src/actions/mod.rs | 2 + .../python/platform/src/actions/setup_tool.rs | 50 ++++++++ legacy/python/platform/src/python_platform.rs | 25 ++-- legacy/python/tool/src/python_tool.rs | 112 +----------------- packages/types/src/toolchain-config.ts | 4 +- tests/fixtures/python/base/moon.yml | 12 ++ website/static/schemas/toolchain.json | 2 +- 19 files changed, 285 insertions(+), 213 deletions(-) create mode 100644 crates/cli/tests/run_python_test.rs create mode 100644 crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap create mode 100644 crates/cli/tests/snapshots/run_python_test__runs_standard_script.snap create mode 100644 crates/config/tests/__fixtures__/inheritance/files/tasks/python.yml create mode 100644 legacy/python/platform/src/actions/setup_tool.rs create mode 100644 tests/fixtures/python/base/moon.yml diff --git a/crates/cli/tests/run_python_test.rs b/crates/cli/tests/run_python_test.rs new file mode 100644 index 00000000000..ac82f32816c --- /dev/null +++ b/crates/cli/tests/run_python_test.rs @@ -0,0 +1,63 @@ +use moon_config::{PartialPipConfig, PartialPythonConfig}; +use moon_test_utils::{ + assert_snapshot, create_sandbox_with_config, get_python_fixture_configs, Sandbox, +}; +use proto_core::UnresolvedVersionSpec; + +fn python_sandbox(config: PartialPythonConfig) -> Sandbox { + python_sandbox_with_config(|_| {}, config) +} + +fn python_sandbox_with_config(callback: C, config: PartialPythonConfig) -> Sandbox +where + C: FnOnce(&mut PartialPythonConfig), +{ + let (workspace_config, mut toolchain_config, tasks_config) = get_python_fixture_configs(); + + toolchain_config.python = Some(config); + + if let Some(python_config) = &mut toolchain_config.python { + callback(python_config); + } + + let sandbox = create_sandbox_with_config( + "python", + Some(workspace_config), + Some(toolchain_config), + Some(tasks_config), + ); + + sandbox.enable_git(); + sandbox +} + +#[test] +fn runs_standard_script() { + let sandbox = python_sandbox(PartialPythonConfig { + version: Some(UnresolvedVersionSpec::parse("3.11.10").unwrap()), + ..PartialPythonConfig::default() + }); + let assert = sandbox.run_moon(|cmd| { + cmd.arg("run").arg("python:standard"); + }); + + assert_snapshot!(assert.output()); +} + +#[test] +fn runs_install_deps_via_args() { + let sandbox = python_sandbox(PartialPythonConfig { + version: Some(UnresolvedVersionSpec::parse("3.11.10").unwrap()), + pip: Some(PartialPipConfig { + version: Some(UnresolvedVersionSpec::parse("latest").unwrap()), + install_args: Some(vec!["poetry==1.8.4".to_string()]), + ..PartialPipConfig::default() + }), + ..PartialPythonConfig::default() + }); + let assert = sandbox.run_moon(|cmd| { + cmd.arg("run").arg("python:poetry"); + }); + + assert_snapshot!(assert.output()); +} diff --git a/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap b/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap new file mode 100644 index 00000000000..3f7391f5497 --- /dev/null +++ b/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap @@ -0,0 +1,12 @@ +--- +source: crates/cli/tests/run_python_test.rs +assertion_line: 62 +expression: assert.output() +--- +▪▪▪▪ Install pip dependencies from install args +▪▪▪▪ python:poetry +Poetry (version 1.8.4) +▪▪▪▪ python:poetry (100ms) + +Tasks: 1 completed + Time: 100ms diff --git a/crates/cli/tests/snapshots/run_python_test__runs_standard_script.snap b/crates/cli/tests/snapshots/run_python_test__runs_standard_script.snap new file mode 100644 index 00000000000..5873dd9d926 --- /dev/null +++ b/crates/cli/tests/snapshots/run_python_test__runs_standard_script.snap @@ -0,0 +1,11 @@ +--- +source: crates/cli/tests/run_python_test.rs +assertion_line: 38 +expression: assert.output() +--- +▪▪▪▪ python:standard +Python 3.11.10 +▪▪▪▪ python:standard (100ms) + +Tasks: 1 completed + Time: 100ms diff --git a/crates/config/src/language_platform.rs b/crates/config/src/language_platform.rs index 2c9377b7fd7..d291c8044a8 100644 --- a/crates/config/src/language_platform.rs +++ b/crates/config/src/language_platform.rs @@ -104,6 +104,7 @@ mod tests { assert_eq!(LanguageType::Go.to_string(), "go"); assert_eq!(LanguageType::JavaScript.to_string(), "javascript"); assert_eq!(LanguageType::Ruby.to_string(), "ruby"); + assert_eq!(LanguageType::Python.to_string(), "python"); assert_eq!(LanguageType::Unknown.to_string(), "unknown"); assert_eq!(LanguageType::Other(Id::raw("dotnet")).to_string(), "dotnet"); } diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs index 1dd8d8577f9..74eff708e6f 100644 --- a/crates/config/src/toolchain/python_config.rs +++ b/crates/config/src/toolchain/python_config.rs @@ -9,7 +9,7 @@ pub struct PipConfig { /// List of arguments to append to `pip install` commands. pub install_args: Option>, - /// The version of pip to download, install, and run `pip` tasks with. + /// The version of pip to download, install, and run `pip` tasks with. pub version: Option, } @@ -18,15 +18,10 @@ pub struct PythonConfig { /// Location of the WASM plugin to use for Python support. pub plugin: Option, - /// Options for pnpm, when used as a package manager. + /// Options for pip, when used as a package manager. #[setting(nested)] pub pip: Option, - /// The relative root of the virtual environment workspace. Default to moon's - /// workspace root - #[setting(default = ".", skip)] - pub venv_root: String, - #[setting(default = ".venv", skip)] pub venv_name: String, diff --git a/crates/config/tests/__fixtures__/inheritance/files/tasks/python.yml b/crates/config/tests/__fixtures__/inheritance/files/tasks/python.yml new file mode 100644 index 00000000000..02c17dc9145 --- /dev/null +++ b/crates/config/tests/__fixtures__/inheritance/files/tasks/python.yml @@ -0,0 +1,3 @@ +tasks: + python: + command: python diff --git a/crates/config/tests/inherited_tasks_config_test.rs b/crates/config/tests/inherited_tasks_config_test.rs index 90c37987536..d7aa6ce5dc4 100644 --- a/crates/config/tests/inherited_tasks_config_test.rs +++ b/crates/config/tests/inherited_tasks_config_test.rs @@ -524,6 +524,7 @@ mod task_manager { "javascript-tool", "kotlin", "node", + "python", "node-application", "node-library", "rust", @@ -792,6 +793,38 @@ mod task_manager { ); } + #[test] + fn creates_python_config() { + let sandbox = create_sandbox("inheritance/files"); + let manager = load_manager_from_root(sandbox.path(), sandbox.path()).unwrap(); + + let config = manager + .get_inherited_config( + &PlatformType::System, + &LanguageType::Python, + &StackType::Frontend, + &ProjectType::Library, + &[], + ) + .unwrap(); + + assert_eq!( + config.config.tasks, + BTreeMap::from_iter([ + ( + Id::raw("global"), + stub_task("global", PlatformType::Unknown) + ), + (Id::raw("python"), stub_task("python", PlatformType::System)), + ]), + ); + + assert_eq!( + config.layers.keys().collect::>(), + vec!["tasks.yml", "tasks/python.yml",] + ); + } + #[test] fn creates_js_config_via_bun() { let sandbox = create_sandbox("inheritance/files"); diff --git a/crates/config/tests/toolchain_config_test.rs b/crates/config/tests/toolchain_config_test.rs index b3440139879..19562ab1fdc 100644 --- a/crates/config/tests/toolchain_config_test.rs +++ b/crates/config/tests/toolchain_config_test.rs @@ -48,6 +48,7 @@ mod toolchain_config { assert!(config.deno.is_none()); assert!(config.node.is_none()); + assert!(config.python.is_none()); assert!(config.rust.is_none()); assert!(config.typescript.is_none()); } diff --git a/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml b/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml index fb3ad2ba314..91935a27e17 100644 --- a/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml +++ b/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml @@ -9,6 +9,10 @@ tasks: command: bun deno-via-cmd: command: deno + python: + platform: python + python-via-cmd: + command: python node: platform: node node-via-cmd: diff --git a/crates/toolchain/src/detect/task_platform.rs b/crates/toolchain/src/detect/task_platform.rs index 30df9a79a6e..d294942854d 100644 --- a/crates/toolchain/src/detect/task_platform.rs +++ b/crates/toolchain/src/detect/task_platform.rs @@ -4,6 +4,7 @@ use std::sync::OnceLock; pub static BUN_COMMANDS: OnceLock = OnceLock::new(); pub static DENO_COMMANDS: OnceLock = OnceLock::new(); +pub static PYTHON_COMMANDS: OnceLock = OnceLock::new(); pub static RUST_COMMANDS: OnceLock = OnceLock::new(); pub static NODE_COMMANDS: OnceLock = OnceLock::new(); pub static UNIX_SYSTEM_COMMANDS: OnceLock = OnceLock::new(); @@ -17,6 +18,9 @@ fn use_platform_if_enabled( PlatformType::Bun if enabled_platforms.contains(&PlatformType::Bun) => return platform, PlatformType::Deno if enabled_platforms.contains(&PlatformType::Deno) => return platform, PlatformType::Node if enabled_platforms.contains(&PlatformType::Node) => return platform, + PlatformType::Python if enabled_platforms.contains(&PlatformType::Python) => { + return platform + } PlatformType::Rust if enabled_platforms.contains(&PlatformType::Rust) => return platform, _ => {} }; @@ -55,6 +59,13 @@ pub fn detect_task_platform(command: &str, enabled_platforms: &[PlatformType]) - return use_platform_if_enabled(PlatformType::Deno, enabled_platforms); } + if PYTHON_COMMANDS + .get_or_init(|| Regex::new("^(python|python3|pip|pip3)$").unwrap()) + .is_match(command) + { + return use_platform_if_enabled(PlatformType::Python, enabled_platforms); + } + if RUST_COMMANDS .get_or_init(|| Regex::new("^(rust-|rustc|rustdoc|rustfmt|rustup|cargo)").unwrap()) .is_match(command) diff --git a/legacy/core/test-utils/src/configs.rs b/legacy/core/test-utils/src/configs.rs index 7e5fe2ab892..1cb626f1612 100644 --- a/legacy/core/test-utils/src/configs.rs +++ b/legacy/core/test-utils/src/configs.rs @@ -484,6 +484,48 @@ pub fn get_node_fixture_configs() -> ( (workspace_config, toolchain_config, tasks_config) } +pub fn get_python_fixture_configs() -> ( + PartialWorkspaceConfig, + PartialToolchainConfig, + PartialInheritedTasksConfig, +) { + let workspace_config = PartialWorkspaceConfig { + projects: Some(PartialWorkspaceProjects::Sources(FxHashMap::from_iter([ + ("python".try_into().unwrap(), "base".to_owned()), + ]))), + ..PartialWorkspaceConfig::default() + }; + + let mut toolchain_config = get_default_toolchain(); + toolchain_config.python = Some(PartialPythonConfig { + version: Some(UnresolvedVersionSpec::parse("3.11.10").unwrap()), + ..PartialPythonConfig::default() + }); + + let tasks_config = PartialInheritedTasksConfig { + tasks: Some(BTreeMap::from_iter([ + ( + "version".try_into().unwrap(), + PartialTaskConfig { + command: Some(PartialTaskArgs::String("python".into())), + args: Some(PartialTaskArgs::String("--version".into())), + ..PartialTaskConfig::default() + }, + ), + ( + "noop".try_into().unwrap(), + PartialTaskConfig { + command: Some(PartialTaskArgs::String("noop".into())), + ..PartialTaskConfig::default() + }, + ), + ])), + ..PartialInheritedTasksConfig::default() + }; + + (workspace_config, toolchain_config, tasks_config) +} + pub fn get_node_depman_fixture_configs( depman: &str, ) -> ( diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs index 8c2c20129c5..180d0df5ac4 100644 --- a/legacy/python/platform/src/actions/install_deps.rs +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -1,4 +1,5 @@ use moon_action::Operation; +use moon_common::color; use moon_console::{Checkpoint, Console}; use moon_python_tool::PythonTool; use moon_utils::get_workspace_root; @@ -13,97 +14,43 @@ pub async fn install_deps( ) -> miette::Result> { let mut operations = vec![]; - // python.exec_python(args, working_dir) - if let Some(pip_config) = &python.config.pip { - // Very first step: Activate virtual environment - console - .out - .print_checkpoint(Checkpoint::Setup, format!("activate virtual environment"))?; - let virtual_environment = &get_workspace_root().join(python.config.venv_name.clone()); - - if !virtual_environment.exists() { - let args = vec![ - "-m", - "venv", - virtual_environment.as_os_str().to_str().unwrap(), - ]; - operations.push( - Operation::task_execution(format!("python {} ", args.join(" "))) - .track_async(|| python.exec_python(args, working_dir)) - .await?, - ); - } - - if let Some(pip_version) = &pip_config.version { - console - .out - .print_checkpoint(Checkpoint::Setup, format!("install pip {pip_version}"))?; - - let p_version: String = if pip_version.is_latest() { - format!("pip") - } else { - format!( - "pip{}", - pip_version.to_owned().to_string().replace("~", "~=") - ) - }; - let args = vec!["-m", "pip", "install", "-U", &p_version]; - // #"--quiet", - operations.push( - Operation::task_execution(format!(" {} ", args.join(" "))) - .track_async(|| python.exec_python(args, working_dir)) - .await?, - ); - } - let requirements_path = find_requirements_txt(working_dir, &get_workspace_root()); if let Some(install_args) = &pip_config.install_args { - if install_args.iter().any(|x| !x.starts_with("-")) { - if let Some(_) = requirements_path { - console.out.print_checkpoint( - Checkpoint::Setup, - "Skip installation via install args, found requirements.txt additional.", - )?; - } else { - console.out.print_checkpoint( - Checkpoint::Setup, - "pip dependencies from install args", - )?; - - let mut args = vec!["-m", "pip", "install"]; - if pip_config.install_args.is_some() { - args.extend( - pip_config - .install_args - .as_ref() - .unwrap() - .iter() - .map(|c| c.as_str()), - ); - } - - operations.push( - Operation::task_execution(format!(" {}", args.join(" "))) - .track_async(|| python.exec_python(args, working_dir)) - .await?, + if install_args.iter().any(|x| !x.starts_with("-")) && requirements_path.is_none() { + console.out.print_checkpoint( + Checkpoint::Setup, + "Install pip dependencies from install args", + )?; + + let mut args = vec!["-m", "pip", "install", "--quiet"]; + if pip_config.install_args.is_some() { + args.extend( + pip_config + .install_args + .as_ref() + .unwrap() + .iter() + .map(|c| c.as_str()), ); } + + operations.push( + Operation::task_execution(format!(" {}", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); } } if let Some(req) = requirements_path { console.out.print_checkpoint( Checkpoint::Setup, - format!( - "pip dependencies from {}", - req.as_os_str().to_str().unwrap() - ), + format!("Install pip dependencies from {}", color::path(&req)), )?; - let mut args = vec!["-m", "pip", "install"]; - // #, "--quiet" + let mut args = vec!["-m", "pip", "install", "--quiet"]; if pip_config.install_args.is_some() { args.extend( pip_config diff --git a/legacy/python/platform/src/actions/mod.rs b/legacy/python/platform/src/actions/mod.rs index 31a65cfdfdf..b4bae854747 100644 --- a/legacy/python/platform/src/actions/mod.rs +++ b/legacy/python/platform/src/actions/mod.rs @@ -1,3 +1,5 @@ mod install_deps; +mod setup_tool; pub use install_deps::*; +pub use setup_tool::*; diff --git a/legacy/python/platform/src/actions/setup_tool.rs b/legacy/python/platform/src/actions/setup_tool.rs new file mode 100644 index 00000000000..cdb4281dab4 --- /dev/null +++ b/legacy/python/platform/src/actions/setup_tool.rs @@ -0,0 +1,50 @@ +use moon_action::Operation; +use moon_python_tool::PythonTool; +use starbase_utils::fs; +use std::path::Path; + +pub async fn setup_tool(python: &PythonTool, workspace_root: &Path) -> miette::Result<()> { + let mut operations = vec![]; + + if let Some(pip_config) = &python.config.pip { + let virtual_environment = &workspace_root.join(python.config.venv_name.clone()); + + if !virtual_environment.exists() { + let args = vec![ + "-m", + "venv", + virtual_environment.as_os_str().to_str().unwrap(), + ]; + operations.push( + Operation::task_execution(format!("python {} ", args.join(" "))) + .track_async(|| python.exec_python(args, workspace_root)) + .await?, + ); + } + + if let Some(pip_version) = &pip_config.version { + let p_version: String = if pip_version.is_latest() { + format!("pip") + } else { + format!( + "pip{}", + pip_version.to_owned().to_string().replace("~", "~=") + ) + }; + let args = vec!["-m", "pip", "install", "--quiet", "-U", &p_version]; + operations.push( + Operation::task_execution(format!(" {} ", args.join(" "))) + .track_async(|| python.exec_python(args, workspace_root)) + .await?, + ); + } + } + + // Create version file + if let Some(python_version) = &python.config.version { + let rc_path = workspace_root.join(".python-version".to_string()); + fs::write_file(&rc_path, python_version.to_string())?; + } + + Ok(()) +} diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 45692908179..c752dc171fb 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -8,7 +8,7 @@ use moon_config::{ }; use moon_console::Console; use moon_hash::ContentHasher; -// use moon_logger::debug; +use moon_logger::info; use moon_platform::{Platform, Runtime, RuntimeReq}; use moon_process::Command; use moon_project::Project; @@ -26,8 +26,6 @@ use std::{ }; use tracing::instrument; -// const LOG_TARGET: &str = "moon:python-platform"; - pub struct PythonPlatform { pub config: PythonConfig, @@ -101,7 +99,7 @@ impl Platform for PythonPlatform { // PROJECT GRAPH fn is_project_in_dependency_workspace(&self, _project_source: &str) -> miette::Result { - // Single version policy / only a root package.json + // Single version policy / only a root requirements.txt Ok(true) } @@ -111,12 +109,7 @@ impl Platform for PythonPlatform { _projects_list: &ProjectsSourcesList, _aliases_list: &mut ProjectsAliasesList, ) -> miette::Result<()> { - // Extract the alias from the Cargo project relative to the lockfile - // for (id, source) in projects_list { - // let project_root = source.to_path(&self.workspace_root); - - // } - + // Not supported Ok(()) } @@ -168,11 +161,6 @@ impl Platform for PythonPlatform { self.toolchain.setup(&req, &mut last_versions).await?; - // info!( - // target: LOG_TARGET, - // "Setup toolchain" - // ); - Ok(()) } @@ -205,7 +193,12 @@ impl Platform for PythonPlatform { .await?, ); } - Ok(self.toolchain.setup(req, last_versions).await?) + + let installed = self.toolchain.setup(req, last_versions).await?; + + actions::setup_tool(self.toolchain.get_for_version(req)?, &self.workspace_root).await?; + + Ok(installed) } #[instrument(skip_all)] diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs index fc6ce537131..1a13ea556b7 100644 --- a/legacy/python/tool/src/python_tool.rs +++ b/legacy/python/tool/src/python_tool.rs @@ -1,13 +1,10 @@ use moon_config::PythonConfig; use moon_console::{Checkpoint, Console}; -use moon_python_lang::pip_requirements::load_lockfile_dependencies; -use moon_python_lang::LockfileDependencyVersions; -// use moon_logger::debug; use moon_logger::{debug, map_list}; use moon_process::Command; use moon_tool::{ - async_trait, get_proto_env_vars, get_proto_paths, get_proto_version_env, load_tool_plugin, - prepend_path_env_var, use_global_tool_on_path, DependencyManager, Tool, + async_trait, get_proto_env_vars, get_proto_paths, load_tool_plugin, prepend_path_env_var, + use_global_tool_on_path, Tool, }; use moon_toolchain::RuntimeReq; use moon_utils::get_workspace_root; @@ -15,8 +12,6 @@ use proto_core::flow::install::InstallOptions; use proto_core::{Id, ProtoEnvironment, Tool as ProtoTool, UnresolvedVersionSpec}; use rustc_hash::FxHashMap; use starbase_styles::color; -use starbase_utils::fs; -use std::env; use std::path::PathBuf; use std::sync::Arc; use std::{ffi::OsStr, path::Path}; @@ -174,106 +169,3 @@ impl Tool for PythonTool { Ok(()) } } - -#[async_trait] -impl DependencyManager for PythonTool { - fn create_command(&self, python: &PythonTool) -> miette::Result { - let mut cmd = Command::new("python"); - cmd.with_console(self.console.clone()); - cmd.envs(get_proto_env_vars()); - if !self.global { - cmd.env("PATH", prepend_path_env_var(get_python_tool_paths(&self))); - } - - if let Some(version) = get_proto_version_env(&self.tool) { - cmd.env("PROTO_PYTHON_VERSION", version); - } - - if let Some(version) = get_proto_version_env(&python.tool) { - cmd.env("PROTO_PYTHON_VERSION", version); - } - - Ok(cmd) - } - - #[instrument(skip_all)] - async fn dedupe_dependencies( - &self, - _python: &PythonTool, - _working_dir: &Path, - _log: bool, - ) -> miette::Result<()> { - // Not supported! - - Ok(()) - } - - fn get_lock_filename(&self) -> String { - String::from("requirements.txt") - } - - fn get_manifest_filename(&self) -> String { - String::from("requirements.txt") - } - - #[instrument(skip_all)] - async fn get_resolved_dependencies( - &self, - project_root: &Path, - ) -> miette::Result { - let Some(lockfile_path) = - fs::find_upwards_until("requirements.txt", project_root, get_workspace_root()) - else { - return Ok(FxHashMap::default()); - }; - - Ok(load_lockfile_dependencies(lockfile_path)?) - } - #[instrument(skip_all)] - async fn install_dependencies( - &self, - python: &PythonTool, - working_dir: &Path, - log: bool, - ) -> miette::Result<()> { - let mut cmd = self.create_command(python)?; - - if let Some(pip_config) = &self.config.pip { - cmd.args(["install"]) - .cwd(working_dir) - .set_print_command(log); - - //TODO: only read from root, but ready for sub virtual environments - if let Some(requirements_path) = fs::find_upwards_until( - "requirements.txt", - get_workspace_root(), - get_workspace_root(), - ) { - cmd.args(["-r", &requirements_path.as_os_str().to_str().unwrap()]); - } - if let Some(install_args) = &pip_config.install_args { - cmd.args(install_args); - } - - let mut cmd = cmd.create_async(); - if env::var("MOON_TEST_HIDE_INSTALL_OUTPUT").is_ok() { - cmd.exec_capture_output().await?; - } else { - cmd.exec_stream_output().await?; - } - } - - Ok(()) - } - - #[instrument(skip_all)] - async fn install_focused_dependencies( - &self, - _python: &PythonTool, - _packages: &[String], - _production_only: bool, - ) -> miette::Result<()> { - // TODO: Implement for docker purposes - Ok(()) - } -} diff --git a/packages/types/src/toolchain-config.ts b/packages/types/src/toolchain-config.ts index b4002fb7f37..9fc148e8255 100644 --- a/packages/types/src/toolchain-config.ts +++ b/packages/types/src/toolchain-config.ts @@ -258,7 +258,7 @@ export interface PipConfig { } export interface PythonConfig { - /** Options for pnpm, when used as a package manager. */ + /** Options for pip, when used as a package manager. */ pip: PipConfig | null; /** Location of the WASM plugin to use for Python support. */ plugin: PluginLocator | null; @@ -637,7 +637,7 @@ export interface PartialPipConfig { } export interface PartialPythonConfig { - /** Options for pnpm, when used as a package manager. */ + /** Options for pip, when used as a package manager. */ pip?: PartialPipConfig | null; /** Location of the WASM plugin to use for Python support. */ plugin?: PluginLocator | null; diff --git a/tests/fixtures/python/base/moon.yml b/tests/fixtures/python/base/moon.yml new file mode 100644 index 00000000000..d9bcbc7a3a3 --- /dev/null +++ b/tests/fixtures/python/base/moon.yml @@ -0,0 +1,12 @@ +language: python + +tasks: + standard: + command: python + args: + - --version + + poetry: + command: poetry + args: + - --version \ No newline at end of file diff --git a/website/static/schemas/toolchain.json b/website/static/schemas/toolchain.json index 2f56f231ab5..8686ed8dcf1 100644 --- a/website/static/schemas/toolchain.json +++ b/website/static/schemas/toolchain.json @@ -665,7 +665,7 @@ "properties": { "pip": { "title": "pip", - "description": "Options for pnpm, when used as a package manager.", + "description": "Options for pip, when used as a package manager.", "anyOf": [ { "$ref": "#/definitions/PipConfig" From 0e984d4388e47219da9f8581ef81bd363c3fd724 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:35:18 +0100 Subject: [PATCH 17/28] Add documentation --- CHANGELOG.md | 9 +++ legacy/core/test-utils/src/configs.rs | 7 +- .../setup-toolchain/python/tier2.mdx | 10 ++- .../setup-toolchain/python/tier3.mdx | 12 ++-- website/docs/config/toolchain.mdx | 69 +++++++++++++++++++ 5 files changed, 92 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59c3552c17c..c3b851e1313 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +- Added Python tier 3 support. + - Will download and install Python into the toolchain when a `version` is configured. + - Will parse the `requirements.txt` to resolve and install dependencies. + - Added a `python.version` setting to `.moon/toolchain.yml`. + - Added a `toolchain.python` setting to `moon.yml`. + - Updated `moon bin` and `moon docker` commands to support Python. + ## 1.29.4 #### 🚀 Updates diff --git a/legacy/core/test-utils/src/configs.rs b/legacy/core/test-utils/src/configs.rs index 1cb626f1612..e06d71aa219 100644 --- a/legacy/core/test-utils/src/configs.rs +++ b/legacy/core/test-utils/src/configs.rs @@ -490,9 +490,10 @@ pub fn get_python_fixture_configs() -> ( PartialInheritedTasksConfig, ) { let workspace_config = PartialWorkspaceConfig { - projects: Some(PartialWorkspaceProjects::Sources(FxHashMap::from_iter([ - ("python".try_into().unwrap(), "base".to_owned()), - ]))), + projects: Some(PartialWorkspaceProjects::Sources(FxHashMap::from_iter([( + "python".try_into().unwrap(), + "base".to_owned(), + )]))), ..PartialWorkspaceConfig::default() }; diff --git a/website/docs/__partials__/setup-toolchain/python/tier2.mdx b/website/docs/__partials__/setup-toolchain/python/tier2.mdx index 21ef77ca7f2..622d4ce24b7 100644 --- a/website/docs/__partials__/setup-toolchain/python/tier2.mdx +++ b/website/docs/__partials__/setup-toolchain/python/tier2.mdx @@ -1,6 +1,4 @@ -:::warning - -Python does not implement tier 2 platform support. However, Python based tasks can still be executed -using the default system platform. - -::: +```yaml title=".moon/toolchain.yml" +python: + pip: {} +``` \ No newline at end of file diff --git a/website/docs/__partials__/setup-toolchain/python/tier3.mdx b/website/docs/__partials__/setup-toolchain/python/tier3.mdx index f95b497d0d6..902b7adb8ea 100644 --- a/website/docs/__partials__/setup-toolchain/python/tier3.mdx +++ b/website/docs/__partials__/setup-toolchain/python/tier3.mdx @@ -1,6 +1,6 @@ -:::warning - -Python does not implement tier 3 tool support. The required `python` and related binaries must exist -on `PATH`. - -::: +```yaml title=".moon/toolchain.yml" +python: + version: '3.11.10' + pip: + version: 'latest' +``` diff --git a/website/docs/config/toolchain.mdx b/website/docs/config/toolchain.mdx index 2324de2734b..0efb7ccae7b 100644 --- a/website/docs/config/toolchain.mdx +++ b/website/docs/config/toolchain.mdx @@ -721,6 +721,75 @@ Both imports can optionally be nested within a `src` directory. > This setting runs _after_ [`syncProjectReferences`](#syncprojectreferences) and will inherit any > synced references from that setting. +## Python + +## `python` + + + +Enables and configures Python. + +### `version` + + + +Defines the explicit Python toolchain +If this field is _not defined_, the global `python` binary will be used. + +```yaml title=".moon/toolchain.yml" {2} +python: + version: '3.11.10' +``` + +> Version can also be defined with [`.prototools`](../proto/config). + +### `venv_name` + + + +Defines the virtual environment name which will be created on workspace root, +project dependencies will be installed into this. Defaults to `.venv` + +```yaml title=".moon/toolchain.yml" {2} +python: + venv_name: '.my-custom-venv' +``` + +### `pip` + + + +#### `version` + + + +The `version` setting defines the explicit pip +[version specification](../concepts/toolchain#version-specification) to use. If this field is _not +defined_, the built-in library will be used. + +A list of plugins that will automatically be imported using `yarn plugin import` (Yarn 2+ only). For +performance reasons, plugins will only be imported when the Yarn version changes. + +```yaml title=".moon/toolchain.yml" {4} +python: + version: '3.11.10' + pip: + version: 'latest' +``` + +### `install_args` + + + +Customize the arguments that will be passed to the pip install command, when the +`InstallDeps` action is triggered in the pipeline. These arguments are used both locally and in CI. + +```yaml title=".moon/toolchain.yml" {3} +python: + pip: + installArgs: ['--trusted-host company.repo.com', '-i https://company.repo.com/simple'] +``` + ## Rust ## `rust` From 89d155746526bf8e9acde01986a333d997bfd419 Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Wed, 23 Oct 2024 20:41:15 -0700 Subject: [PATCH 18/28] new: Add new workspace builder. (#1697) * Add workspace crate. * Add projects loading. * Add build. * Clean up build data. * Flesh out build. * Add caching. * Add to session. * Use focused graphs. * Add workspace mocker. * Delete old graph builder. * Polish. * Fix tests. * Fix tests. --- crates/test-utils/src/workspace_mocker.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/test-utils/src/workspace_mocker.rs b/crates/test-utils/src/workspace_mocker.rs index a8e488bfdc3..e62a950feac 100644 --- a/crates/test-utils/src/workspace_mocker.rs +++ b/crates/test-utils/src/workspace_mocker.rs @@ -108,7 +108,6 @@ impl WorkspaceMocker { extend_project: Emitter::::new(), extend_project_graph: Emitter::::new(), inherited_tasks: &self.inherited_tasks, - strict_project_ids: self.workspace_config.experiments.strict_project_ids, toolchain_config: &self.toolchain_config, vcs: self.vcs.clone(), working_dir: &self.workspace_root, From c29a7dcbf2fb4e2c519aaa625a23d67786ee58c1 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Mon, 4 Nov 2024 09:36:12 +0100 Subject: [PATCH 19/28] Add last commit --- Cargo.lock | 53 +++++++++++++++ legacy/python/lang/Cargo.toml | 1 + legacy/python/lang/src/pip_requirements.rs | 28 ++++++-- .../platform/src/actions/install_deps.rs | 4 +- legacy/python/platform/src/python_platform.rs | 34 ++++++---- scripts/new-language.md | 64 +++++++++---------- 6 files changed, 131 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index adb503c192f..59458fe93e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -773,6 +773,19 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "chumsky" +version = "1.0.0-alpha.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c28d4e5dd9a9262a38b231153591da6ce1471b818233f4727985d3dd0ed93c" +dependencies = [ + "hashbrown 0.14.5", + "regex-automata 0.3.9", + "serde", + "stacker", + "unicode-ident", +] + [[package]] name = "chunked_transfer" version = "1.5.0" @@ -3767,6 +3780,7 @@ dependencies = [ "moon_lang", "moon_logger", "moon_test_utils", + "pep-508", "rustc-hash 2.0.0", "serde", "starbase_styles", @@ -4535,6 +4549,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d61c5ce1153ab5b689d0c074c4e7fc613e942dfb7dd9eea5ab202d2ad91fe361" +[[package]] +name = "pep-508" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56536b95df75cc5801a27ae2b53381d5d295fb30837be65f72916ecef5d1e4f" +dependencies = [ + "chumsky", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -5075,6 +5098,17 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.5", +] + [[package]] name = "regex-automata" version = "0.4.8" @@ -5092,6 +5126,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + [[package]] name = "regex-syntax" version = "0.8.5" @@ -5745,6 +5785,19 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stacker" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] + [[package]] name = "starbase" version = "0.8.4" diff --git a/legacy/python/lang/Cargo.toml b/legacy/python/lang/Cargo.toml index 6074c3866eb..99c7c02b7cd 100644 --- a/legacy/python/lang/Cargo.toml +++ b/legacy/python/lang/Cargo.toml @@ -11,6 +11,7 @@ cached = { workspace = true } cargo-lock = "9.0.0" cargo_toml = "0.20.4" miette = { workspace = true } +pep-508 = "0.4.0" rustc-hash = { workspace = true } serde = { workspace = true } starbase_styles = { workspace = true } diff --git a/legacy/python/lang/src/pip_requirements.rs b/legacy/python/lang/src/pip_requirements.rs index c636b7dc347..d172eb9f6ca 100644 --- a/legacy/python/lang/src/pip_requirements.rs +++ b/legacy/python/lang/src/pip_requirements.rs @@ -1,15 +1,33 @@ use cached::proc_macro::cached; use moon_lang::LockfileDependencyVersions; +use pep_508::parse; use rustc_hash::FxHashMap; -use std::fs; -use std::path::PathBuf; +use std::fs::File; +use std::io::BufRead; +use std::io; +use std::path::{Path, PathBuf}; + +fn read_lines

(filename: P) -> io::Result>> +where P: AsRef, { + let file = File::open(filename)?; + Ok(io::BufReader::new(file).lines()) +} #[cached(result)] pub fn load_lockfile_dependencies(path: PathBuf) -> miette::Result { let mut deps: LockfileDependencyVersions = FxHashMap::default(); - let lockfile = fs::read_to_string(path.as_path()).expect("Unable to read file"); - let dep = deps.entry(path.display().to_string()).or_default(); - dep.push(lockfile); + if let Ok(lines) = read_lines(&path) { + for line in lines.flatten() { + + if let Ok(parsed) = parse(&line) { + deps.entry(parsed.name.to_string()) + .and_modify(|dep| { + dep.push(line.clone()); + }); + } + } + } + Ok(deps) } diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs index 180d0df5ac4..8ea00da8796 100644 --- a/legacy/python/platform/src/actions/install_deps.rs +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -24,7 +24,7 @@ pub async fn install_deps( "Install pip dependencies from install args", )?; - let mut args = vec!["-m", "pip", "install", "--quiet"]; + let mut args = vec!["-m", "pip", "install"]; if pip_config.install_args.is_some() { args.extend( pip_config @@ -50,7 +50,7 @@ pub async fn install_deps( format!("Install pip dependencies from {}", color::path(&req)), )?; - let mut args = vec!["-m", "pip", "install", "--quiet"]; + let mut args = vec!["-m", "pip", "install"]; if pip_config.install_args.is_some() { args.extend( pip_config diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index c752dc171fb..efc6d4992de 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -1,14 +1,14 @@ use crate::{actions, find_requirements_txt, toolchain_hash::PythonToolchainHash}; use moon_action::Operation; use moon_action_context::ActionContext; -use moon_common::Id; +use moon_common::{color, Id}; use moon_config::{ HasherConfig, PlatformType, ProjectConfig, ProjectsAliasesList, ProjectsSourcesList, PythonConfig, UnresolvedVersionSpec, }; use moon_console::Console; use moon_hash::ContentHasher; -use moon_logger::info; +use moon_logger::{debug, info}; use moon_platform::{Platform, Runtime, RuntimeReq}; use moon_process::Command; use moon_project::Project; @@ -26,6 +26,9 @@ use std::{ }; use tracing::instrument; +const LOG_TARGET: &str = "moon:python-platform"; + + pub struct PythonPlatform { pub config: PythonConfig, @@ -197,6 +200,7 @@ impl Platform for PythonPlatform { let installed = self.toolchain.setup(req, last_versions).await?; actions::setup_tool(self.toolchain.get_for_version(req)?, &self.workspace_root).await?; + actions::install_deps(self.toolchain.get_for_version(req)?, &self.workspace_root, &self.console).await?; Ok(installed) } @@ -229,22 +233,20 @@ impl Platform for PythonPlatform { Ok(mutated_files) } - // # Lockfile or manifests have not changed since last run, skipping dependency install #[instrument(skip_all)] async fn hash_manifest_deps( &self, - _manifest_path: &Path, + manifest_path: &Path, hasher: &mut ContentHasher, _hasher_config: &HasherConfig, ) -> miette::Result<()> { if let Some(python_version) = &self.config.version { - let mut deps = BTreeMap::new(); - if let Some(pip_requirements) = - find_requirements_txt(&get_workspace_root(), &get_workspace_root()) - { - deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); - } - + let deps = BTreeMap::from_iter(load_lockfile_dependencies(manifest_path.to_path_buf())?); + debug!( + target: LOG_TARGET, + "HASH MANIFEST {}", + color::path(manifest_path) + ); hasher.hash_content(PythonToolchainHash { version: python_version.clone(), dependencies: deps, @@ -257,7 +259,7 @@ impl Platform for PythonPlatform { #[instrument(skip_all)] async fn hash_run_target( &self, - _project: &Project, + project: &Project, _runtime: &Runtime, hasher: &mut ContentHasher, _hasher_config: &HasherConfig, @@ -265,11 +267,15 @@ impl Platform for PythonPlatform { if let Some(python_version) = &self.config.version { let mut deps = BTreeMap::new(); if let Some(pip_requirements) = - find_requirements_txt(&get_workspace_root(), &get_workspace_root()) + find_requirements_txt(&project.root, &self.workspace_root) { deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); } - + debug!( + target: LOG_TARGET, + "HASH RUN TARGET {}", + color::path(&project.root) + ); hasher.hash_content(PythonToolchainHash { version: python_version.clone(), dependencies: deps, diff --git a/scripts/new-language.md b/scripts/new-language.md index 260b67c10af..4e76816c123 100644 --- a/scripts/new-language.md +++ b/scripts/new-language.md @@ -17,9 +17,9 @@ enum LanguageType { } ``` -- [ ] Updated enum -- [ ] Verified all `match` callsites handle the new variant -- [ ] Ran `just schemas` and updated the JSON schemas/types +- [Done ] Updated enum +- [Done ] Verified all `match` callsites handle the new variant +- [Done ] Ran `just schemas` and updated the JSON schemas/types ### Create language crate @@ -47,9 +47,9 @@ can be used as a reference. moon implements a lot of inference, detection, and automation, to avoid explicit configuration from the developer. The `moon_toolchain` handles this, and must be updated to support the new language. -- [ ] Updated `languages.rs` -- [ ] Updated `detect_language_files` -- [ ] Updated `detect_project_language` +- [done ] Updated `languages.rs` +- [done ] Updated `detect_language_files` +- [done ] Updated `detect_project_language` > The `detect_task_platform` and `detect_project_platform` can be skipped as it's required for > tier 2. @@ -58,10 +58,10 @@ the developer. The `moon_toolchain` handles this, and must be updated to support Of course this should all be tested. -- [ ] Added fixture to `crates/config/tests/__fixtures__/inheritance` -- [ ] Added fixture to `crates/project-builder/tests/__fixtures__/langs` -- [ ] Added fixture to `crates/task-builder/tests/__fixtures__/builder/platforms` -- [ ] Updated `crates/config/tests/inherited_tasks_config_test.rs` +- [Done ] Added fixture to `crates/config/tests/__fixtures__/inheritance` +- [Done ] Added fixture to `crates/project-builder/tests/__fixtures__/langs` +- [Done ] Added fixture to `crates/task-builder/tests/__fixtures__/builder/platforms` +- [Done ] Updated `crates/config/tests/inherited_tasks_config_test.rs` ### Create a pull request @@ -101,11 +101,11 @@ pub struct ToolchainConfig { } ``` -- [ ] Created language struct -- [ ] Created config template file -- [ ] Updated `ToolchainConfig` struct -- [ ] Ran `just schemas` -- [ ] Add `.prototools` support in `crates/config/src/toolchain_config.rs` +- [Done ] Created language struct +- [Done ] Created config template file +- [Done ] Updated `ToolchainConfig` struct +- [Done ] Ran `just schemas` +- [Done ] Add `.prototools` support in `crates/config/src/toolchain_config.rs` - [ ] Add tests to `crates/config/tests/toolchain_config_test.rs` ### Add variant to `PlatformType` enum in `moon_config` @@ -119,16 +119,16 @@ enum PlatformType { } ``` -- [ ] Updated enum +- [Done ] Updated enum - [ ] Verified all `match` callsites handle the new variant -- [ ] Ran `just schemas` +- [Done ] Ran `just schemas` ### Update `moon_toolchain` crate Tasks run against the platform, so we can now attempt to detect this. -- [ ] Updated `detect_task_platform` -- [ ] Updated `detect_project_platform` +- [Done ] Updated `detect_task_platform` +- [Done ] Updated `detect_project_platform` ### Create tool crate @@ -150,9 +150,9 @@ This is required _even when not using_ the toolchain, as we fallback to a global Crate must exist at `legacy//tool`. Feel free to copy an existing tool crate and update the implementation. -- [ ] Implemented `Tool` trait -- [ ] Implemented `PackageManager` trait (when applicable) -- [ ] Handled `global` binary +- [Done ] Implemented `Tool` trait +- [Done ] Implemented `PackageManager` trait (when applicable) +- [Done ] Handled `global` binary ### Create platform crate @@ -163,10 +163,10 @@ moon. Crate must exist at `legacy//platform`. Feel free to copy an existing platform crate and update the implementation. -- [ ] Implemented `Platform` trait -- [ ] Implemented manifest hashing -- [ ] Implemented target hashing -- [ ] Implemented action handlers +- [Done ] Implemented `Platform` trait +- [Done ] Implemented manifest hashing +- [Done ] Implemented target hashing +- [Done ] Implemented action handlers - [ ] Implemented project graph bridge ### Update docs @@ -235,9 +235,9 @@ When the moon tool has been integrated with proto's, we can update the platform Refer to the Node.js implementation for examples (it can mostly be copied). -- [ ] Enabled `is_toolchain_enabled` method -- [ ] Updated `get_runtime_from_config` with `version` field -- [ ] Updated `setup_toolchain`, `setup_tool`, and `teardown_toolchain` methods +- [Done ] Enabled `is_toolchain_enabled` method +- [Done ] Updated `get_runtime_from_config` with `version` field +- [Done ] Updated `setup_toolchain`, `setup_tool`, and `teardown_toolchain` methods - [ ] Updated `create_run_target_command` to use the tool instance ### Support project-level config overrides @@ -245,8 +245,8 @@ Refer to the Node.js implementation for examples (it can mostly be copied). Different projects may have different version requirements, so we need to support this through project-level toolchain overrides. -- [ ] Updated `crates/config/src/project/overrides_config.rs` -- [ ] Updated `get_runtime_from_config` in platform crate +- [Done ] Updated `crates/config/src/project/overrides_config.rs` +- [Done ] Updated `get_runtime_from_config` in platform crate ### Integrate `--profile` option @@ -258,7 +258,7 @@ variants. The `moon bin` command uses a hard-coded tool list, and is not based on the `PlatformType` or `LanguageType` enums. Because of this, tools will need to be handled manually. -- [ ] Updated `crates/app/src/commands/bin.rs` +- [Done ] Updated `crates/app/src/commands/bin.rs` ### Update `docker prune` and `docker scaffold` commands From 0a9757b6cf1e5516b2e34c307ddf63f25f2a812b Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:08:12 +0100 Subject: [PATCH 20/28] Rebase completed --- .yarn/versions/3fbef0f1.yml | 13 +++++------ legacy/python/lang/src/pip_requirements.rs | 14 ++++++------ legacy/python/platform/src/python_platform.rs | 11 +++++++--- packages/types/src/project-config.ts | 22 +++++++++++++++++-- packages/types/src/tasks-config.ts | 11 +++++++++- packages/types/src/template-config.ts | 12 ++++++++-- packages/types/src/toolchain-config.ts | 11 +++++++++- packages/types/src/workspace-config.ts | 5 ++++- 8 files changed, 74 insertions(+), 25 deletions(-) diff --git a/.yarn/versions/3fbef0f1.yml b/.yarn/versions/3fbef0f1.yml index 28fac241db0..e0dae2265d0 100644 --- a/.yarn/versions/3fbef0f1.yml +++ b/.yarn/versions/3fbef0f1.yml @@ -1,9 +1,6 @@ releases: - "@moonrepo/cli": minor - "@moonrepo/core-linux-arm64-gnu": minor - "@moonrepo/core-linux-arm64-musl": minor - "@moonrepo/core-linux-x64-gnu": minor - "@moonrepo/core-linux-x64-musl": minor - "@moonrepo/core-macos-arm64": minor - "@moonrepo/core-macos-x64": minor - "@moonrepo/core-windows-x64-msvc": minor + "@moonrepo/nx-compat": minor + "@moonrepo/report": minor + "@moonrepo/runtime": minor + "@moonrepo/types": minor + website: minor diff --git a/legacy/python/lang/src/pip_requirements.rs b/legacy/python/lang/src/pip_requirements.rs index d172eb9f6ca..91c0e4ea246 100644 --- a/legacy/python/lang/src/pip_requirements.rs +++ b/legacy/python/lang/src/pip_requirements.rs @@ -3,12 +3,14 @@ use moon_lang::LockfileDependencyVersions; use pep_508::parse; use rustc_hash::FxHashMap; use std::fs::File; -use std::io::BufRead; use std::io; +use std::io::BufRead; use std::path::{Path, PathBuf}; fn read_lines

(filename: P) -> io::Result>> -where P: AsRef, { +where + P: AsRef, +{ let file = File::open(filename)?; Ok(io::BufReader::new(file).lines()) } @@ -19,12 +21,10 @@ pub fn load_lockfile_dependencies(path: PathBuf) -> miette::Result miette::Result<()> { if let Some(python_version) = &self.config.version { - let deps = BTreeMap::from_iter(load_lockfile_dependencies(manifest_path.to_path_buf())?); + let deps = + BTreeMap::from_iter(load_lockfile_dependencies(manifest_path.to_path_buf())?); debug!( target: LOG_TARGET, "HASH MANIFEST {}", diff --git a/packages/types/src/project-config.ts b/packages/types/src/project-config.ts index 9be59614c27..b1f0331f23f 100644 --- a/packages/types/src/project-config.ts +++ b/packages/types/src/project-config.ts @@ -66,7 +66,18 @@ export interface ProjectDockerConfig { } /** Supported programming languages that each project can be written in. */ -export type LanguageType = 'bash' | 'batch' | 'go' | 'javascript' | 'php' | 'python' | 'ruby' | 'rust' | 'typescript' | 'unknown' | string; +export type LanguageType = + | 'bash' + | 'batch' + | 'go' + | 'javascript' + | 'php' + | 'python' + | 'ruby' + | 'rust' + | 'typescript' + | 'unknown' + | string; export type OwnersPaths = string[] | Record; @@ -162,7 +173,14 @@ export interface ProjectToolchainConfig { } /** The type of project, for categorizing. */ -export type ProjectType = 'application' | 'automation' | 'configuration' | 'library' | 'scaffolding' | 'tool' | 'unknown'; +export type ProjectType = + | 'application' + | 'automation' + | 'configuration' + | 'library' + | 'scaffolding' + | 'tool' + | 'unknown'; /** Controls how tasks are inherited. */ export interface ProjectWorkspaceInheritedTasksConfig { diff --git a/packages/types/src/tasks-config.ts b/packages/types/src/tasks-config.ts index 189bcabda8c..84dfbd33761 100644 --- a/packages/types/src/tasks-config.ts +++ b/packages/types/src/tasks-config.ts @@ -32,7 +32,16 @@ export type TaskOperatingSystem = 'linux' | 'macos' | 'windows'; export type TaskOutputStyle = 'buffer' | 'buffer-only-failure' | 'hash' | 'none' | 'stream'; /** A list of available shells on Unix. */ -export type TaskUnixShell = 'bash' | 'elvish' | 'fish' | 'ion' | 'murex' | 'nu' | 'pwsh' | 'xonsh' | 'zsh'; +export type TaskUnixShell = + | 'bash' + | 'elvish' + | 'fish' + | 'ion' + | 'murex' + | 'nu' + | 'pwsh' + | 'xonsh' + | 'zsh'; /** A list of available shells on Windows. */ export type TaskWindowsShell = 'bash' | 'elvish' | 'fish' | 'murex' | 'nu' | 'pwsh' | 'xonsh'; diff --git a/packages/types/src/template-config.ts b/packages/types/src/template-config.ts index 9c1ffcb55cd..db9ea77308c 100644 --- a/packages/types/src/template-config.ts +++ b/packages/types/src/template-config.ts @@ -92,7 +92,11 @@ export interface TemplateVariableStringSetting { type: 'string'; } -export type TemplateVariable = TemplateVariableBoolSetting | TemplateVariableEnumSetting | TemplateVariableNumberSetting | TemplateVariableStringSetting; +export type TemplateVariable = + | TemplateVariableBoolSetting + | TemplateVariableEnumSetting + | TemplateVariableNumberSetting + | TemplateVariableStringSetting; /** * Configures a template and its files to be scaffolded. @@ -193,7 +197,11 @@ export interface PartialTemplateVariableStringSetting { type?: 'string' | null; } -export type PartialTemplateVariable = PartialTemplateVariableBoolSetting | PartialTemplateVariableEnumSetting | PartialTemplateVariableNumberSetting | PartialTemplateVariableStringSetting; +export type PartialTemplateVariable = + | PartialTemplateVariableBoolSetting + | PartialTemplateVariableEnumSetting + | PartialTemplateVariableNumberSetting + | PartialTemplateVariableStringSetting; /** * Configures a template and its files to be scaffolded. diff --git a/packages/types/src/toolchain-config.ts b/packages/types/src/toolchain-config.ts index 9fc148e8255..c34887945e4 100644 --- a/packages/types/src/toolchain-config.ts +++ b/packages/types/src/toolchain-config.ts @@ -3,7 +3,16 @@ /* eslint-disable */ /** Formats that a `package.json` version dependency can be. */ -export type NodeVersionFormat = 'file' | 'link' | 'star' | 'version' | 'version-caret' | 'version-tilde' | 'workspace' | 'workspace-caret' | 'workspace-tilde'; +export type NodeVersionFormat = + | 'file' + | 'link' + | 'star' + | 'version' + | 'version-caret' + | 'version-tilde' + | 'workspace' + | 'workspace-caret' + | 'workspace-tilde'; export type PluginLocator = string; diff --git a/packages/types/src/workspace-config.ts b/packages/types/src/workspace-config.ts index bae3fe69deb..d03b98f62d3 100644 --- a/packages/types/src/workspace-config.ts +++ b/packages/types/src/workspace-config.ts @@ -529,7 +529,10 @@ export interface PartialWorkspaceProjectsConfig { sources?: Record | null; } -export type PartialWorkspaceProjects = PartialWorkspaceProjectsConfig | string[] | Record; +export type PartialWorkspaceProjects = + | PartialWorkspaceProjectsConfig + | string[] + | Record; /** Configures aspects of the task runner (also known as the action pipeline). */ export interface PartialRunnerConfig { From fb181779675d23f8240c8b4746d0b99fb82cd6b9 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:16:39 +0100 Subject: [PATCH 21/28] Revert new language --- scripts/new-language.md | 66 ++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/scripts/new-language.md b/scripts/new-language.md index 4e76816c123..5fd30c05969 100644 --- a/scripts/new-language.md +++ b/scripts/new-language.md @@ -17,9 +17,9 @@ enum LanguageType { } ``` -- [Done ] Updated enum -- [Done ] Verified all `match` callsites handle the new variant -- [Done ] Ran `just schemas` and updated the JSON schemas/types +- [ ] Updated enum +- [ ] Verified all `match` callsites handle the new variant +- [ ] Ran `just schemas` and updated the JSON schemas/types ### Create language crate @@ -47,9 +47,9 @@ can be used as a reference. moon implements a lot of inference, detection, and automation, to avoid explicit configuration from the developer. The `moon_toolchain` handles this, and must be updated to support the new language. -- [done ] Updated `languages.rs` -- [done ] Updated `detect_language_files` -- [done ] Updated `detect_project_language` +- [ ] Updated `languages.rs` +- [ ] Updated `detect_language_files` +- [ ] Updated `detect_project_language` > The `detect_task_platform` and `detect_project_platform` can be skipped as it's required for > tier 2. @@ -58,10 +58,10 @@ the developer. The `moon_toolchain` handles this, and must be updated to support Of course this should all be tested. -- [Done ] Added fixture to `crates/config/tests/__fixtures__/inheritance` -- [Done ] Added fixture to `crates/project-builder/tests/__fixtures__/langs` -- [Done ] Added fixture to `crates/task-builder/tests/__fixtures__/builder/platforms` -- [Done ] Updated `crates/config/tests/inherited_tasks_config_test.rs` +- [ ] Added fixture to `crates/config/tests/__fixtures__/inheritance` +- [ ] Added fixture to `crates/project-builder/tests/__fixtures__/langs` +- [ ] Added fixture to `crates/task-builder/tests/__fixtures__/builder/platforms` +- [ ] Updated `crates/config/tests/inherited_tasks_config_test.rs` ### Create a pull request @@ -101,11 +101,11 @@ pub struct ToolchainConfig { } ``` -- [Done ] Created language struct -- [Done ] Created config template file -- [Done ] Updated `ToolchainConfig` struct -- [Done ] Ran `just schemas` -- [Done ] Add `.prototools` support in `crates/config/src/toolchain_config.rs` +- [ ] Created language struct +- [ ] Created config template file +- [ ] Updated `ToolchainConfig` struct +- [ ] Ran `just schemas` +- [ ] Add `.prototools` support in `crates/config/src/toolchain_config.rs` - [ ] Add tests to `crates/config/tests/toolchain_config_test.rs` ### Add variant to `PlatformType` enum in `moon_config` @@ -119,16 +119,16 @@ enum PlatformType { } ``` -- [Done ] Updated enum +- [ ] Updated enum - [ ] Verified all `match` callsites handle the new variant -- [Done ] Ran `just schemas` +- [ ] Ran `just schemas` ### Update `moon_toolchain` crate Tasks run against the platform, so we can now attempt to detect this. -- [Done ] Updated `detect_task_platform` -- [Done ] Updated `detect_project_platform` +- [ ] Updated `detect_task_platform` +- [ ] Updated `detect_project_platform` ### Create tool crate @@ -150,9 +150,9 @@ This is required _even when not using_ the toolchain, as we fallback to a global Crate must exist at `legacy//tool`. Feel free to copy an existing tool crate and update the implementation. -- [Done ] Implemented `Tool` trait -- [Done ] Implemented `PackageManager` trait (when applicable) -- [Done ] Handled `global` binary +- [ ] Implemented `Tool` trait +- [ ] Implemented `PackageManager` trait (when applicable) +- [ ] Handled `global` binary ### Create platform crate @@ -163,10 +163,10 @@ moon. Crate must exist at `legacy//platform`. Feel free to copy an existing platform crate and update the implementation. -- [Done ] Implemented `Platform` trait -- [Done ] Implemented manifest hashing -- [Done ] Implemented target hashing -- [Done ] Implemented action handlers +- [ ] Implemented `Platform` trait +- [ ] Implemented manifest hashing +- [ ] Implemented target hashing +- [ ] Implemented action handlers - [ ] Implemented project graph bridge ### Update docs @@ -235,9 +235,9 @@ When the moon tool has been integrated with proto's, we can update the platform Refer to the Node.js implementation for examples (it can mostly be copied). -- [Done ] Enabled `is_toolchain_enabled` method -- [Done ] Updated `get_runtime_from_config` with `version` field -- [Done ] Updated `setup_toolchain`, `setup_tool`, and `teardown_toolchain` methods +- [ ] Enabled `is_toolchain_enabled` method +- [ ] Updated `get_runtime_from_config` with `version` field +- [ ] Updated `setup_toolchain`, `setup_tool`, and `teardown_toolchain` methods - [ ] Updated `create_run_target_command` to use the tool instance ### Support project-level config overrides @@ -245,8 +245,8 @@ Refer to the Node.js implementation for examples (it can mostly be copied). Different projects may have different version requirements, so we need to support this through project-level toolchain overrides. -- [Done ] Updated `crates/config/src/project/overrides_config.rs` -- [Done ] Updated `get_runtime_from_config` in platform crate +- [ ] Updated `crates/config/src/project/overrides_config.rs` +- [ ] Updated `get_runtime_from_config` in platform crate ### Integrate `--profile` option @@ -258,7 +258,7 @@ variants. The `moon bin` command uses a hard-coded tool list, and is not based on the `PlatformType` or `LanguageType` enums. Because of this, tools will need to be handled manually. -- [Done ] Updated `crates/app/src/commands/bin.rs` +- [ ] Updated `crates/app/src/commands/bin.rs` ### Update `docker prune` and `docker scaffold` commands @@ -288,4 +288,4 @@ At this point we should start updating docs, primarily these sections: ### Create a pull request Once everything is good, create a pull request and include it in the next release. Ideally tiers are -released separately! +released separately! \ No newline at end of file From d192b37f242f9aee55e70ba815a2c16c44478bf1 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Mon, 4 Nov 2024 10:36:13 +0100 Subject: [PATCH 22/28] Final commit after format, lint and build --- Cargo.lock | 29 ------------------- crates/cli/tests/run_python_test.rs | 1 - crates/test-utils/src/workspace_mocker.rs | 1 + legacy/python/lang/src/pip_requirements.rs | 2 +- .../python/platform/src/actions/setup_tool.rs | 4 +-- legacy/python/platform/src/python_platform.rs | 4 +-- legacy/python/tool/src/python_tool.rs | 23 ++++----------- 7 files changed, 12 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9bac2a9d94..59458fe93e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3032,9 +3032,6 @@ dependencies = [ "moon_python_lang", "moon_python_platform", "moon_python_tool", - "moon_python_lang", - "moon_python_platform", - "moon_python_tool", "moon_query", "moon_rust_lang", "moon_rust_platform", @@ -3047,7 +3044,6 @@ dependencies = [ "moon_typescript_lang", "moon_vcs", "moon_workspace", - "moon_workspace", "once_cell", "open", "petgraph", @@ -3761,7 +3757,6 @@ dependencies = [ "moon_task", "moon_test_utils2", "moon_workspace", - "moon_workspace", "petgraph", "rustc-hash 2.0.0", "scc", @@ -4098,7 +4093,6 @@ dependencies = [ "moon_system_platform", "moon_vcs", "moon_workspace", - "moon_workspace", "proto_core", "starbase_events", "starbase_sandbox", @@ -4275,29 +4269,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "moon_workspace" -version = "0.0.1" -dependencies = [ - "miette", - "moon_cache", - "moon_common", - "moon_config", - "moon_hash", - "moon_project", - "moon_project_builder", - "moon_project_constraints", - "moon_project_graph", - "moon_vcs", - "petgraph", - "rustc-hash 2.0.0", - "serde", - "starbase_events", - "starbase_utils", - "thiserror", - "tracing", -] - [[package]] name = "native-tls" version = "0.2.12" diff --git a/crates/cli/tests/run_python_test.rs b/crates/cli/tests/run_python_test.rs index ac82f32816c..26e64284d44 100644 --- a/crates/cli/tests/run_python_test.rs +++ b/crates/cli/tests/run_python_test.rs @@ -51,7 +51,6 @@ fn runs_install_deps_via_args() { pip: Some(PartialPipConfig { version: Some(UnresolvedVersionSpec::parse("latest").unwrap()), install_args: Some(vec!["poetry==1.8.4".to_string()]), - ..PartialPipConfig::default() }), ..PartialPythonConfig::default() }); diff --git a/crates/test-utils/src/workspace_mocker.rs b/crates/test-utils/src/workspace_mocker.rs index e62a950feac..a8e488bfdc3 100644 --- a/crates/test-utils/src/workspace_mocker.rs +++ b/crates/test-utils/src/workspace_mocker.rs @@ -108,6 +108,7 @@ impl WorkspaceMocker { extend_project: Emitter::::new(), extend_project_graph: Emitter::::new(), inherited_tasks: &self.inherited_tasks, + strict_project_ids: self.workspace_config.experiments.strict_project_ids, toolchain_config: &self.toolchain_config, vcs: self.vcs.clone(), working_dir: &self.workspace_root, diff --git a/legacy/python/lang/src/pip_requirements.rs b/legacy/python/lang/src/pip_requirements.rs index 91c0e4ea246..3cc94dd5066 100644 --- a/legacy/python/lang/src/pip_requirements.rs +++ b/legacy/python/lang/src/pip_requirements.rs @@ -20,7 +20,7 @@ pub fn load_lockfile_dependencies(path: PathBuf) -> miette::Result miette::R if let Some(pip_version) = &pip_config.version { let p_version: String = if pip_version.is_latest() { - format!("pip") + "pip".to_string() } else { format!( "pip{}", @@ -42,7 +42,7 @@ pub async fn setup_tool(python: &PythonTool, workspace_root: &Path) -> miette::R // Create version file if let Some(python_version) = &python.config.version { - let rc_path = workspace_root.join(".python-version".to_string()); + let rc_path = workspace_root.join(".python-version"); fs::write_file(&rc_path, python_version.to_string())?; } diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index e6183980b2b..2661ba9be34 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -8,7 +8,7 @@ use moon_config::{ }; use moon_console::Console; use moon_hash::ContentHasher; -use moon_logger::{debug, info}; +use moon_logger::debug; use moon_platform::{Platform, Runtime, RuntimeReq}; use moon_process::Command; use moon_project::Project; @@ -307,7 +307,7 @@ impl Platform for PythonPlatform { if let Ok(python) = self.toolchain.get_for_version(&runtime.requirement) { if let Some(version) = get_proto_version_env(&python.tool) { command.env("PROTO_PYTHON_VERSION", version); - command.env("PATH", prepend_path_env_var(get_python_tool_paths(&python))); + command.env("PATH", prepend_path_env_var(get_python_tool_paths(python))); } } diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs index 1a13ea556b7..54675299880 100644 --- a/legacy/python/tool/src/python_tool.rs +++ b/legacy/python/tool/src/python_tool.rs @@ -20,27 +20,16 @@ use tracing::instrument; const LOG_TARGET: &str = "moon:python-tool"; pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { - // let mut paths = get_proto_paths(proto_env); - // let mut paths:Vec = []; - - // let mut python_command = "python"; - let venv_python = &get_workspace_root().join(python_tool.config.venv_name.clone()); - let paths; - - if venv_python.exists() { - paths = vec![ + let paths = if venv_python.exists() { + vec![ venv_python.join("Scripts").clone(), venv_python.join("bin").clone(), - ]; + ] } else { - // paths = python_tool.tool.get_globals_dirs() - // .iter() - // .cloned() - // .collect::>(); - paths = get_proto_paths(&python_tool.proto_env); - } + get_proto_paths(&python_tool.proto_env) + }; debug!( target: LOG_TARGET, @@ -101,7 +90,7 @@ impl PythonTool { Command::new("python") .args(args) .envs(get_proto_env_vars()) - .env("PATH", prepend_path_env_var(get_python_tool_paths(&self))) + .env("PATH", prepend_path_env_var(get_python_tool_paths(self))) .cwd(working_dir) .with_console(self.console.clone()) .create_async() From 116cb27f3115701ac045efcd124782fdcd9ff46c Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:04:20 +0100 Subject: [PATCH 23/28] Resolve Issues based on suggestions - remove yarn version modification - remove version configuration for pip, if user want's to update the pip dependency he can do this via the requirements.txt or installArgs method - add root_requirements_only, to be able to create venv in project scope or in workspace scope - pip_requirements fixed the occupied/vaccant - clean up Cargo.toml - reworked and rethinked the install_deps and setup_tool function; install_deps phase is absolute sufficient - remove not needed comments - remove all references to get_workspace_root - Cleaned up CHANGELOG.md --- .yarn/versions/3fbef0f1.yml | 14 ----- CHANGELOG.md | 39 +----------- Cargo.lock | 6 -- crates/cli/tests/run_python_test.rs | 1 - crates/config/src/toolchain/python_config.rs | 8 ++- legacy/python/lang/Cargo.toml | 6 -- legacy/python/lang/src/pip_requirements.rs | 8 ++- .../platform/src/actions/install_deps.rs | 62 +++++++++---------- legacy/python/platform/src/actions/mod.rs | 2 - .../python/platform/src/actions/setup_tool.rs | 50 --------------- legacy/python/platform/src/lib.rs | 1 - legacy/python/platform/src/manifest_hash.rs | 10 --- legacy/python/platform/src/python_platform.rs | 54 +++++++--------- legacy/python/platform/src/toolchain_hash.rs | 9 --- legacy/python/tool/src/python_tool.rs | 10 +-- packages/types/src/project-config.ts | 2 +- packages/types/src/toolchain-config.ts | 18 ++++-- website/docs/config/toolchain.mdx | 34 +++++----- website/static/schemas/toolchain.json | 20 +++--- 19 files changed, 106 insertions(+), 248 deletions(-) delete mode 100644 .yarn/versions/3fbef0f1.yml delete mode 100644 legacy/python/platform/src/actions/setup_tool.rs delete mode 100644 legacy/python/platform/src/manifest_hash.rs diff --git a/.yarn/versions/3fbef0f1.yml b/.yarn/versions/3fbef0f1.yml deleted file mode 100644 index 280701e452c..00000000000 --- a/.yarn/versions/3fbef0f1.yml +++ /dev/null @@ -1,14 +0,0 @@ -releases: - "@moonrepo/nx-compat": minor - "@moonrepo/report": minor - "@moonrepo/runtime": minor - "@moonrepo/types": minor -website: minor - "@moonrepo/cli": minor - "@moonrepo/core-linux-arm64-gnu": minor - "@moonrepo/core-linux-arm64-musl": minor - "@moonrepo/core-linux-x64-gnu": minor - "@moonrepo/core-linux-x64-musl": minor - "@moonrepo/core-macos-arm64": minor - "@moonrepo/core-macos-x64": minor - "@moonrepo/core-windows-x64-msvc": minor diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b7e02da45b..37452d73381 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - Will parse the `requirements.txt` to resolve and install dependencies. - Added a `python.version` setting to `.moon/toolchain.yml`. - Added a `toolchain.python` setting to `moon.yml`. - - Updated `moon bin` and `moon docker` commands to support Python. + - Updated `moon bin` commands to support Python. ## 1.29.4 @@ -37,43 +37,6 @@ - Fixed an issue running `npx` commands (primarily through lockfile deduping). -## 1.29.4 - -#### 🚀 Updates - -- Added an `experiments.strictProjectIds` setting to `.moon/workspace.yml`. When enabled, will - disallow using original IDs for renamed projects (via the `id` setting) when building the project - graph. -- Updated codegen/template `destination` to be relative from the workspace root if prefixed with - `/`, otherwise the current working directory. - -#### 🐞 Fixes - -- Fixed an issue where moon would attempt to execute a folder if it has the same name as the current - shell. -- Fixed an issue where `[working_dir]` and `[workspace_root]` variables were not working in the - `template.yml` `destination` setting. - -#### ⚙️ Internal - -- Updated dependencies. -- Updated proto to v0.42.0 (from 0.41.3). - -## 1.29.3 - -#### 🐞 Fixes - -- Fixed an issue running `npx` commands (primarily through lockfile deduping). - -## Unreleased - -- Added Python tier 3 support. - - Will download and install Python into the toolchain when a `version` is configured. - - Will parse the `requirements.txt` to resolve and install dependencies. - - Added a `python.version` setting to `.moon/toolchain.yml`. - - Added a `toolchain.python` setting to `moon.yml`. - - Updated `moon bin` and `moon docker` commands to support Python. - ## 1.29.2 #### 🚀 Updates diff --git a/Cargo.lock b/Cargo.lock index 59458fe93e7..717412f1ab7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3774,17 +3774,11 @@ name = "moon_python_lang" version = "0.0.1" dependencies = [ "cached", - "cargo-lock", - "cargo_toml", "miette", "moon_lang", - "moon_logger", "moon_test_utils", "pep-508", "rustc-hash 2.0.0", - "serde", - "starbase_styles", - "starbase_utils", ] [[package]] diff --git a/crates/cli/tests/run_python_test.rs b/crates/cli/tests/run_python_test.rs index 26e64284d44..670b3d08c85 100644 --- a/crates/cli/tests/run_python_test.rs +++ b/crates/cli/tests/run_python_test.rs @@ -49,7 +49,6 @@ fn runs_install_deps_via_args() { let sandbox = python_sandbox(PartialPythonConfig { version: Some(UnresolvedVersionSpec::parse("3.11.10").unwrap()), pip: Some(PartialPipConfig { - version: Some(UnresolvedVersionSpec::parse("latest").unwrap()), install_args: Some(vec!["poetry==1.8.4".to_string()]), }), ..PartialPythonConfig::default() diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs index 74eff708e6f..ef8012a9a75 100644 --- a/crates/config/src/toolchain/python_config.rs +++ b/crates/config/src/toolchain/python_config.rs @@ -8,9 +8,6 @@ use warpgate_api::PluginLocator; pub struct PipConfig { /// List of arguments to append to `pip install` commands. pub install_args: Option>, - - /// The version of pip to download, install, and run `pip` tasks with. - pub version: Option, } #[derive(Clone, Config, Debug, PartialEq)] @@ -25,6 +22,11 @@ pub struct PythonConfig { #[setting(default = ".venv", skip)] pub venv_name: String, + /// Assumes only the root `requirements.txt` is used for dependencies. + /// Can be used to support the "one version policy" pattern. + #[setting(default = true)] + pub root_requirements_only: bool, + /// The version of Python to download, install, and run `python` tasks with. #[setting(env = "MOON_PYTHON_VERSION")] pub version: Option, diff --git a/legacy/python/lang/Cargo.toml b/legacy/python/lang/Cargo.toml index 99c7c02b7cd..4ca4c5cd01b 100644 --- a/legacy/python/lang/Cargo.toml +++ b/legacy/python/lang/Cargo.toml @@ -6,16 +6,10 @@ publish = false [dependencies] moon_lang = { path = "../../core/lang" } -moon_logger = { path = "../../core/logger" } cached = { workspace = true } -cargo-lock = "9.0.0" -cargo_toml = "0.20.4" miette = { workspace = true } pep-508 = "0.4.0" rustc-hash = { workspace = true } -serde = { workspace = true } -starbase_styles = { workspace = true } -starbase_utils = { workspace = true, features = ["glob", "toml"] } [dev-dependencies] moon_test_utils = { path = "../../core/test-utils" } diff --git a/legacy/python/lang/src/pip_requirements.rs b/legacy/python/lang/src/pip_requirements.rs index 3cc94dd5066..9bef4a5be5b 100644 --- a/legacy/python/lang/src/pip_requirements.rs +++ b/legacy/python/lang/src/pip_requirements.rs @@ -22,9 +22,11 @@ pub fn load_lockfile_dependencies(path: PathBuf) -> miette::Result miette::Result> { let mut operations = vec![]; if let Some(pip_config) = &python.config.pip { - let requirements_path = find_requirements_txt(working_dir, &get_workspace_root()); + let requirements_path = find_requirements_txt(working_dir, workspace_root); + let virtual_environment = &working_dir.join(python.config.venv_name.clone()); + + if !virtual_environment.exists() { + console + .out + .print_checkpoint(Checkpoint::Setup, "activate virtual environment")?; + let args = vec![ + "-m", + "venv", + virtual_environment.as_os_str().to_str().unwrap(), + ]; + operations.push( + Operation::task_execution(format!("python {} ", args.join(" "))) + .track_async(|| python.exec_python(args, workspace_root)) + .await?, + ); + } if let Some(install_args) = &pip_config.install_args { if install_args.iter().any(|x| !x.starts_with("-")) && requirements_path.is_none() { - console.out.print_checkpoint( - Checkpoint::Setup, - "Install pip dependencies from install args", - )?; + console + .out + .print_checkpoint(Checkpoint::Setup, "pip install")?; let mut args = vec!["-m", "pip", "install"]; - if pip_config.install_args.is_some() { - args.extend( - pip_config - .install_args - .as_ref() - .unwrap() - .iter() - .map(|c| c.as_str()), - ); + if let Some(install_args) = &pip_config.install_args { + args.extend(install_args.iter().map(|c| c.as_str())); } operations.push( - Operation::task_execution(format!(" {}", args.join(" "))) + Operation::task_execution(format!("python {}", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, ); @@ -45,27 +53,19 @@ pub async fn install_deps( } if let Some(req) = requirements_path { - console.out.print_checkpoint( - Checkpoint::Setup, - format!("Install pip dependencies from {}", color::path(&req)), - )?; + console + .out + .print_checkpoint(Checkpoint::Setup, "pip install")?; let mut args = vec!["-m", "pip", "install"]; - if pip_config.install_args.is_some() { - args.extend( - pip_config - .install_args - .as_ref() - .unwrap() - .iter() - .map(|c| c.as_str()), - ); + if let Some(install_args) = &pip_config.install_args { + args.extend(install_args.iter().map(|c| c.as_str())); } args.extend(["-r", req.as_os_str().to_str().unwrap()]); operations.push( - Operation::task_execution(format!(" {}", args.join(" "))) + Operation::task_execution(format!("python {}", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) .await?, ); diff --git a/legacy/python/platform/src/actions/mod.rs b/legacy/python/platform/src/actions/mod.rs index b4bae854747..31a65cfdfdf 100644 --- a/legacy/python/platform/src/actions/mod.rs +++ b/legacy/python/platform/src/actions/mod.rs @@ -1,5 +1,3 @@ mod install_deps; -mod setup_tool; pub use install_deps::*; -pub use setup_tool::*; diff --git a/legacy/python/platform/src/actions/setup_tool.rs b/legacy/python/platform/src/actions/setup_tool.rs deleted file mode 100644 index 2d79ea8c93a..00000000000 --- a/legacy/python/platform/src/actions/setup_tool.rs +++ /dev/null @@ -1,50 +0,0 @@ -use moon_action::Operation; -use moon_python_tool::PythonTool; -use starbase_utils::fs; -use std::path::Path; - -pub async fn setup_tool(python: &PythonTool, workspace_root: &Path) -> miette::Result<()> { - let mut operations = vec![]; - - if let Some(pip_config) = &python.config.pip { - let virtual_environment = &workspace_root.join(python.config.venv_name.clone()); - - if !virtual_environment.exists() { - let args = vec![ - "-m", - "venv", - virtual_environment.as_os_str().to_str().unwrap(), - ]; - operations.push( - Operation::task_execution(format!("python {} ", args.join(" "))) - .track_async(|| python.exec_python(args, workspace_root)) - .await?, - ); - } - - if let Some(pip_version) = &pip_config.version { - let p_version: String = if pip_version.is_latest() { - "pip".to_string() - } else { - format!( - "pip{}", - pip_version.to_owned().to_string().replace("~", "~=") - ) - }; - let args = vec!["-m", "pip", "install", "--quiet", "-U", &p_version]; - operations.push( - Operation::task_execution(format!(" {} ", args.join(" "))) - .track_async(|| python.exec_python(args, workspace_root)) - .await?, - ); - } - } - - // Create version file - if let Some(python_version) = &python.config.version { - let rc_path = workspace_root.join(".python-version"); - fs::write_file(&rc_path, python_version.to_string())?; - } - - Ok(()) -} diff --git a/legacy/python/platform/src/lib.rs b/legacy/python/platform/src/lib.rs index d08d7820a8c..b04a0f6282b 100644 --- a/legacy/python/platform/src/lib.rs +++ b/legacy/python/platform/src/lib.rs @@ -1,5 +1,4 @@ pub mod actions; -mod manifest_hash; mod python_platform; mod target_hash; mod toolchain_hash; diff --git a/legacy/python/platform/src/manifest_hash.rs b/legacy/python/platform/src/manifest_hash.rs deleted file mode 100644 index 89acff80801..00000000000 --- a/legacy/python/platform/src/manifest_hash.rs +++ /dev/null @@ -1,10 +0,0 @@ -use moon_hash::hash_content; -// use moon_python_lang::pipfile::DependencyDetail; -// use std::collections::BTreeMap; - -hash_content!( - pub struct PythonManifestHash { - // pub dependencies: BTreeMap, - pub name: String, - } -); diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 2661ba9be34..3d25132e8bd 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -1,14 +1,13 @@ use crate::{actions, find_requirements_txt, toolchain_hash::PythonToolchainHash}; use moon_action::Operation; use moon_action_context::ActionContext; -use moon_common::{color, Id}; +use moon_common::{path::is_root_level_source, Id}; use moon_config::{ HasherConfig, PlatformType, ProjectConfig, ProjectsAliasesList, ProjectsSourcesList, PythonConfig, UnresolvedVersionSpec, }; use moon_console::Console; use moon_hash::ContentHasher; -use moon_logger::debug; use moon_platform::{Platform, Runtime, RuntimeReq}; use moon_process::Command; use moon_project::Project; @@ -16,7 +15,7 @@ use moon_python_lang::pip_requirements::load_lockfile_dependencies; use moon_python_tool::{get_python_tool_paths, PythonTool}; use moon_task::Task; use moon_tool::{get_proto_version_env, prepend_path_env_var, Tool, ToolManager}; -use moon_utils::{async_trait, get_workspace_root}; +use moon_utils::async_trait; use proto_core::ProtoEnvironment; use rustc_hash::FxHashMap; use std::{ @@ -26,8 +25,6 @@ use std::{ }; use tracing::instrument; -const LOG_TARGET: &str = "moon:python-platform"; - pub struct PythonPlatform { pub config: PythonConfig, @@ -100,9 +97,17 @@ impl Platform for PythonPlatform { // PROJECT GRAPH - fn is_project_in_dependency_workspace(&self, _project_source: &str) -> miette::Result { + fn is_project_in_dependency_workspace(&self, project_source: &str) -> miette::Result { // Single version policy / only a root requirements.txt - Ok(true) + if self.config.root_requirements_only { + return Ok(true); + } + + if is_root_level_source(project_source) { + return Ok(true); + } + + Ok(false) } #[instrument(skip_all)] @@ -198,14 +203,6 @@ impl Platform for PythonPlatform { let installed = self.toolchain.setup(req, last_versions).await?; - actions::setup_tool(self.toolchain.get_for_version(req)?, &self.workspace_root).await?; - actions::install_deps( - self.toolchain.get_for_version(req)?, - &self.workspace_root, - &self.console, - ) - .await?; - Ok(installed) } @@ -214,11 +211,12 @@ impl Platform for PythonPlatform { &self, _context: &ActionContext, runtime: &Runtime, - _working_dir: &Path, + working_dir: &Path, ) -> miette::Result> { actions::install_deps( self.toolchain.get_for_version(&runtime.requirement)?, - &get_workspace_root(), + self.workspace_root.as_path(), + working_dir, &self.console, ) .await @@ -231,10 +229,7 @@ impl Platform for PythonPlatform { _project: &Project, _dependencies: &FxHashMap>, ) -> miette::Result { - let mutated_files = false; - //TODO: Here we can modifiy something - - Ok(mutated_files) + Ok(false) } #[instrument(skip_all)] @@ -247,11 +242,6 @@ impl Platform for PythonPlatform { if let Some(python_version) = &self.config.version { let deps = BTreeMap::from_iter(load_lockfile_dependencies(manifest_path.to_path_buf())?); - debug!( - target: LOG_TARGET, - "HASH MANIFEST {}", - color::path(manifest_path) - ); hasher.hash_content(PythonToolchainHash { version: python_version.clone(), dependencies: deps, @@ -276,11 +266,6 @@ impl Platform for PythonPlatform { { deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); } - debug!( - target: LOG_TARGET, - "HASH RUN TARGET {}", - color::path(&project.root) - ); hasher.hash_content(PythonToolchainHash { version: python_version.clone(), dependencies: deps, @@ -297,7 +282,7 @@ impl Platform for PythonPlatform { _project: &Project, task: &Task, runtime: &Runtime, - _working_dir: &Path, + working_dir: &Path, ) -> miette::Result { let mut command = Command::new(&task.command); command.with_console(self.console.clone()); @@ -307,7 +292,10 @@ impl Platform for PythonPlatform { if let Ok(python) = self.toolchain.get_for_version(&runtime.requirement) { if let Some(version) = get_proto_version_env(&python.tool) { command.env("PROTO_PYTHON_VERSION", version); - command.env("PATH", prepend_path_env_var(get_python_tool_paths(python))); + command.env( + "PATH", + prepend_path_env_var(get_python_tool_paths(python, working_dir)), + ); } } diff --git a/legacy/python/platform/src/toolchain_hash.rs b/legacy/python/platform/src/toolchain_hash.rs index 63df731d848..318539d8937 100644 --- a/legacy/python/platform/src/toolchain_hash.rs +++ b/legacy/python/platform/src/toolchain_hash.rs @@ -8,12 +8,3 @@ hash_content!( pub dependencies: BTreeMap>, } ); - -// impl PythonToolchainHash { -// pub fn new(python_version: UnresolvedVersionSpec) -> Self { -// PythonToolchainHash { -// version: python_version, -// dependencies: BTreeMap::new(), -// } -// } -// } diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs index 54675299880..1b62a57ee85 100644 --- a/legacy/python/tool/src/python_tool.rs +++ b/legacy/python/tool/src/python_tool.rs @@ -7,7 +7,6 @@ use moon_tool::{ use_global_tool_on_path, Tool, }; use moon_toolchain::RuntimeReq; -use moon_utils::get_workspace_root; use proto_core::flow::install::InstallOptions; use proto_core::{Id, ProtoEnvironment, Tool as ProtoTool, UnresolvedVersionSpec}; use rustc_hash::FxHashMap; @@ -19,8 +18,8 @@ use tracing::instrument; const LOG_TARGET: &str = "moon:python-tool"; -pub fn get_python_tool_paths(python_tool: &PythonTool) -> Vec { - let venv_python = &get_workspace_root().join(python_tool.config.venv_name.clone()); +pub fn get_python_tool_paths(python_tool: &PythonTool, working_dir: &Path) -> Vec { + let venv_python = working_dir.join(python_tool.config.venv_name.clone()); let paths = if venv_python.exists() { vec![ @@ -90,7 +89,10 @@ impl PythonTool { Command::new("python") .args(args) .envs(get_proto_env_vars()) - .env("PATH", prepend_path_env_var(get_python_tool_paths(self))) + .env( + "PATH", + prepend_path_env_var(get_python_tool_paths(self, working_dir)), + ) .cwd(working_dir) .with_console(self.console.clone()) .create_async() diff --git a/packages/types/src/project-config.ts b/packages/types/src/project-config.ts index 9be59614c27..578432efbed 100644 --- a/packages/types/src/project-config.ts +++ b/packages/types/src/project-config.ts @@ -2,8 +2,8 @@ /* eslint-disable */ -import type { UnresolvedVersionSpec } from './toolchain-config'; import type { PartialTaskConfig, PlatformType, TaskConfig } from './tasks-config'; +import type { UnresolvedVersionSpec } from './toolchain-config'; /** The scope and or relationship of the dependency. */ export type DependencyScope = 'build' | 'development' | 'peer' | 'production' | 'root'; diff --git a/packages/types/src/toolchain-config.ts b/packages/types/src/toolchain-config.ts index 9fc148e8255..eda92dfd5c2 100644 --- a/packages/types/src/toolchain-config.ts +++ b/packages/types/src/toolchain-config.ts @@ -253,8 +253,6 @@ export interface NodeConfig { export interface PipConfig { /** List of arguments to append to `pip install` commands. */ installArgs: string[] | null; - /** The version of pip to download, install, and run `pip` tasks with. */ - version: UnresolvedVersionSpec | null; } export interface PythonConfig { @@ -262,6 +260,13 @@ export interface PythonConfig { pip: PipConfig | null; /** Location of the WASM plugin to use for Python support. */ plugin: PluginLocator | null; + /** + * Assumes only the root `requirements.txt` is used for dependencies. + * Can be used to support the "one version policy" pattern. + * + * @default true + */ + rootRequirementsOnly?: boolean; /** * The version of Python to download, install, and run `python` tasks with. * @@ -632,8 +637,6 @@ export interface PartialNodeConfig { export interface PartialPipConfig { /** List of arguments to append to `pip install` commands. */ installArgs?: string[] | null; - /** The version of pip to download, install, and run `pip` tasks with. */ - version?: UnresolvedVersionSpec | null; } export interface PartialPythonConfig { @@ -641,6 +644,13 @@ export interface PartialPythonConfig { pip?: PartialPipConfig | null; /** Location of the WASM plugin to use for Python support. */ plugin?: PluginLocator | null; + /** + * Assumes only the root `requirements.txt` is used for dependencies. + * Can be used to support the "one version policy" pattern. + * + * @default true + */ + rootRequirementsOnly?: boolean | null; /** * The version of Python to download, install, and run `python` tasks with. * diff --git a/website/docs/config/toolchain.mdx b/website/docs/config/toolchain.mdx index 0efb7ccae7b..b3d547b510c 100644 --- a/website/docs/config/toolchain.mdx +++ b/website/docs/config/toolchain.mdx @@ -743,6 +743,20 @@ python: > Version can also be defined with [`.prototools`](../proto/config). +### `rootPackageOnly` + + + +Supports the "single version policy" or "one version rule" patterns by only allowing dependencies in +the root `requirements.txt`, and only installing dependencies in the workspace root, and not within +individual projects. It also bypasses all `workspaces` checks to determine package locations. +Defaults to `true`. + +```yaml title=".moon/toolchain.yml" {2} +python: + rootPackageOnly: false +``` + ### `venv_name` @@ -759,25 +773,7 @@ python: -#### `version` - - - -The `version` setting defines the explicit pip -[version specification](../concepts/toolchain#version-specification) to use. If this field is _not -defined_, the built-in library will be used. - -A list of plugins that will automatically be imported using `yarn plugin import` (Yarn 2+ only). For -performance reasons, plugins will only be imported when the Yarn version changes. - -```yaml title=".moon/toolchain.yml" {4} -python: - version: '3.11.10' - pip: - version: 'latest' -``` - -### `install_args` +#### `install_args` diff --git a/website/static/schemas/toolchain.json b/website/static/schemas/toolchain.json index 8686ed8dcf1..6a8ebe7f591 100644 --- a/website/static/schemas/toolchain.json +++ b/website/static/schemas/toolchain.json @@ -598,19 +598,6 @@ } ], "markdownDescription": "List of arguments to append to `pip install` commands." - }, - "version": { - "title": "version", - "description": "The version of pip to download, install, and run pip tasks with.", - "anyOf": [ - { - "$ref": "#/definitions/UnresolvedVersionSpec" - }, - { - "type": "null" - } - ], - "markdownDescription": "The version of pip to download, install, and run `pip` tasks with." } }, "additionalProperties": false @@ -687,6 +674,13 @@ } ] }, + "rootRequirementsOnly": { + "title": "rootRequirementsOnly", + "description": "Assumes only the root requirements.txt is used for dependencies. Can be used to support the \"one version policy\" pattern.", + "default": true, + "type": "boolean", + "markdownDescription": "Assumes only the root `requirements.txt` is used for dependencies. Can be used to support the \"one version policy\" pattern." + }, "version": { "title": "version", "description": "The version of Python to download, install, and run python tasks with.", From c99a36985723e41703eb2e818251987ae978a02d Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Mon, 11 Nov 2024 12:43:39 +0100 Subject: [PATCH 24/28] Execute lint and format --- CHANGELOG.md | 6 +++++ packages/types/src/project-config.ts | 22 +++++++++++++++++-- packages/types/src/tasks-config.ts | 11 +++++++++- packages/types/src/template-config.ts | 12 ++++++++-- packages/types/src/toolchain-config.ts | 11 +++++++++- packages/types/src/workspace-config.ts | 5 ++++- .../setup-toolchain/python/tier2.mdx | 2 +- website/docs/config/toolchain.mdx | 12 +++++----- 8 files changed, 67 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92f8f07888b..223b795a905 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ - Resolved the `disallowRunInCiMismatch` experiment and you can no longer have a CI based task depend on a non-CI based task. - Added a new task graph, that enables new granular based functionality for task related features. +- Added Python tier 3 support. + - Will download and install Python into the toolchain when a `version` is configured. + - Will parse the `requirements.txt` to resolve and install dependencies. + - Added a `python.version` setting to `.moon/toolchain.yml`. + - Added a `toolchain.python` setting to `moon.yml`. + - Updated `moon bin` commands to support Python. ## 1.29.4 diff --git a/packages/types/src/project-config.ts b/packages/types/src/project-config.ts index 3e147e7feba..548d96ff69c 100644 --- a/packages/types/src/project-config.ts +++ b/packages/types/src/project-config.ts @@ -69,7 +69,18 @@ export interface ProjectDockerConfig { } /** Supported programming languages that each project can be written in. */ -export type LanguageType = 'bash' | 'batch' | 'go' | 'javascript' | 'php' | 'python' | 'ruby' | 'rust' | 'typescript' | 'unknown' | string; +export type LanguageType = + | 'bash' + | 'batch' + | 'go' + | 'javascript' + | 'php' + | 'python' + | 'ruby' + | 'rust' + | 'typescript' + | 'unknown' + | string; export type OwnersPaths = string[] | Record; @@ -165,7 +176,14 @@ export interface ProjectToolchainConfig { } /** The type of project, for categorizing. */ -export type ProjectType = 'application' | 'automation' | 'configuration' | 'library' | 'scaffolding' | 'tool' | 'unknown'; +export type ProjectType = + | 'application' + | 'automation' + | 'configuration' + | 'library' + | 'scaffolding' + | 'tool' + | 'unknown'; /** Controls how tasks are inherited. */ export interface ProjectWorkspaceInheritedTasksConfig { diff --git a/packages/types/src/tasks-config.ts b/packages/types/src/tasks-config.ts index 189bcabda8c..84dfbd33761 100644 --- a/packages/types/src/tasks-config.ts +++ b/packages/types/src/tasks-config.ts @@ -32,7 +32,16 @@ export type TaskOperatingSystem = 'linux' | 'macos' | 'windows'; export type TaskOutputStyle = 'buffer' | 'buffer-only-failure' | 'hash' | 'none' | 'stream'; /** A list of available shells on Unix. */ -export type TaskUnixShell = 'bash' | 'elvish' | 'fish' | 'ion' | 'murex' | 'nu' | 'pwsh' | 'xonsh' | 'zsh'; +export type TaskUnixShell = + | 'bash' + | 'elvish' + | 'fish' + | 'ion' + | 'murex' + | 'nu' + | 'pwsh' + | 'xonsh' + | 'zsh'; /** A list of available shells on Windows. */ export type TaskWindowsShell = 'bash' | 'elvish' | 'fish' | 'murex' | 'nu' | 'pwsh' | 'xonsh'; diff --git a/packages/types/src/template-config.ts b/packages/types/src/template-config.ts index 9c1ffcb55cd..db9ea77308c 100644 --- a/packages/types/src/template-config.ts +++ b/packages/types/src/template-config.ts @@ -92,7 +92,11 @@ export interface TemplateVariableStringSetting { type: 'string'; } -export type TemplateVariable = TemplateVariableBoolSetting | TemplateVariableEnumSetting | TemplateVariableNumberSetting | TemplateVariableStringSetting; +export type TemplateVariable = + | TemplateVariableBoolSetting + | TemplateVariableEnumSetting + | TemplateVariableNumberSetting + | TemplateVariableStringSetting; /** * Configures a template and its files to be scaffolded. @@ -193,7 +197,11 @@ export interface PartialTemplateVariableStringSetting { type?: 'string' | null; } -export type PartialTemplateVariable = PartialTemplateVariableBoolSetting | PartialTemplateVariableEnumSetting | PartialTemplateVariableNumberSetting | PartialTemplateVariableStringSetting; +export type PartialTemplateVariable = + | PartialTemplateVariableBoolSetting + | PartialTemplateVariableEnumSetting + | PartialTemplateVariableNumberSetting + | PartialTemplateVariableStringSetting; /** * Configures a template and its files to be scaffolded. diff --git a/packages/types/src/toolchain-config.ts b/packages/types/src/toolchain-config.ts index eda92dfd5c2..ffc1933822a 100644 --- a/packages/types/src/toolchain-config.ts +++ b/packages/types/src/toolchain-config.ts @@ -3,7 +3,16 @@ /* eslint-disable */ /** Formats that a `package.json` version dependency can be. */ -export type NodeVersionFormat = 'file' | 'link' | 'star' | 'version' | 'version-caret' | 'version-tilde' | 'workspace' | 'workspace-caret' | 'workspace-tilde'; +export type NodeVersionFormat = + | 'file' + | 'link' + | 'star' + | 'version' + | 'version-caret' + | 'version-tilde' + | 'workspace' + | 'workspace-caret' + | 'workspace-tilde'; export type PluginLocator = string; diff --git a/packages/types/src/workspace-config.ts b/packages/types/src/workspace-config.ts index bae3fe69deb..d03b98f62d3 100644 --- a/packages/types/src/workspace-config.ts +++ b/packages/types/src/workspace-config.ts @@ -529,7 +529,10 @@ export interface PartialWorkspaceProjectsConfig { sources?: Record | null; } -export type PartialWorkspaceProjects = PartialWorkspaceProjectsConfig | string[] | Record; +export type PartialWorkspaceProjects = + | PartialWorkspaceProjectsConfig + | string[] + | Record; /** Configures aspects of the task runner (also known as the action pipeline). */ export interface PartialRunnerConfig { diff --git a/website/docs/__partials__/setup-toolchain/python/tier2.mdx b/website/docs/__partials__/setup-toolchain/python/tier2.mdx index 622d4ce24b7..e7c4c6e2d8f 100644 --- a/website/docs/__partials__/setup-toolchain/python/tier2.mdx +++ b/website/docs/__partials__/setup-toolchain/python/tier2.mdx @@ -1,4 +1,4 @@ ```yaml title=".moon/toolchain.yml" python: pip: {} -``` \ No newline at end of file +``` diff --git a/website/docs/config/toolchain.mdx b/website/docs/config/toolchain.mdx index b3d547b510c..0f52bb17b8d 100644 --- a/website/docs/config/toolchain.mdx +++ b/website/docs/config/toolchain.mdx @@ -733,8 +733,8 @@ Enables and configures Python. -Defines the explicit Python toolchain -If this field is _not defined_, the global `python` binary will be used. +Defines the explicit Python toolchain If this field is _not defined_, the global `python` binary +will be used. ```yaml title=".moon/toolchain.yml" {2} python: @@ -761,8 +761,8 @@ python: -Defines the virtual environment name which will be created on workspace root, -project dependencies will be installed into this. Defaults to `.venv` +Defines the virtual environment name which will be created on workspace root, project dependencies +will be installed into this. Defaults to `.venv` ```yaml title=".moon/toolchain.yml" {2} python: @@ -777,8 +777,8 @@ python: -Customize the arguments that will be passed to the pip install command, when the -`InstallDeps` action is triggered in the pipeline. These arguments are used both locally and in CI. +Customize the arguments that will be passed to the pip install command, when the `InstallDeps` +action is triggered in the pipeline. These arguments are used both locally and in CI. ```yaml title=".moon/toolchain.yml" {3} python: From 7e21f57b11dd99bc6b702360b4ef2a36a21664bb Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:27:43 +0100 Subject: [PATCH 25/28] Fix testing for python areas --- crates/cli/tests/run_python_test.rs | 6 +++++- ...ython_test__runs_install_deps_via_args.snap | 6 ++++-- .../tests/inherited_tasks_config_test.rs | 2 +- crates/config/tests/project_config_test.rs | 2 +- crates/config/tests/task_config_test.rs | 2 +- .../platform/src/actions/install_deps.rs | 8 ++++++-- legacy/python/platform/src/lib.rs | 1 - legacy/python/platform/src/python_platform.rs | 9 ++++++++- legacy/python/platform/src/target_hash.rs | 18 ------------------ 9 files changed, 26 insertions(+), 28 deletions(-) delete mode 100644 legacy/python/platform/src/target_hash.rs diff --git a/crates/cli/tests/run_python_test.rs b/crates/cli/tests/run_python_test.rs index 670b3d08c85..d805e17198c 100644 --- a/crates/cli/tests/run_python_test.rs +++ b/crates/cli/tests/run_python_test.rs @@ -49,7 +49,11 @@ fn runs_install_deps_via_args() { let sandbox = python_sandbox(PartialPythonConfig { version: Some(UnresolvedVersionSpec::parse("3.11.10").unwrap()), pip: Some(PartialPipConfig { - install_args: Some(vec!["poetry==1.8.4".to_string()]), + install_args: Some(vec![ + "--quiet".to_string(), + "--disable-pip-version-check".to_string(), + "poetry==1.8.4".to_string(), + ]), }), ..PartialPythonConfig::default() }); diff --git a/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap b/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap index 3f7391f5497..4fa7044b750 100644 --- a/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap +++ b/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap @@ -1,9 +1,11 @@ --- source: crates/cli/tests/run_python_test.rs -assertion_line: 62 +assertion_line: 60 expression: assert.output() +snapshot_kind: text --- -▪▪▪▪ Install pip dependencies from install args +▪▪▪▪ activate virtual environment +▪▪▪▪ pip install ▪▪▪▪ python:poetry Poetry (version 1.8.4) ▪▪▪▪ python:poetry (100ms) diff --git a/crates/config/tests/inherited_tasks_config_test.rs b/crates/config/tests/inherited_tasks_config_test.rs index d7aa6ce5dc4..6ec2f0cd815 100644 --- a/crates/config/tests/inherited_tasks_config_test.rs +++ b/crates/config/tests/inherited_tasks_config_test.rs @@ -524,9 +524,9 @@ mod task_manager { "javascript-tool", "kotlin", "node", - "python", "node-application", "node-library", + "python", "rust", "tag-camelCase", "tag-dot.case", diff --git a/crates/config/tests/project_config_test.rs b/crates/config/tests/project_config_test.rs index e12fde1ee99..01be7b1ec9f 100644 --- a/crates/config/tests/project_config_test.rs +++ b/crates/config/tests/project_config_test.rs @@ -283,7 +283,7 @@ fileGroups: #[test] #[should_panic( - expected = "unknown variant `perl`, expected one of `bun`, `deno`, `node`, `rust`, `system`, `unknown`" + expected = "Failed to parse TaskConfig. platform: unknown variant `perl`, expected one of `bun`, `deno`, `node`, `python`, `rust`, `system`, `unknown`" )] fn errors_on_invalid_variant() { test_load_config("moon.yml", "platform: perl", |path| { diff --git a/crates/config/tests/task_config_test.rs b/crates/config/tests/task_config_test.rs index 9da09900580..f3d2d5de906 100644 --- a/crates/config/tests/task_config_test.rs +++ b/crates/config/tests/task_config_test.rs @@ -350,7 +350,7 @@ outputs: #[test] #[should_panic( - expected = "unknown variant `perl`, expected one of `bun`, `deno`, `node`, `rust`, `system`, `unknown`" + expected = "Failed to parse TaskConfig. platform: unknown variant `perl`, expected one of `bun`, `deno`, `node`, `python`, `rust`, `system`, `unknown`" )] fn errors_on_invalid_variant() { test_parse_config("platform: perl", load_config_from_code); diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs index dd89194ae84..883e6f0f3fa 100644 --- a/legacy/python/platform/src/actions/install_deps.rs +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -15,7 +15,11 @@ pub async fn install_deps( if let Some(pip_config) = &python.config.pip { let requirements_path = find_requirements_txt(working_dir, workspace_root); - let virtual_environment = &working_dir.join(python.config.venv_name.clone()); + let virtual_environment = if python.config.root_requirements_only { + &workspace_root.join(python.config.venv_name.clone()) + } else { + &working_dir.join(python.config.venv_name.clone()) + }; if !virtual_environment.exists() { console @@ -34,7 +38,7 @@ pub async fn install_deps( } if let Some(install_args) = &pip_config.install_args { - if install_args.iter().any(|x| !x.starts_with("-")) && requirements_path.is_none() { + if install_args.iter().any(|x| !x.starts_with("-")) { console .out .print_checkpoint(Checkpoint::Setup, "pip install")?; diff --git a/legacy/python/platform/src/lib.rs b/legacy/python/platform/src/lib.rs index b04a0f6282b..3e1a597bf12 100644 --- a/legacy/python/platform/src/lib.rs +++ b/legacy/python/platform/src/lib.rs @@ -1,6 +1,5 @@ pub mod actions; mod python_platform; -mod target_hash; mod toolchain_hash; pub use python_platform::*; diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index 3d25132e8bd..eea267bb77c 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -285,16 +285,23 @@ impl Platform for PythonPlatform { working_dir: &Path, ) -> miette::Result { let mut command = Command::new(&task.command); + command.with_console(self.console.clone()); command.args(&task.args); command.envs(&task.env); if let Ok(python) = self.toolchain.get_for_version(&runtime.requirement) { if let Some(version) = get_proto_version_env(&python.tool) { + let cwd = if python.config.root_requirements_only { + self.workspace_root.as_path() + } else { + working_dir + }; + command.env("PROTO_PYTHON_VERSION", version); command.env( "PATH", - prepend_path_env_var(get_python_tool_paths(python, working_dir)), + prepend_path_env_var(get_python_tool_paths(python, cwd)), ); } } diff --git a/legacy/python/platform/src/target_hash.rs b/legacy/python/platform/src/target_hash.rs deleted file mode 100644 index 155f3e1d195..00000000000 --- a/legacy/python/platform/src/target_hash.rs +++ /dev/null @@ -1,18 +0,0 @@ -use moon_hash::hash_content; -use std::collections::BTreeMap; - -hash_content!( - pub struct PythonTargetHash { - pub python_version: String, - pub locked_dependencies: BTreeMap>, - } -); - -// impl PythonTargetHash { -// pub fn new(python_version: Option) -> Self { -// PythonTargetHash { -// python_version: python_version.unwrap_or_else(|| "unknown".into()), -// locked_dependencies: BTreeMap::new(), -// } -// } -// } From 069d361002b5d9a8cca24f5004a701aa02d166f6 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:45:37 +0100 Subject: [PATCH 26/28] Fix project_config_test --- crates/config/tests/project_config_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/config/tests/project_config_test.rs b/crates/config/tests/project_config_test.rs index 01be7b1ec9f..59a6a8a22f3 100644 --- a/crates/config/tests/project_config_test.rs +++ b/crates/config/tests/project_config_test.rs @@ -283,7 +283,7 @@ fileGroups: #[test] #[should_panic( - expected = "Failed to parse TaskConfig. platform: unknown variant `perl`, expected one of `bun`, `deno`, `node`, `python`, `rust`, `system`, `unknown`" + expected = "Failed to parse moon.yml. platform: unknown variant `perl`, expected one of `bun`, `deno`, `node`, `python`, `rust`, `system`, `unknown`" )] fn errors_on_invalid_variant() { test_load_config("moon.yml", "platform: perl", |path| { From c655e88bdc8ae6a496a51e9ad78ccb9561da1abb Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Thu, 14 Nov 2024 09:12:05 +0100 Subject: [PATCH 27/28] Resolve latest comments --- crates/config/src/toolchain/python_config.rs | 4 +- .../platform/src/actions/install_deps.rs | 34 +++++---------- legacy/python/platform/src/python_platform.rs | 41 ++++++++++--------- legacy/python/platform/src/toolchain_hash.rs | 3 +- packages/types/src/toolchain-config.ts | 14 +++++++ website/static/schemas/toolchain.json | 7 ++++ 6 files changed, 58 insertions(+), 45 deletions(-) diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs index ef8012a9a75..2cdb1b3ca17 100644 --- a/crates/config/src/toolchain/python_config.rs +++ b/crates/config/src/toolchain/python_config.rs @@ -19,7 +19,9 @@ pub struct PythonConfig { #[setting(nested)] pub pip: Option, - #[setting(default = ".venv", skip)] + /// Defines the virtual environment name which will be created on workspace root. + /// Project dependencies will be installed into this. Defaults to `.venv` + #[setting(default = ".venv")] pub venv_name: String, /// Assumes only the root `requirements.txt` is used for dependencies. diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs index 883e6f0f3fa..03adf199b3d 100644 --- a/legacy/python/platform/src/actions/install_deps.rs +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -37,37 +37,25 @@ pub async fn install_deps( ); } - if let Some(install_args) = &pip_config.install_args { - if install_args.iter().any(|x| !x.starts_with("-")) { - console - .out - .print_checkpoint(Checkpoint::Setup, "pip install")?; + let mut args = vec![]; - let mut args = vec!["-m", "pip", "install"]; - if let Some(install_args) = &pip_config.install_args { - args.extend(install_args.iter().map(|c| c.as_str())); - } + // Add pip installArgs, if users have given + if let Some(install_args) = &pip_config.install_args { + args.extend(install_args.iter().map(|c| c.as_str())); + } - operations.push( - Operation::task_execution(format!("python {}", args.join(" "))) - .track_async(|| python.exec_python(args, working_dir)) - .await?, - ); - } + // Add requirements.txt path, if found + if let Some(req) = &requirements_path { + args.extend(["-r", req.as_os_str().to_str().unwrap()]); } - if let Some(req) = requirements_path { + if !args.is_empty() { + args.splice(0..0, vec!["-m", "pip", "install"]); + console .out .print_checkpoint(Checkpoint::Setup, "pip install")?; - let mut args = vec!["-m", "pip", "install"]; - if let Some(install_args) = &pip_config.install_args { - args.extend(install_args.iter().map(|c| c.as_str())); - } - - args.extend(["-r", req.as_os_str().to_str().unwrap()]); - operations.push( Operation::task_execution(format!("python {}", args.join(" "))) .track_async(|| python.exec_python(args, working_dir)) diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs index eea267bb77c..5a8e23c0301 100644 --- a/legacy/python/platform/src/python_platform.rs +++ b/legacy/python/platform/src/python_platform.rs @@ -239,14 +239,16 @@ impl Platform for PythonPlatform { hasher: &mut ContentHasher, _hasher_config: &HasherConfig, ) -> miette::Result<()> { - if let Some(python_version) = &self.config.version { - let deps = - BTreeMap::from_iter(load_lockfile_dependencies(manifest_path.to_path_buf())?); - hasher.hash_content(PythonToolchainHash { - version: python_version.clone(), - dependencies: deps, - })?; - } + let deps = BTreeMap::from_iter(load_lockfile_dependencies(manifest_path.to_path_buf())?); + hasher.hash_content(PythonToolchainHash { + version: self + .config + .version + .as_ref() + .map(|v| v.to_string()) + .unwrap_or_default(), + dependencies: deps, + })?; Ok(()) } @@ -259,18 +261,19 @@ impl Platform for PythonPlatform { hasher: &mut ContentHasher, _hasher_config: &HasherConfig, ) -> miette::Result<()> { - if let Some(python_version) = &self.config.version { - let mut deps = BTreeMap::new(); - if let Some(pip_requirements) = - find_requirements_txt(&project.root, &self.workspace_root) - { - deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); - } - hasher.hash_content(PythonToolchainHash { - version: python_version.clone(), - dependencies: deps, - })?; + let mut deps = BTreeMap::new(); + if let Some(pip_requirements) = find_requirements_txt(&project.root, &self.workspace_root) { + deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); } + hasher.hash_content(PythonToolchainHash { + version: self + .config + .version + .as_ref() + .map(|v| v.to_string()) + .unwrap_or_default(), + dependencies: deps, + })?; Ok(()) } diff --git a/legacy/python/platform/src/toolchain_hash.rs b/legacy/python/platform/src/toolchain_hash.rs index 318539d8937..fcaf96ddc6e 100644 --- a/legacy/python/platform/src/toolchain_hash.rs +++ b/legacy/python/platform/src/toolchain_hash.rs @@ -1,10 +1,9 @@ -use moon_config::UnresolvedVersionSpec; use moon_hash::hash_content; use std::collections::BTreeMap; hash_content!( pub struct PythonToolchainHash { - pub version: UnresolvedVersionSpec, + pub version: String, pub dependencies: BTreeMap>, } ); diff --git a/packages/types/src/toolchain-config.ts b/packages/types/src/toolchain-config.ts index ffc1933822a..27c8a6d0487 100644 --- a/packages/types/src/toolchain-config.ts +++ b/packages/types/src/toolchain-config.ts @@ -276,6 +276,13 @@ export interface PythonConfig { * @default true */ rootRequirementsOnly?: boolean; + /** + * Defines the virtual environment name which will be created on workspace root. + * Project dependencies will be installed into this. Defaults to `.venv` + * + * @default '.venv' + */ + venvName?: string; /** * The version of Python to download, install, and run `python` tasks with. * @@ -660,6 +667,13 @@ export interface PartialPythonConfig { * @default true */ rootRequirementsOnly?: boolean | null; + /** + * Defines the virtual environment name which will be created on workspace root. + * Project dependencies will be installed into this. Defaults to `.venv` + * + * @default '.venv' + */ + venvName?: string | null; /** * The version of Python to download, install, and run `python` tasks with. * diff --git a/website/static/schemas/toolchain.json b/website/static/schemas/toolchain.json index 6a8ebe7f591..bd2c893db6a 100644 --- a/website/static/schemas/toolchain.json +++ b/website/static/schemas/toolchain.json @@ -681,6 +681,13 @@ "type": "boolean", "markdownDescription": "Assumes only the root `requirements.txt` is used for dependencies. Can be used to support the \"one version policy\" pattern." }, + "venvName": { + "title": "venvName", + "description": "Defines the virtual environment name which will be created on workspace root. Project dependencies will be installed into this. Defaults to .venv", + "default": ".venv", + "type": "string", + "markdownDescription": "Defines the virtual environment name which will be created on workspace root. Project dependencies will be installed into this. Defaults to `.venv`" + }, "version": { "title": "version", "description": "The version of Python to download, install, and run python tasks with.", From d8a12b19d878bc61096e16b0065ad6a4e4bbcaf5 Mon Sep 17 00:00:00 2001 From: harlquin <547042+harlequin@users.noreply.github.com> Date: Thu, 14 Nov 2024 09:16:21 +0100 Subject: [PATCH 28/28] Update changelog.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 369d1c3d746..1905727d209 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,12 @@ - Added 7 new token variables: `$arch`, `$os`, `$osFamily`, `$vcsBranch`, `$vcsRepository`, `$vcsRevision`, `$workingDir` - Added a `rust.binstallVersion` setting to `.moon/toolchain.yml`. +- Added Python tier 3 support. + - Will download and install Python into the toolchain when a `version` is configured. + - Will parse the `requirements.txt` to resolve and install dependencies. + - Added a `python.version` setting to `.moon/toolchain.yml`. + - Added a `toolchain.python` setting to `moon.yml`. + - Updated `moon bin` commands to support Python. #### 🐞 Fixes