diff --git a/Cargo.lock b/Cargo.lock index df6b832adfd..627e152d4a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -773,6 +773,19 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "chumsky" +version = "1.0.0-alpha.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c28d4e5dd9a9262a38b231153591da6ce1471b818233f4727985d3dd0ed93c" +dependencies = [ + "hashbrown 0.14.5", + "regex-automata 0.3.9", + "serde", + "stacker", + "unicode-ident", +] + [[package]] name = "chunked_transfer" version = "1.5.0" @@ -3151,6 +3164,9 @@ dependencies = [ "moon_plugin", "moon_project", "moon_project_graph", + "moon_python_lang", + "moon_python_platform", + "moon_python_tool", "moon_query", "moon_rust_lang", "moon_rust_platform", @@ -3894,6 +3910,69 @@ dependencies = [ "tracing", ] +[[package]] +name = "moon_python_lang" +version = "0.0.1" +dependencies = [ + "cached", + "miette", + "moon_lang", + "moon_test_utils", + "pep-508", + "rustc-hash 2.0.0", +] + +[[package]] +name = "moon_python_platform" +version = "0.0.1" +dependencies = [ + "miette", + "moon_action", + "moon_action_context", + "moon_common", + "moon_config", + "moon_console", + "moon_hash", + "moon_logger", + "moon_platform", + "moon_process", + "moon_project", + "moon_python_lang", + "moon_python_tool", + "moon_task", + "moon_test_utils", + "moon_tool", + "moon_utils", + "proto_core", + "rustc-hash 2.0.0", + "serde", + "starbase_styles", + "starbase_utils", + "tokio", + "tracing", +] + +[[package]] +name = "moon_python_tool" +version = "0.0.1" +dependencies = [ + "miette", + "moon_common", + "moon_config", + "moon_console", + "moon_logger", + "moon_process", + "moon_python_lang", + "moon_tool", + "moon_toolchain", + "moon_utils", + "proto_core", + "rustc-hash 2.0.0", + "starbase_styles", + "starbase_utils", + "tracing", +] + [[package]] name = "moon_query" version = "0.0.1" @@ -4653,6 +4732,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d61c5ce1153ab5b689d0c074c4e7fc613e942dfb7dd9eea5ab202d2ad91fe361" +[[package]] +name = "pep-508" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56536b95df75cc5801a27ae2b53381d5d295fb30837be65f72916ecef5d1e4f" +dependencies = [ + "chumsky", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -5193,6 +5281,17 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.5", +] + [[package]] name = "regex-automata" version = "0.4.8" @@ -5210,6 +5309,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + [[package]] name = "regex-syntax" version = "0.8.5" @@ -5863,6 +5968,19 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stacker" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] + [[package]] name = "starbase" version = "0.9.4" diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index 30b70a2387c..b4a4ecc581a 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -89,6 +89,9 @@ moon_deno_platform = { path = "../../legacy/deno/platform" } moon_node_lang = { path = "../../legacy/node/lang" } moon_node_tool = { path = "../../legacy/node/tool" } moon_node_platform = { path = "../../legacy/node/platform" } +moon_python_lang = { path = "../../legacy/python/lang" } +moon_python_tool = { path = "../../legacy/python/tool" } +moon_python_platform = { path = "../../legacy/python/platform" } moon_rust_lang = { path = "../../legacy/rust/lang" } moon_rust_tool = { path = "../../legacy/rust/tool" } moon_rust_platform = { path = "../../legacy/rust/platform" } diff --git a/crates/app/src/systems/analyze.rs b/crates/app/src/systems/analyze.rs index a8d331231e6..b9185e5d803 100644 --- a/crates/app/src/systems/analyze.rs +++ b/crates/app/src/systems/analyze.rs @@ -8,6 +8,7 @@ use moon_console::{Checkpoint, Console}; use moon_deno_platform::DenoPlatform; use moon_node_platform::NodePlatform; use moon_platform::PlatformManager; +use moon_python_platform::PythonPlatform; use moon_rust_platform::RustPlatform; use moon_system_platform::SystemPlatform; use moon_toolchain_plugin::ToolchainRegistry; @@ -172,6 +173,18 @@ pub async fn register_platforms( ); } + if let Some(python_config) = &toolchain_config.python { + registry.register( + PlatformType::Python, + Box::new(PythonPlatform::new( + python_config, + workspace_root, + Arc::clone(proto_env), + Arc::clone(&console), + )), + ); + } + if let Some(rust_config) = &toolchain_config.rust { registry.register( PlatformType::Rust, diff --git a/crates/cli/tests/run_python_test.rs b/crates/cli/tests/run_python_test.rs new file mode 100644 index 00000000000..d805e17198c --- /dev/null +++ b/crates/cli/tests/run_python_test.rs @@ -0,0 +1,65 @@ +use moon_config::{PartialPipConfig, PartialPythonConfig}; +use moon_test_utils::{ + assert_snapshot, create_sandbox_with_config, get_python_fixture_configs, Sandbox, +}; +use proto_core::UnresolvedVersionSpec; + +fn python_sandbox(config: PartialPythonConfig) -> Sandbox { + python_sandbox_with_config(|_| {}, config) +} + +fn python_sandbox_with_config(callback: C, config: PartialPythonConfig) -> Sandbox +where + C: FnOnce(&mut PartialPythonConfig), +{ + let (workspace_config, mut toolchain_config, tasks_config) = get_python_fixture_configs(); + + toolchain_config.python = Some(config); + + if let Some(python_config) = &mut toolchain_config.python { + callback(python_config); + } + + let sandbox = create_sandbox_with_config( + "python", + Some(workspace_config), + Some(toolchain_config), + Some(tasks_config), + ); + + sandbox.enable_git(); + sandbox +} + +#[test] +fn runs_standard_script() { + let sandbox = python_sandbox(PartialPythonConfig { + version: Some(UnresolvedVersionSpec::parse("3.11.10").unwrap()), + ..PartialPythonConfig::default() + }); + let assert = sandbox.run_moon(|cmd| { + cmd.arg("run").arg("python:standard"); + }); + + assert_snapshot!(assert.output()); +} + +#[test] +fn runs_install_deps_via_args() { + let sandbox = python_sandbox(PartialPythonConfig { + version: Some(UnresolvedVersionSpec::parse("3.11.10").unwrap()), + pip: Some(PartialPipConfig { + install_args: Some(vec![ + "--quiet".to_string(), + "--disable-pip-version-check".to_string(), + "poetry==1.8.4".to_string(), + ]), + }), + ..PartialPythonConfig::default() + }); + let assert = sandbox.run_moon(|cmd| { + cmd.arg("run").arg("python:poetry"); + }); + + assert_snapshot!(assert.output()); +} diff --git a/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap b/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap new file mode 100644 index 00000000000..4fa7044b750 --- /dev/null +++ b/crates/cli/tests/snapshots/run_python_test__runs_install_deps_via_args.snap @@ -0,0 +1,14 @@ +--- +source: crates/cli/tests/run_python_test.rs +assertion_line: 60 +expression: assert.output() +snapshot_kind: text +--- +▪▪▪▪ activate virtual environment +▪▪▪▪ pip install +▪▪▪▪ python:poetry +Poetry (version 1.8.4) +▪▪▪▪ python:poetry (100ms) + +Tasks: 1 completed + Time: 100ms diff --git a/crates/cli/tests/snapshots/run_python_test__runs_standard_script.snap b/crates/cli/tests/snapshots/run_python_test__runs_standard_script.snap new file mode 100644 index 00000000000..5873dd9d926 --- /dev/null +++ b/crates/cli/tests/snapshots/run_python_test__runs_standard_script.snap @@ -0,0 +1,11 @@ +--- +source: crates/cli/tests/run_python_test.rs +assertion_line: 38 +expression: assert.output() +--- +▪▪▪▪ python:standard +Python 3.11.10 +▪▪▪▪ python:standard (100ms) + +Tasks: 1 completed + Time: 100ms diff --git a/crates/config/src/language_platform.rs b/crates/config/src/language_platform.rs index c7179e3f572..d291c8044a8 100644 --- a/crates/config/src/language_platform.rs +++ b/crates/config/src/language_platform.rs @@ -70,6 +70,7 @@ derive_enum!( Bun, Deno, Node, + Python, Rust, System, #[default] @@ -103,6 +104,7 @@ mod tests { assert_eq!(LanguageType::Go.to_string(), "go"); assert_eq!(LanguageType::JavaScript.to_string(), "javascript"); assert_eq!(LanguageType::Ruby.to_string(), "ruby"); + assert_eq!(LanguageType::Python.to_string(), "python"); assert_eq!(LanguageType::Unknown.to_string(), "unknown"); assert_eq!(LanguageType::Other(Id::raw("dotnet")).to_string(), "dotnet"); } diff --git a/crates/config/src/project/overrides_config.rs b/crates/config/src/project/overrides_config.rs index 0c87bcda69c..0430d22900a 100644 --- a/crates/config/src/project/overrides_config.rs +++ b/crates/config/src/project/overrides_config.rs @@ -49,6 +49,10 @@ cacheable!( #[setting(nested)] pub deno: Option, + /// Overrides `python` settings. + #[setting(nested)] + pub python: Option, + /// Overrides `node` settings. #[setting(nested)] pub node: Option, diff --git a/crates/config/src/toolchain/mod.rs b/crates/config/src/toolchain/mod.rs index 49836e27e9d..33ad220b361 100644 --- a/crates/config/src/toolchain/mod.rs +++ b/crates/config/src/toolchain/mod.rs @@ -3,6 +3,7 @@ mod bun_config; mod deno_config; mod moon_config; mod node_config; +mod python_config; mod rust_config; mod typescript_config; @@ -11,6 +12,7 @@ pub use bun_config::*; pub use deno_config::*; pub use moon_config::*; pub use node_config::*; +pub use python_config::*; pub use rust_config::*; pub use typescript_config::*; diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs new file mode 100644 index 00000000000..2cdb1b3ca17 --- /dev/null +++ b/crates/config/src/toolchain/python_config.rs @@ -0,0 +1,35 @@ +// use super::bin_config::BinEntry; +use schematic::Config; +use serde::Serialize; +use version_spec::UnresolvedVersionSpec; +use warpgate_api::PluginLocator; + +#[derive(Clone, Config, Debug, PartialEq, Serialize)] +pub struct PipConfig { + /// List of arguments to append to `pip install` commands. + pub install_args: Option>, +} + +#[derive(Clone, Config, Debug, PartialEq)] +pub struct PythonConfig { + /// Location of the WASM plugin to use for Python support. + pub plugin: Option, + + /// Options for pip, when used as a package manager. + #[setting(nested)] + pub pip: Option, + + /// Defines the virtual environment name which will be created on workspace root. + /// Project dependencies will be installed into this. Defaults to `.venv` + #[setting(default = ".venv")] + pub venv_name: String, + + /// Assumes only the root `requirements.txt` is used for dependencies. + /// Can be used to support the "one version policy" pattern. + #[setting(default = true)] + pub root_requirements_only: bool, + + /// The version of Python to download, install, and run `python` tasks with. + #[setting(env = "MOON_PYTHON_VERSION")] + pub version: Option, +} diff --git a/crates/config/src/toolchain_config.rs b/crates/config/src/toolchain_config.rs index 6ec57d5ade7..42fab4166c1 100644 --- a/crates/config/src/toolchain_config.rs +++ b/crates/config/src/toolchain_config.rs @@ -57,6 +57,10 @@ pub struct ToolchainConfig { #[setting(nested)] pub node: Option, + /// Configures and enables the Python platform. + #[setting(nested)] + pub python: Option, + /// Configures and enables the Rust platform. #[setting(nested)] pub rust: Option, @@ -86,6 +90,10 @@ impl ToolchainConfig { tools.push(PlatformType::Node); } + if self.python.is_some() { + tools.push(PlatformType::Python) + } + if self.rust.is_some() { tools.push(PlatformType::Rust); } @@ -141,6 +149,12 @@ impl ToolchainConfig { } } + if let Some(python_config) = &self.python { + if let Some(version) = &python_config.version { + inject("PROTO_PYTHON_VERSION", version); + } + } + // We don't include Rust since it's a special case! env @@ -155,6 +169,8 @@ impl ToolchainConfig { inherit_tool!(NodeConfig, node, "node", inherit_proto_node); + inherit_tool!(PythonConfig, python, "python", inherit_proto_python); + inherit_tool!(RustConfig, rust, "rust", inherit_proto_rust); inherit_tool_without_version!( @@ -171,6 +187,7 @@ impl ToolchainConfig { is_using_tool_version!(self, node, bun); is_using_tool_version!(self, node, pnpm); is_using_tool_version!(self, node, yarn); + is_using_tool_version!(self, python); is_using_tool_version!(self, rust); // Special case @@ -189,6 +206,7 @@ impl ToolchainConfig { self.inherit_proto_bun(proto_config)?; self.inherit_proto_deno(proto_config)?; self.inherit_proto_node(proto_config)?; + self.inherit_proto_python(proto_config)?; self.inherit_proto_rust(proto_config)?; self.inherit_proto_typescript(proto_config)?; diff --git a/crates/config/tests/__fixtures__/inheritance/files/tasks/python.yml b/crates/config/tests/__fixtures__/inheritance/files/tasks/python.yml new file mode 100644 index 00000000000..02c17dc9145 --- /dev/null +++ b/crates/config/tests/__fixtures__/inheritance/files/tasks/python.yml @@ -0,0 +1,3 @@ +tasks: + python: + command: python diff --git a/crates/config/tests/inherited_tasks_config_test.rs b/crates/config/tests/inherited_tasks_config_test.rs index 90c37987536..6ec2f0cd815 100644 --- a/crates/config/tests/inherited_tasks_config_test.rs +++ b/crates/config/tests/inherited_tasks_config_test.rs @@ -526,6 +526,7 @@ mod task_manager { "node", "node-application", "node-library", + "python", "rust", "tag-camelCase", "tag-dot.case", @@ -792,6 +793,38 @@ mod task_manager { ); } + #[test] + fn creates_python_config() { + let sandbox = create_sandbox("inheritance/files"); + let manager = load_manager_from_root(sandbox.path(), sandbox.path()).unwrap(); + + let config = manager + .get_inherited_config( + &PlatformType::System, + &LanguageType::Python, + &StackType::Frontend, + &ProjectType::Library, + &[], + ) + .unwrap(); + + assert_eq!( + config.config.tasks, + BTreeMap::from_iter([ + ( + Id::raw("global"), + stub_task("global", PlatformType::Unknown) + ), + (Id::raw("python"), stub_task("python", PlatformType::System)), + ]), + ); + + assert_eq!( + config.layers.keys().collect::>(), + vec!["tasks.yml", "tasks/python.yml",] + ); + } + #[test] fn creates_js_config_via_bun() { let sandbox = create_sandbox("inheritance/files"); diff --git a/crates/config/tests/project_config_test.rs b/crates/config/tests/project_config_test.rs index e12fde1ee99..59a6a8a22f3 100644 --- a/crates/config/tests/project_config_test.rs +++ b/crates/config/tests/project_config_test.rs @@ -283,7 +283,7 @@ fileGroups: #[test] #[should_panic( - expected = "unknown variant `perl`, expected one of `bun`, `deno`, `node`, `rust`, `system`, `unknown`" + expected = "Failed to parse moon.yml. platform: unknown variant `perl`, expected one of `bun`, `deno`, `node`, `python`, `rust`, `system`, `unknown`" )] fn errors_on_invalid_variant() { test_load_config("moon.yml", "platform: perl", |path| { diff --git a/crates/config/tests/task_config_test.rs b/crates/config/tests/task_config_test.rs index 9da09900580..f3d2d5de906 100644 --- a/crates/config/tests/task_config_test.rs +++ b/crates/config/tests/task_config_test.rs @@ -350,7 +350,7 @@ outputs: #[test] #[should_panic( - expected = "unknown variant `perl`, expected one of `bun`, `deno`, `node`, `rust`, `system`, `unknown`" + expected = "Failed to parse TaskConfig. platform: unknown variant `perl`, expected one of `bun`, `deno`, `node`, `python`, `rust`, `system`, `unknown`" )] fn errors_on_invalid_variant() { test_parse_config("platform: perl", load_config_from_code); diff --git a/crates/config/tests/toolchain_config_test.rs b/crates/config/tests/toolchain_config_test.rs index b3440139879..19562ab1fdc 100644 --- a/crates/config/tests/toolchain_config_test.rs +++ b/crates/config/tests/toolchain_config_test.rs @@ -48,6 +48,7 @@ mod toolchain_config { assert!(config.deno.is_none()); assert!(config.node.is_none()); + assert!(config.python.is_none()); assert!(config.rust.is_none()); assert!(config.typescript.is_none()); } diff --git a/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml b/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml index fb3ad2ba314..91935a27e17 100644 --- a/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml +++ b/crates/task-builder/tests/__fixtures__/builder/platforms/moon.yml @@ -9,6 +9,10 @@ tasks: command: bun deno-via-cmd: command: deno + python: + platform: python + python-via-cmd: + command: python node: platform: node node-via-cmd: diff --git a/crates/toolchain/src/detect/project_platform.rs b/crates/toolchain/src/detect/project_platform.rs index 163e4ccfcc8..54e1c6cb1b0 100644 --- a/crates/toolchain/src/detect/project_platform.rs +++ b/crates/toolchain/src/detect/project_platform.rs @@ -28,6 +28,13 @@ pub fn detect_project_platform( PlatformType::System } } + LanguageType::Python => { + if enabled_platforms.contains(&PlatformType::Python) { + PlatformType::Python + } else { + PlatformType::System + } + } LanguageType::Rust => { if enabled_platforms.contains(&PlatformType::Rust) { PlatformType::Rust diff --git a/crates/toolchain/src/detect/task_platform.rs b/crates/toolchain/src/detect/task_platform.rs index 30df9a79a6e..d294942854d 100644 --- a/crates/toolchain/src/detect/task_platform.rs +++ b/crates/toolchain/src/detect/task_platform.rs @@ -4,6 +4,7 @@ use std::sync::OnceLock; pub static BUN_COMMANDS: OnceLock = OnceLock::new(); pub static DENO_COMMANDS: OnceLock = OnceLock::new(); +pub static PYTHON_COMMANDS: OnceLock = OnceLock::new(); pub static RUST_COMMANDS: OnceLock = OnceLock::new(); pub static NODE_COMMANDS: OnceLock = OnceLock::new(); pub static UNIX_SYSTEM_COMMANDS: OnceLock = OnceLock::new(); @@ -17,6 +18,9 @@ fn use_platform_if_enabled( PlatformType::Bun if enabled_platforms.contains(&PlatformType::Bun) => return platform, PlatformType::Deno if enabled_platforms.contains(&PlatformType::Deno) => return platform, PlatformType::Node if enabled_platforms.contains(&PlatformType::Node) => return platform, + PlatformType::Python if enabled_platforms.contains(&PlatformType::Python) => { + return platform + } PlatformType::Rust if enabled_platforms.contains(&PlatformType::Rust) => return platform, _ => {} }; @@ -55,6 +59,13 @@ pub fn detect_task_platform(command: &str, enabled_platforms: &[PlatformType]) - return use_platform_if_enabled(PlatformType::Deno, enabled_platforms); } + if PYTHON_COMMANDS + .get_or_init(|| Regex::new("^(python|python3|pip|pip3)$").unwrap()) + .is_match(command) + { + return use_platform_if_enabled(PlatformType::Python, enabled_platforms); + } + if RUST_COMMANDS .get_or_init(|| Regex::new("^(rust-|rustc|rustdoc|rustfmt|rustup|cargo)").unwrap()) .is_match(command) diff --git a/legacy/core/test-utils/src/configs.rs b/legacy/core/test-utils/src/configs.rs index 7e5fe2ab892..e06d71aa219 100644 --- a/legacy/core/test-utils/src/configs.rs +++ b/legacy/core/test-utils/src/configs.rs @@ -484,6 +484,49 @@ pub fn get_node_fixture_configs() -> ( (workspace_config, toolchain_config, tasks_config) } +pub fn get_python_fixture_configs() -> ( + PartialWorkspaceConfig, + PartialToolchainConfig, + PartialInheritedTasksConfig, +) { + let workspace_config = PartialWorkspaceConfig { + projects: Some(PartialWorkspaceProjects::Sources(FxHashMap::from_iter([( + "python".try_into().unwrap(), + "base".to_owned(), + )]))), + ..PartialWorkspaceConfig::default() + }; + + let mut toolchain_config = get_default_toolchain(); + toolchain_config.python = Some(PartialPythonConfig { + version: Some(UnresolvedVersionSpec::parse("3.11.10").unwrap()), + ..PartialPythonConfig::default() + }); + + let tasks_config = PartialInheritedTasksConfig { + tasks: Some(BTreeMap::from_iter([ + ( + "version".try_into().unwrap(), + PartialTaskConfig { + command: Some(PartialTaskArgs::String("python".into())), + args: Some(PartialTaskArgs::String("--version".into())), + ..PartialTaskConfig::default() + }, + ), + ( + "noop".try_into().unwrap(), + PartialTaskConfig { + command: Some(PartialTaskArgs::String("noop".into())), + ..PartialTaskConfig::default() + }, + ), + ])), + ..PartialInheritedTasksConfig::default() + }; + + (workspace_config, toolchain_config, tasks_config) +} + pub fn get_node_depman_fixture_configs( depman: &str, ) -> ( diff --git a/legacy/python/lang/Cargo.toml b/legacy/python/lang/Cargo.toml new file mode 100644 index 00000000000..4ca4c5cd01b --- /dev/null +++ b/legacy/python/lang/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "moon_python_lang" +version = "0.0.1" +edition = "2021" +publish = false + +[dependencies] +moon_lang = { path = "../../core/lang" } +cached = { workspace = true } +miette = { workspace = true } +pep-508 = "0.4.0" +rustc-hash = { workspace = true } + +[dev-dependencies] +moon_test_utils = { path = "../../core/test-utils" } + +[lints] +workspace = true diff --git a/legacy/python/lang/src/lib.rs b/legacy/python/lang/src/lib.rs new file mode 100644 index 00000000000..90a0e45a446 --- /dev/null +++ b/legacy/python/lang/src/lib.rs @@ -0,0 +1,4 @@ +pub mod pip_requirements; + +pub use moon_lang::LockfileDependencyVersions; +pub use pip_requirements::*; diff --git a/legacy/python/lang/src/pip_requirements.rs b/legacy/python/lang/src/pip_requirements.rs new file mode 100644 index 00000000000..9bef4a5be5b --- /dev/null +++ b/legacy/python/lang/src/pip_requirements.rs @@ -0,0 +1,35 @@ +use cached::proc_macro::cached; +use moon_lang::LockfileDependencyVersions; +use pep_508::parse; +use rustc_hash::FxHashMap; +use std::fs::File; +use std::io; +use std::io::BufRead; +use std::path::{Path, PathBuf}; + +fn read_lines

(filename: P) -> io::Result>> +where + P: AsRef, +{ + let file = File::open(filename)?; + Ok(io::BufReader::new(file).lines()) +} + +#[cached(result)] +pub fn load_lockfile_dependencies(path: PathBuf) -> miette::Result { + let mut deps: LockfileDependencyVersions = FxHashMap::default(); + + if let Ok(lines) = read_lines(&path) { + for line in lines.map_while(Result::ok) { + if let Ok(parsed) = parse(&line) { + deps.entry(parsed.name.to_string()) + .and_modify(|dep| { + dep.push(line.clone()); + }) + .or_insert(vec![line.clone()]); + } + } + } + + Ok(deps) +} diff --git a/legacy/python/platform/Cargo.toml b/legacy/python/platform/Cargo.toml new file mode 100644 index 00000000000..c797eaa1e1b --- /dev/null +++ b/legacy/python/platform/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "moon_python_platform" +version = "0.0.1" +edition = "2021" +publish = false + +[dependencies] +moon_action = { path = "../../../crates/action" } +moon_action_context = { path = "../../../crates/action-context" } +moon_common = { path = "../../../crates/common" } +moon_config = { path = "../../../crates/config" } +moon_console = { path = "../../../crates/console" } +moon_hash = { path = "../../../crates/hash" } +moon_logger = { path = "../../core/logger" } +moon_platform = { path = "../../core/platform" } +moon_process = { path = "../../../crates/process" } +moon_project = { path = "../../../crates/project" } +moon_python_lang = { path = "../lang" } +moon_python_tool = { path = "../tool" } +moon_task = { path = "../../../crates/task" } +moon_tool = { path = "../../core/tool" } +moon_utils = { path = "../../core/utils" } +miette = { workspace = true } +proto_core = { workspace = true } +rustc-hash = { workspace = true } +serde = { workspace = true } +starbase_styles = { workspace = true } +starbase_utils = { workspace = true, features = ["glob"] } +tokio = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +moon_test_utils = { path = "../../core/test-utils" } + +[lints] +workspace = true diff --git a/legacy/python/platform/src/actions/install_deps.rs b/legacy/python/platform/src/actions/install_deps.rs new file mode 100644 index 00000000000..03adf199b3d --- /dev/null +++ b/legacy/python/platform/src/actions/install_deps.rs @@ -0,0 +1,68 @@ +use moon_action::Operation; +use moon_console::{Checkpoint, Console}; +use moon_python_tool::PythonTool; +use std::path::Path; + +use crate::find_requirements_txt; + +pub async fn install_deps( + python: &PythonTool, + workspace_root: &Path, + working_dir: &Path, + console: &Console, +) -> miette::Result> { + let mut operations = vec![]; + + if let Some(pip_config) = &python.config.pip { + let requirements_path = find_requirements_txt(working_dir, workspace_root); + let virtual_environment = if python.config.root_requirements_only { + &workspace_root.join(python.config.venv_name.clone()) + } else { + &working_dir.join(python.config.venv_name.clone()) + }; + + if !virtual_environment.exists() { + console + .out + .print_checkpoint(Checkpoint::Setup, "activate virtual environment")?; + let args = vec![ + "-m", + "venv", + virtual_environment.as_os_str().to_str().unwrap(), + ]; + operations.push( + Operation::task_execution(format!("python {} ", args.join(" "))) + .track_async(|| python.exec_python(args, workspace_root)) + .await?, + ); + } + + let mut args = vec![]; + + // Add pip installArgs, if users have given + if let Some(install_args) = &pip_config.install_args { + args.extend(install_args.iter().map(|c| c.as_str())); + } + + // Add requirements.txt path, if found + if let Some(req) = &requirements_path { + args.extend(["-r", req.as_os_str().to_str().unwrap()]); + } + + if !args.is_empty() { + args.splice(0..0, vec!["-m", "pip", "install"]); + + console + .out + .print_checkpoint(Checkpoint::Setup, "pip install")?; + + operations.push( + Operation::task_execution(format!("python {}", args.join(" "))) + .track_async(|| python.exec_python(args, working_dir)) + .await?, + ); + } + } + + Ok(operations) +} diff --git a/legacy/python/platform/src/actions/mod.rs b/legacy/python/platform/src/actions/mod.rs new file mode 100644 index 00000000000..31a65cfdfdf --- /dev/null +++ b/legacy/python/platform/src/actions/mod.rs @@ -0,0 +1,3 @@ +mod install_deps; + +pub use install_deps::*; diff --git a/legacy/python/platform/src/lib.rs b/legacy/python/platform/src/lib.rs new file mode 100644 index 00000000000..3e1a597bf12 --- /dev/null +++ b/legacy/python/platform/src/lib.rs @@ -0,0 +1,12 @@ +pub mod actions; +mod python_platform; +mod toolchain_hash; + +pub use python_platform::*; + +use starbase_utils::fs; +use std::path::{Path, PathBuf}; + +fn find_requirements_txt(starting_dir: &Path, workspace_root: &Path) -> Option { + fs::find_upwards_until("requirements.txt", starting_dir, workspace_root) +} diff --git a/legacy/python/platform/src/python_platform.rs b/legacy/python/platform/src/python_platform.rs new file mode 100644 index 00000000000..5a8e23c0301 --- /dev/null +++ b/legacy/python/platform/src/python_platform.rs @@ -0,0 +1,314 @@ +use crate::{actions, find_requirements_txt, toolchain_hash::PythonToolchainHash}; +use moon_action::Operation; +use moon_action_context::ActionContext; +use moon_common::{path::is_root_level_source, Id}; +use moon_config::{ + HasherConfig, PlatformType, ProjectConfig, ProjectsAliasesList, ProjectsSourcesList, + PythonConfig, UnresolvedVersionSpec, +}; +use moon_console::Console; +use moon_hash::ContentHasher; +use moon_platform::{Platform, Runtime, RuntimeReq}; +use moon_process::Command; +use moon_project::Project; +use moon_python_lang::pip_requirements::load_lockfile_dependencies; +use moon_python_tool::{get_python_tool_paths, PythonTool}; +use moon_task::Task; +use moon_tool::{get_proto_version_env, prepend_path_env_var, Tool, ToolManager}; +use moon_utils::async_trait; +use proto_core::ProtoEnvironment; +use rustc_hash::FxHashMap; +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, + sync::Arc, +}; +use tracing::instrument; + +pub struct PythonPlatform { + pub config: PythonConfig, + + console: Arc, + + proto_env: Arc, + + toolchain: ToolManager, + + #[allow(dead_code)] + pub workspace_root: PathBuf, +} + +impl PythonPlatform { + pub fn new( + config: &PythonConfig, + workspace_root: &Path, + proto_env: Arc, + console: Arc, + ) -> Self { + PythonPlatform { + config: config.to_owned(), + proto_env, + toolchain: ToolManager::new(Runtime::new(PlatformType::Python, RuntimeReq::Global)), + workspace_root: workspace_root.to_path_buf(), + console, + } + } +} + +#[async_trait] +impl Platform for PythonPlatform { + fn get_type(&self) -> PlatformType { + PlatformType::Python + } + + fn get_runtime_from_config(&self, project_config: Option<&ProjectConfig>) -> Runtime { + if let Some(config) = &project_config { + if let Some(python_config) = &config.toolchain.python { + if let Some(version) = &python_config.version { + return Runtime::new_override( + PlatformType::Python, + RuntimeReq::Toolchain(version.to_owned()), + ); + } + } + } + + if let Some(version) = &self.config.version { + return Runtime::new( + PlatformType::Python, + RuntimeReq::Toolchain(version.to_owned()), + ); + } + + Runtime::new(PlatformType::Python, RuntimeReq::Global) + } + + fn matches(&self, platform: &PlatformType, runtime: Option<&Runtime>) -> bool { + if matches!(platform, PlatformType::Python) { + return true; + } + + if let Some(runtime) = &runtime { + return matches!(runtime.platform, PlatformType::Python); + } + + false + } + + // PROJECT GRAPH + + fn is_project_in_dependency_workspace(&self, project_source: &str) -> miette::Result { + // Single version policy / only a root requirements.txt + if self.config.root_requirements_only { + return Ok(true); + } + + if is_root_level_source(project_source) { + return Ok(true); + } + + Ok(false) + } + + #[instrument(skip_all)] + fn load_project_graph_aliases( + &mut self, + _projects_list: &ProjectsSourcesList, + _aliases_list: &mut ProjectsAliasesList, + ) -> miette::Result<()> { + // Not supported + Ok(()) + } + + // TOOLCHAIN + + fn is_toolchain_enabled(&self) -> miette::Result { + Ok(self.config.version.is_some()) + } + + fn get_tool(&self) -> miette::Result> { + let tool = self.toolchain.get()?; + + Ok(Box::new(tool)) + } + + fn get_tool_for_version(&self, req: RuntimeReq) -> miette::Result> { + let tool = self.toolchain.get_for_version(&req)?; + + Ok(Box::new(tool)) + } + + fn get_dependency_configs(&self) -> miette::Result> { + Ok(Some(( + "requirements.txt".to_owned(), + "requirements.txt".to_owned(), + ))) + } + + async fn setup_toolchain(&mut self) -> miette::Result<()> { + let req = match &self.config.version { + Some(v) => RuntimeReq::Toolchain(v.to_owned()), + None => RuntimeReq::Global, + }; + + let mut last_versions = FxHashMap::default(); + + if !self.toolchain.has(&req) { + self.toolchain.register( + &req, + PythonTool::new( + Arc::clone(&self.proto_env), + Arc::clone(&self.console), + &self.config, + &req, + ) + .await?, + ); + } + + self.toolchain.setup(&req, &mut last_versions).await?; + + Ok(()) + } + + async fn teardown_toolchain(&mut self) -> miette::Result<()> { + self.toolchain.teardown_all().await?; + + Ok(()) + } + + // ACTIONS + + #[instrument(skip_all)] + async fn setup_tool( + &mut self, + _context: &ActionContext, + runtime: &Runtime, + last_versions: &mut FxHashMap, + ) -> miette::Result { + let req = &runtime.requirement; + + if !self.toolchain.has(req) { + self.toolchain.register( + req, + PythonTool::new( + Arc::clone(&self.proto_env), + Arc::clone(&self.console), + &self.config, + req, + ) + .await?, + ); + } + + let installed = self.toolchain.setup(req, last_versions).await?; + + Ok(installed) + } + + #[instrument(skip_all)] + async fn install_deps( + &self, + _context: &ActionContext, + runtime: &Runtime, + working_dir: &Path, + ) -> miette::Result> { + actions::install_deps( + self.toolchain.get_for_version(&runtime.requirement)?, + self.workspace_root.as_path(), + working_dir, + &self.console, + ) + .await + } + + #[instrument(skip_all)] + async fn sync_project( + &self, + _context: &ActionContext, + _project: &Project, + _dependencies: &FxHashMap>, + ) -> miette::Result { + Ok(false) + } + + #[instrument(skip_all)] + async fn hash_manifest_deps( + &self, + manifest_path: &Path, + hasher: &mut ContentHasher, + _hasher_config: &HasherConfig, + ) -> miette::Result<()> { + let deps = BTreeMap::from_iter(load_lockfile_dependencies(manifest_path.to_path_buf())?); + hasher.hash_content(PythonToolchainHash { + version: self + .config + .version + .as_ref() + .map(|v| v.to_string()) + .unwrap_or_default(), + dependencies: deps, + })?; + + Ok(()) + } + + #[instrument(skip_all)] + async fn hash_run_target( + &self, + project: &Project, + _runtime: &Runtime, + hasher: &mut ContentHasher, + _hasher_config: &HasherConfig, + ) -> miette::Result<()> { + let mut deps = BTreeMap::new(); + if let Some(pip_requirements) = find_requirements_txt(&project.root, &self.workspace_root) { + deps = BTreeMap::from_iter(load_lockfile_dependencies(pip_requirements)?); + } + hasher.hash_content(PythonToolchainHash { + version: self + .config + .version + .as_ref() + .map(|v| v.to_string()) + .unwrap_or_default(), + dependencies: deps, + })?; + + Ok(()) + } + + #[instrument(skip_all)] + async fn create_run_target_command( + &self, + _context: &ActionContext, + _project: &Project, + task: &Task, + runtime: &Runtime, + working_dir: &Path, + ) -> miette::Result { + let mut command = Command::new(&task.command); + + command.with_console(self.console.clone()); + command.args(&task.args); + command.envs(&task.env); + + if let Ok(python) = self.toolchain.get_for_version(&runtime.requirement) { + if let Some(version) = get_proto_version_env(&python.tool) { + let cwd = if python.config.root_requirements_only { + self.workspace_root.as_path() + } else { + working_dir + }; + + command.env("PROTO_PYTHON_VERSION", version); + command.env( + "PATH", + prepend_path_env_var(get_python_tool_paths(python, cwd)), + ); + } + } + + Ok(command) + } +} diff --git a/legacy/python/platform/src/toolchain_hash.rs b/legacy/python/platform/src/toolchain_hash.rs new file mode 100644 index 00000000000..fcaf96ddc6e --- /dev/null +++ b/legacy/python/platform/src/toolchain_hash.rs @@ -0,0 +1,9 @@ +use moon_hash::hash_content; +use std::collections::BTreeMap; + +hash_content!( + pub struct PythonToolchainHash { + pub version: String, + pub dependencies: BTreeMap>, + } +); diff --git a/legacy/python/tool/Cargo.toml b/legacy/python/tool/Cargo.toml new file mode 100644 index 00000000000..8e9cba3f79b --- /dev/null +++ b/legacy/python/tool/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "moon_python_tool" +version = "0.0.1" +edition = "2021" +publish = false + +[dependencies] +moon_python_lang = { path = "../lang" } +moon_common = { path = "../../../crates/common" } +moon_config = { path = "../../../crates/config" } +moon_console = { path = "../../../crates/console" } +moon_logger = { path = "../../core/logger" } +moon_process = { path = "../../../crates/process" } +moon_tool = { path = "../../core/tool" } +moon_utils = { path = "../../core/utils" } +moon_toolchain = { path = "../../../crates/toolchain" } +starbase_styles = { workspace = true } +miette = { workspace = true } +proto_core = { workspace = true } +rustc-hash = { workspace = true } +starbase_utils = { workspace = true } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/legacy/python/tool/src/lib.rs b/legacy/python/tool/src/lib.rs new file mode 100644 index 00000000000..84807fe6f11 --- /dev/null +++ b/legacy/python/tool/src/lib.rs @@ -0,0 +1,3 @@ +mod python_tool; + +pub use python_tool::*; diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs new file mode 100644 index 00000000000..1b62a57ee85 --- /dev/null +++ b/legacy/python/tool/src/python_tool.rs @@ -0,0 +1,162 @@ +use moon_config::PythonConfig; +use moon_console::{Checkpoint, Console}; +use moon_logger::{debug, map_list}; +use moon_process::Command; +use moon_tool::{ + async_trait, get_proto_env_vars, get_proto_paths, load_tool_plugin, prepend_path_env_var, + use_global_tool_on_path, Tool, +}; +use moon_toolchain::RuntimeReq; +use proto_core::flow::install::InstallOptions; +use proto_core::{Id, ProtoEnvironment, Tool as ProtoTool, UnresolvedVersionSpec}; +use rustc_hash::FxHashMap; +use starbase_styles::color; +use std::path::PathBuf; +use std::sync::Arc; +use std::{ffi::OsStr, path::Path}; +use tracing::instrument; + +const LOG_TARGET: &str = "moon:python-tool"; + +pub fn get_python_tool_paths(python_tool: &PythonTool, working_dir: &Path) -> Vec { + let venv_python = working_dir.join(python_tool.config.venv_name.clone()); + + let paths = if venv_python.exists() { + vec![ + venv_python.join("Scripts").clone(), + venv_python.join("bin").clone(), + ] + } else { + get_proto_paths(&python_tool.proto_env) + }; + + debug!( + target: LOG_TARGET, + "Proto Env {} ", + map_list(&paths, |c| color::label(c.display().to_string())), + ); + paths +} + +pub struct PythonTool { + pub config: PythonConfig, + + pub global: bool, + + pub tool: ProtoTool, + + console: Arc, + + proto_env: Arc, +} + +impl PythonTool { + pub async fn new( + proto_env: Arc, + console: Arc, + config: &PythonConfig, + req: &RuntimeReq, + ) -> miette::Result { + let mut python = PythonTool { + config: config.to_owned(), + global: false, + tool: load_tool_plugin( + &Id::raw("python"), + &proto_env, + config.plugin.as_ref().unwrap(), + ) + .await?, + proto_env, + console, + }; + + if use_global_tool_on_path("python") || req.is_global() { + python.global = true; + python.config.version = None; + } else { + python.config.version = req.to_spec(); + }; + + Ok(python) + } + + #[instrument(skip_all)] + pub async fn exec_python(&self, args: I, working_dir: &Path) -> miette::Result<()> + where + I: IntoIterator, + S: AsRef, + { + Command::new("python") + .args(args) + .envs(get_proto_env_vars()) + .env( + "PATH", + prepend_path_env_var(get_python_tool_paths(self, working_dir)), + ) + .cwd(working_dir) + .with_console(self.console.clone()) + .create_async() + .exec_stream_output() + .await?; + + Ok(()) + } +} + +#[async_trait] +impl Tool for PythonTool { + fn as_any(&self) -> &(dyn std::any::Any + Send + Sync) { + self + } + + #[instrument(skip_all)] + async fn setup( + &mut self, + last_versions: &mut FxHashMap, + ) -> miette::Result { + let mut installed = 0; + + let Some(version) = &self.config.version else { + return Ok(installed); + }; + + if self.global { + debug!("Using global binary in PATH"); + } else if self.tool.is_setup(version).await? { + debug!("Python has already been setup"); + + // When offline and the tool doesn't exist, fallback to the global binary + } else if proto_core::is_offline() { + debug!( + "No internet connection and Python has not been setup, falling back to global binary in PATH" + ); + + self.global = true; + + // Otherwise try and install the tool + } else { + let setup = match last_versions.get("python") { + Some(last) => version != last, + None => true, + }; + + if setup || !self.tool.get_product_dir().exists() { + self.console + .out + .print_checkpoint(Checkpoint::Setup, format!("installing python {version}"))?; + + if self.tool.setup(version, InstallOptions::default()).await? { + last_versions.insert("python".into(), version.to_owned()); + installed += 1; + } + } + } + self.tool.locate_globals_dirs().await?; + Ok(installed) + } + + async fn teardown(&mut self) -> miette::Result<()> { + self.tool.teardown().await?; + Ok(()) + } +} diff --git a/packages/types/src/project-config.ts b/packages/types/src/project-config.ts index f72824ed212..548d96ff69c 100644 --- a/packages/types/src/project-config.ts +++ b/packages/types/src/project-config.ts @@ -167,6 +167,8 @@ export interface ProjectToolchainConfig { deno: ProjectToolchainCommonToolConfig | null; /** Overrides `node` settings. */ node: ProjectToolchainCommonToolConfig | null; + /** Overrides `python` settings. */ + python: ProjectToolchainCommonToolConfig | null; /** Overrides `rust` settings. */ rust: ProjectToolchainCommonToolConfig | null; /** Overrides `typescript` settings. */ @@ -408,6 +410,8 @@ export interface PartialProjectToolchainConfig { deno?: PartialProjectToolchainCommonToolConfig | null; /** Overrides `node` settings. */ node?: PartialProjectToolchainCommonToolConfig | null; + /** Overrides `python` settings. */ + python?: PartialProjectToolchainCommonToolConfig | null; /** Overrides `rust` settings. */ rust?: PartialProjectToolchainCommonToolConfig | null; /** Overrides `typescript` settings. */ diff --git a/packages/types/src/tasks-config.ts b/packages/types/src/tasks-config.ts index cc4773fce9a..84dfbd33761 100644 --- a/packages/types/src/tasks-config.ts +++ b/packages/types/src/tasks-config.ts @@ -165,7 +165,7 @@ export interface TaskOptionsConfig { } /** Platforms that each programming language can belong to. */ -export type PlatformType = 'bun' | 'deno' | 'node' | 'rust' | 'system' | 'unknown'; +export type PlatformType = 'bun' | 'deno' | 'node' | 'python' | 'rust' | 'system' | 'unknown'; /** Preset options to inherit. */ export type TaskPreset = 'server' | 'watcher'; @@ -228,7 +228,7 @@ export interface TaskConfig { * be automatically detected. * * @default 'unknown' - * @type {'bun' | 'deno' | 'node' | 'rust' | 'system' | 'unknown'} + * @type {'bun' | 'deno' | 'node' | 'python' | 'rust' | 'system' | 'unknown'} */ platform: PlatformType; /** The preset to apply for the task. Will inherit default options. */ diff --git a/packages/types/src/toolchain-config.ts b/packages/types/src/toolchain-config.ts index 23e782bea43..7f1917f411a 100644 --- a/packages/types/src/toolchain-config.ts +++ b/packages/types/src/toolchain-config.ts @@ -259,6 +259,38 @@ export interface NodeConfig { yarn: YarnConfig | null; } +export interface PipConfig { + /** List of arguments to append to `pip install` commands. */ + installArgs: string[] | null; +} + +export interface PythonConfig { + /** Options for pip, when used as a package manager. */ + pip: PipConfig | null; + /** Location of the WASM plugin to use for Python support. */ + plugin: PluginLocator | null; + /** + * Assumes only the root `requirements.txt` is used for dependencies. + * Can be used to support the "one version policy" pattern. + * + * @default true + */ + rootRequirementsOnly?: boolean; + /** + * Defines the virtual environment name which will be created on workspace root. + * Project dependencies will be installed into this. Defaults to `.venv` + * + * @default '.venv' + */ + venvName?: string; + /** + * The version of Python to download, install, and run `python` tasks with. + * + * @envvar MOON_PYTHON_VERSION + */ + version: UnresolvedVersionSpec | null; +} + /** * Configures and enables the Rust platform. * Docs: https://moonrepo.dev/docs/config/toolchain#rust @@ -378,6 +410,8 @@ export interface ToolchainConfig { moon: MoonConfig; /** Configures and enables the Node.js platform. */ node: NodeConfig | null; + /** Configures and enables the Python platform. */ + python: PythonConfig | null; /** Configures and enables the Rust platform. */ rust: RustConfig | null; /** All configured toolchains by unique ID. */ @@ -618,6 +652,38 @@ export interface PartialNodeConfig { yarn?: PartialYarnConfig | null; } +export interface PartialPipConfig { + /** List of arguments to append to `pip install` commands. */ + installArgs?: string[] | null; +} + +export interface PartialPythonConfig { + /** Options for pip, when used as a package manager. */ + pip?: PartialPipConfig | null; + /** Location of the WASM plugin to use for Python support. */ + plugin?: PluginLocator | null; + /** + * Assumes only the root `requirements.txt` is used for dependencies. + * Can be used to support the "one version policy" pattern. + * + * @default true + */ + rootRequirementsOnly?: boolean | null; + /** + * Defines the virtual environment name which will be created on workspace root. + * Project dependencies will be installed into this. Defaults to `.venv` + * + * @default '.venv' + */ + venvName?: string | null; + /** + * The version of Python to download, install, and run `python` tasks with. + * + * @envvar MOON_PYTHON_VERSION + */ + version?: UnresolvedVersionSpec | null; +} + /** * Configures and enables the Rust platform. * Docs: https://moonrepo.dev/docs/config/toolchain#rust @@ -737,6 +803,8 @@ export interface PartialToolchainConfig { moon?: PartialMoonConfig | null; /** Configures and enables the Node.js platform. */ node?: PartialNodeConfig | null; + /** Configures and enables the Python platform. */ + python?: PartialPythonConfig | null; /** Configures and enables the Rust platform. */ rust?: PartialRustConfig | null; /** All configured toolchains by unique ID. */ diff --git a/scripts/new-language.md b/scripts/new-language.md index 260b67c10af..5fd30c05969 100644 --- a/scripts/new-language.md +++ b/scripts/new-language.md @@ -288,4 +288,4 @@ At this point we should start updating docs, primarily these sections: ### Create a pull request Once everything is good, create a pull request and include it in the next release. Ideally tiers are -released separately! +released separately! \ No newline at end of file diff --git a/tests/fixtures/python/base/moon.yml b/tests/fixtures/python/base/moon.yml new file mode 100644 index 00000000000..d9bcbc7a3a3 --- /dev/null +++ b/tests/fixtures/python/base/moon.yml @@ -0,0 +1,12 @@ +language: python + +tasks: + standard: + command: python + args: + - --version + + poetry: + command: poetry + args: + - --version \ No newline at end of file diff --git a/website/docs/__partials__/setup-toolchain/python/tier2.mdx b/website/docs/__partials__/setup-toolchain/python/tier2.mdx index 21ef77ca7f2..e7c4c6e2d8f 100644 --- a/website/docs/__partials__/setup-toolchain/python/tier2.mdx +++ b/website/docs/__partials__/setup-toolchain/python/tier2.mdx @@ -1,6 +1,4 @@ -:::warning - -Python does not implement tier 2 platform support. However, Python based tasks can still be executed -using the default system platform. - -::: +```yaml title=".moon/toolchain.yml" +python: + pip: {} +``` diff --git a/website/docs/__partials__/setup-toolchain/python/tier3.mdx b/website/docs/__partials__/setup-toolchain/python/tier3.mdx index f95b497d0d6..902b7adb8ea 100644 --- a/website/docs/__partials__/setup-toolchain/python/tier3.mdx +++ b/website/docs/__partials__/setup-toolchain/python/tier3.mdx @@ -1,6 +1,6 @@ -:::warning - -Python does not implement tier 3 tool support. The required `python` and related binaries must exist -on `PATH`. - -::: +```yaml title=".moon/toolchain.yml" +python: + version: '3.11.10' + pip: + version: 'latest' +``` diff --git a/website/docs/config/toolchain.mdx b/website/docs/config/toolchain.mdx index 5bfafc33f8b..0edd47e571a 100644 --- a/website/docs/config/toolchain.mdx +++ b/website/docs/config/toolchain.mdx @@ -721,6 +721,71 @@ Both imports can optionally be nested within a `src` directory. > This setting runs _after_ [`syncProjectReferences`](#syncprojectreferences) and will inherit any > synced references from that setting. +## Python + +## `python` + + + +Enables and configures Python. + +### `version` + + + +Defines the explicit Python toolchain If this field is _not defined_, the global `python` binary +will be used. + +```yaml title=".moon/toolchain.yml" {2} +python: + version: '3.11.10' +``` + +> Version can also be defined with [`.prototools`](../proto/config). + +### `rootPackageOnly` + + + +Supports the "single version policy" or "one version rule" patterns by only allowing dependencies in +the root `requirements.txt`, and only installing dependencies in the workspace root, and not within +individual projects. It also bypasses all `workspaces` checks to determine package locations. +Defaults to `true`. + +```yaml title=".moon/toolchain.yml" {2} +python: + rootPackageOnly: false +``` + +### `venv_name` + + + +Defines the virtual environment name which will be created on workspace root, project dependencies +will be installed into this. Defaults to `.venv` + +```yaml title=".moon/toolchain.yml" {2} +python: + venv_name: '.my-custom-venv' +``` + +### `pip` + + + +#### `install_args` + + + +Customize the arguments that will be passed to the pip install command, when the `InstallDeps` +action is triggered in the pipeline. These arguments are used both locally and in CI. + +```yaml title=".moon/toolchain.yml" {3} +python: + pip: + installArgs: ['--trusted-host company.repo.com', '-i https://company.repo.com/simple'] +``` + ## Rust ## `rust` diff --git a/website/static/schemas/project.json b/website/static/schemas/project.json index e1f7e4830a1..d95b059945e 100644 --- a/website/static/schemas/project.json +++ b/website/static/schemas/project.json @@ -376,6 +376,7 @@ "bun", "deno", "node", + "python", "rust", "system", "unknown" @@ -616,6 +617,19 @@ ], "markdownDescription": "Overrides `node` settings." }, + "python": { + "title": "python", + "description": "Overrides python settings.", + "anyOf": [ + { + "$ref": "#/definitions/ProjectToolchainCommonToolConfig" + }, + { + "type": "null" + } + ], + "markdownDescription": "Overrides `python` settings." + }, "rust": { "title": "rust", "description": "Overrides rust settings.", diff --git a/website/static/schemas/tasks.json b/website/static/schemas/tasks.json index 0f9cbedbce9..e2e106b1538 100644 --- a/website/static/schemas/tasks.json +++ b/website/static/schemas/tasks.json @@ -83,6 +83,7 @@ "bun", "deno", "node", + "python", "rust", "system", "unknown" diff --git a/website/static/schemas/toolchain.json b/website/static/schemas/toolchain.json index febaccbfe36..88707e5dfdd 100644 --- a/website/static/schemas/toolchain.json +++ b/website/static/schemas/toolchain.json @@ -65,6 +65,18 @@ } ] }, + "python": { + "title": "python", + "description": "Configures and enables the Python platform.", + "anyOf": [ + { + "$ref": "#/definitions/PythonConfig" + }, + { + "type": "null" + } + ] + }, "rust": { "title": "rust", "description": "Configures and enables the Rust platform.", @@ -568,6 +580,28 @@ }, "additionalProperties": false }, + "PipConfig": { + "type": "object", + "properties": { + "installArgs": { + "title": "installArgs", + "description": "List of arguments to append to pip install commands.", + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ], + "markdownDescription": "List of arguments to append to `pip install` commands." + } + }, + "additionalProperties": false + }, "PluginLocator": { "description": "Strategies and protocols for locating plugins.", "type": "string" @@ -613,6 +647,63 @@ }, "additionalProperties": false }, + "PythonConfig": { + "type": "object", + "properties": { + "pip": { + "title": "pip", + "description": "Options for pip, when used as a package manager.", + "anyOf": [ + { + "$ref": "#/definitions/PipConfig" + }, + { + "type": "null" + } + ] + }, + "plugin": { + "title": "plugin", + "description": "Location of the WASM plugin to use for Python support.", + "anyOf": [ + { + "$ref": "#/definitions/PluginLocator" + }, + { + "type": "null" + } + ] + }, + "rootRequirementsOnly": { + "title": "rootRequirementsOnly", + "description": "Assumes only the root requirements.txt is used for dependencies. Can be used to support the \"one version policy\" pattern.", + "default": true, + "type": "boolean", + "markdownDescription": "Assumes only the root `requirements.txt` is used for dependencies. Can be used to support the \"one version policy\" pattern." + }, + "venvName": { + "title": "venvName", + "description": "Defines the virtual environment name which will be created on workspace root. Project dependencies will be installed into this. Defaults to .venv", + "default": ".venv", + "type": "string", + "markdownDescription": "Defines the virtual environment name which will be created on workspace root. Project dependencies will be installed into this. Defaults to `.venv`" + }, + "version": { + "title": "version", + "description": "The version of Python to download, install, and run python tasks with.", + "anyOf": [ + { + "$ref": "#/definitions/UnresolvedVersionSpec" + }, + { + "type": "null" + } + ], + "markdownDescription": "The version of Python to download, install, and run `python` tasks with." + } + }, + "additionalProperties": false + }, "RustConfig": { "description": "Configures and enables the Rust platform. Docs: https://moonrepo.dev/docs/config/toolchain#rust", "type": "object",