diff --git a/Cargo.lock b/Cargo.lock index 7d397295104..0c19065c973 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1460,6 +1460,7 @@ version = "0.1.0" dependencies = [ "regex", "serde_json", + "serde_yaml 0.9.4", "thiserror", ] @@ -1494,9 +1495,11 @@ dependencies = [ "moon_lang", "moon_logger", "moon_utils", + "pretty_assertions", "regex", "serde", "serde_json", + "serde_yaml 0.9.4", "serial_test", "tokio", ] diff --git a/crates/action-runner/src/actions/run_target.rs b/crates/action-runner/src/actions/run_target.rs index d2e105988b0..1022da1f1e2 100644 --- a/crates/action-runner/src/actions/run_target.rs +++ b/crates/action-runner/src/actions/run_target.rs @@ -610,7 +610,7 @@ pub async fn run_target( runner .is_cached( common_hasher, - node_actions::create_target_hasher(&workspace, &project)?, + node_actions::create_target_hasher(&workspace, &project).await?, ) .await? } diff --git a/crates/config/src/workspace/config.rs b/crates/config/src/workspace/config.rs index e4c7bb405c0..49c833d1552 100644 --- a/crates/config/src/workspace/config.rs +++ b/crates/config/src/workspace/config.rs @@ -6,6 +6,7 @@ use crate::providers::url::Url; use crate::types::{FileGlob, FilePath}; use crate::validators::{validate_child_relative_path, validate_extends, validate_id}; use crate::workspace::action_runner::ActionRunnerConfig; +use crate::workspace::hasher::HasherConfig; use crate::workspace::node::NodeConfig; use crate::workspace::typescript::TypeScriptConfig; use crate::workspace::vcs::VcsConfig; @@ -68,6 +69,9 @@ pub struct WorkspaceConfig { #[validate(custom = "validate_extends")] pub extends: Option, + #[validate] + pub hasher: HasherConfig, + #[validate] pub node: Option, diff --git a/crates/config/src/workspace/hasher.rs b/crates/config/src/workspace/hasher.rs new file mode 100644 index 00000000000..a423002611d --- /dev/null +++ b/crates/config/src/workspace/hasher.rs @@ -0,0 +1,18 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use validator::Validate; + +#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum HasherOptimization { + #[default] + Accuracy, + Performance, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize, Validate)] +#[schemars(default)] +#[serde(rename_all = "camelCase")] +pub struct HasherConfig { + pub optimization: HasherOptimization, +} diff --git a/crates/config/src/workspace/mod.rs b/crates/config/src/workspace/mod.rs index 1598cd0d686..3a045aebb0a 100644 --- a/crates/config/src/workspace/mod.rs +++ b/crates/config/src/workspace/mod.rs @@ -1,11 +1,13 @@ mod action_runner; mod config; +mod hasher; mod node; mod typescript; mod vcs; pub use action_runner::*; pub use config::*; +pub use hasher::*; pub use node::*; pub use typescript::*; pub use vcs::*; diff --git a/crates/config/tests/workspace_test.rs b/crates/config/tests/workspace_test.rs index b816609a5b7..68191a1d07d 100644 --- a/crates/config/tests/workspace_test.rs +++ b/crates/config/tests/workspace_test.rs @@ -1,6 +1,6 @@ use moon_config::{ - ActionRunnerConfig, ConfigError, NodeConfig, VcsConfig, VcsManager, WorkspaceConfig, - WorkspaceProjects, + ActionRunnerConfig, ConfigError, HasherConfig, NodeConfig, VcsConfig, VcsManager, + WorkspaceConfig, WorkspaceProjects, }; use moon_constants::CONFIG_WORKSPACE_FILENAME; use moon_utils::test::get_fixtures_dir; @@ -29,6 +29,7 @@ fn loads_defaults() { WorkspaceConfig { action_runner: ActionRunnerConfig::default(), extends: None, + hasher: HasherConfig::default(), node: None, projects: WorkspaceProjects::default(), typescript: None, @@ -287,6 +288,7 @@ node: WorkspaceConfig { action_runner: ActionRunnerConfig::default(), extends: None, + hasher: HasherConfig::default(), node: Some(NodeConfig { package_manager: NodePackageManager::Yarn, ..NodeConfig::default() @@ -786,6 +788,7 @@ vcs: WorkspaceConfig { action_runner: ActionRunnerConfig::default(), extends: None, + hasher: HasherConfig::default(), node: None, // NodeConfig::default(), projects: WorkspaceProjects::default(), typescript: None, diff --git a/crates/error/Cargo.toml b/crates/error/Cargo.toml index cc0415e0d73..2ccf1ef2669 100644 --- a/crates/error/Cargo.toml +++ b/crates/error/Cargo.toml @@ -6,4 +6,5 @@ edition = "2021" [dependencies] regex = "1.6.0" serde_json = { version = "1.0.82", default-features = false } +serde_yaml = { version = "0.9.4", default-features = false } thiserror = "1.0.31" diff --git a/crates/error/src/lib.rs b/crates/error/src/lib.rs index 64e94ad23dd..470c0a78b05 100644 --- a/crates/error/src/lib.rs +++ b/crates/error/src/lib.rs @@ -1,5 +1,6 @@ use regex::Error as RegexError; use serde_json::Error as JsonError; +use serde_yaml::Error as YamlError; use std::io::{Error as IoError, ErrorKind as IoErrorKind}; use std::path::PathBuf; use thiserror::Error; @@ -40,6 +41,9 @@ pub enum MoonError { #[error("Process {0} failed with a {1} exit code.\n{2}")] ProcessNonZeroWithOutput(String, i32, String), + #[error("Failed to parse {0}: {1}")] + Yaml(PathBuf, #[source] YamlError), + #[error(transparent)] Io(#[from] IoError), diff --git a/crates/lang-node/Cargo.toml b/crates/lang-node/Cargo.toml index ab019959fa6..db4d30baf89 100644 --- a/crates/lang-node/Cargo.toml +++ b/crates/lang-node/Cargo.toml @@ -15,8 +15,10 @@ lazy_static = "1.4.0" regex = "1.6.0" serde = { version = "1.0.140", features = ["derive"] } serde_json = { version = "1.0.82", features = ["preserve_order"] } +serde_yaml = "0.9.4" [dev-dependencies] assert_fs = "1.0.7" +pretty_assertions = "1.2.1" serial_test = "0.8.0" tokio = { version = "1.20.0", features = ["test-util"] } diff --git a/crates/lang-node/src/lib.rs b/crates/lang-node/src/lib.rs index 352058c74c5..e43a210f9dd 100644 --- a/crates/lang-node/src/lib.rs +++ b/crates/lang-node/src/lib.rs @@ -1,6 +1,10 @@ pub mod node; +pub mod npm; pub mod package; +pub mod pnpm; pub mod tsconfig; +pub mod yarn; +pub mod yarn_classic; use moon_lang::{Language, PackageManager, VersionManager}; diff --git a/crates/lang-node/src/npm.rs b/crates/lang-node/src/npm.rs new file mode 100644 index 00000000000..dcb3939ead3 --- /dev/null +++ b/crates/lang-node/src/npm.rs @@ -0,0 +1,167 @@ +use cached::proc_macro::cached; +use moon_error::MoonError; +use moon_lang::config_cache; +use moon_lang::LockfileDependencyVersions; +use moon_utils::fs::sync_read_json; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +config_cache!( + PackageLock, + "package-lock.json", + sync_read_json, + write_lockfile +); + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PackageLockDependency { + pub dependencies: Option>, + pub dev: Option, + pub integrity: Option, + pub requires: Option>, + pub resolved: Option, + pub version: String, + + #[serde(flatten)] + pub unknown: HashMap, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PackageLock { + pub lockfile_version: Value, + pub name: String, + pub dependencies: Option>, + pub packages: Option>, + pub requires: Option, + + #[serde(flatten)] + pub unknown: HashMap, + + #[serde(skip)] + pub path: PathBuf, +} + +fn write_lockfile(_path: &Path, _lockfile: &PackageLock) -> Result<(), MoonError> { + Ok(()) // Do nothing +} + +#[cached(result)] +pub fn load_lockfile_dependencies(path: PathBuf) -> Result { + let mut deps: LockfileDependencyVersions = HashMap::new(); + + if let Some(lockfile) = PackageLock::read(path)? { + // TODO: This isn't entirely accurate as npm does not hoist all dependencies + // to the root of the lockfile. We'd need to recursively extract everything, + // but for now, this will get us most of the way. + for (name, dep) in lockfile.dependencies.unwrap_or_default() { + if let Some(versions) = deps.get_mut(&name) { + versions.push(dep.version.clone()); + } else { + deps.insert(name, vec![dep.version.clone()]); + } + } + } + + Ok(deps) +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_fs::prelude::*; + use moon_utils::string_vec; + use pretty_assertions::assert_eq; + use serde_json::Number; + + #[test] + fn parses_lockfile() { + let temp = assert_fs::TempDir::new().unwrap(); + + temp.child("package-lock.json") + .write_str(r#" +{ + "name": "moon-examples", + "lockfileVersion": 2, + "requires": true, + "dependencies": { + "@babel/helper-function-name": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.18.9.tgz", + "integrity": "sha512-fJgWlZt7nxGksJS9a0XdSaI4XvpExnNIgRP+rVefWh5U7BL8pPuir6SJUmFKRfjWQ51OtWSzwOxhaH/EBWWc0A==", + "requires": { + "@babel/template": "^7.18.6", + "@babel/types": "^7.18.9" + } + }, + "rollup-plugin-polyfill-node": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/rollup-plugin-polyfill-node/-/rollup-plugin-polyfill-node-0.10.2.tgz", + "integrity": "sha512-5GMywXiLiuQP6ZzED/LO/Q0HyDi2W6b8VN+Zd3oB0opIjyRs494Me2ZMaqKWDNbGiW4jvvzl6L2n4zRgxS9cSQ==", + "dev": true, + "requires": { + "@rollup/plugin-inject": "^4.0.0" + } + } + } +}"#, + ) + .unwrap(); + + let lockfile: PackageLock = sync_read_json(temp.path().join("package-lock.json")).unwrap(); + + assert_eq!( + lockfile, + PackageLock { + lockfile_version: Value::Number(Number::from(2)), + name: "moon-examples".into(), + requires: Some(true), + dependencies: Some(HashMap::from([( + "@babel/helper-function-name".to_owned(), + PackageLockDependency { + integrity: Some("sha512-fJgWlZt7nxGksJS9a0XdSaI4XvpExnNIgRP+rVefWh5U7BL8pPuir6SJUmFKRfjWQ51OtWSzwOxhaH/EBWWc0A==".into()), + requires: Some(HashMap::from([ + ("@babel/template".to_owned(), "^7.18.6".to_owned()), + ("@babel/types".to_owned(), "^7.18.9".to_owned()) + ])), + resolved: Some("https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.18.9.tgz".into()), + version: "7.18.9".into(), + ..PackageLockDependency::default() + } + ), ( + "rollup-plugin-polyfill-node".to_owned(), + PackageLockDependency { + dev: Some(true), + integrity: Some("sha512-5GMywXiLiuQP6ZzED/LO/Q0HyDi2W6b8VN+Zd3oB0opIjyRs494Me2ZMaqKWDNbGiW4jvvzl6L2n4zRgxS9cSQ==".into()), + requires: Some(HashMap::from([ + ("@rollup/plugin-inject".to_owned(), "^4.0.0".to_owned()) + ])), + resolved: Some("https://registry.npmjs.org/rollup-plugin-polyfill-node/-/rollup-plugin-polyfill-node-0.10.2.tgz".into()), + version: "0.10.2".into(), + ..PackageLockDependency::default() + } + )])), + ..PackageLock::default() + } + ); + + assert_eq!( + load_lockfile_dependencies(temp.path().join("package-lock.json")).unwrap(), + HashMap::from([ + ( + "@babel/helper-function-name".to_owned(), + string_vec!["7.18.9"] + ), + ( + "rollup-plugin-polyfill-node".to_owned(), + string_vec!["0.10.2"] + ), + ]) + ); + + temp.close().unwrap(); + } +} diff --git a/crates/lang-node/src/pnpm.rs b/crates/lang-node/src/pnpm.rs new file mode 100644 index 00000000000..ce57d5a8627 --- /dev/null +++ b/crates/lang-node/src/pnpm.rs @@ -0,0 +1,253 @@ +use cached::proc_macro::cached; +use moon_error::{map_io_to_fs_error, MoonError}; +use moon_lang::config_cache; +use moon_lang::LockfileDependencyVersions; +use serde::{Deserialize, Serialize}; +use serde_yaml::Value; +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +config_cache!(PnpmLock, "pnpm-lock.yaml", load_lockfile, write_lockfile); + +type DependencyMap = HashMap; + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PnpmLockPackage { + pub cpu: Option>, + pub dependencies: Option, + pub dev: Option, + pub engines: Option>, + pub has_bin: Option, + pub optional: Option, + pub optional_dependencies: Option, + pub os: Option>, + pub peer_dependencies: Option, + pub requires_build: Option, + pub transitive_peer_dependencies: Option>, + pub resolution: Option>, + + #[serde(flatten)] + pub unknown: HashMap, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PnpmLock { + pub lockfile_version: Value, + pub importers: HashMap, + pub packages: HashMap, + + #[serde(flatten)] + pub unknown: HashMap, + + #[serde(skip)] + pub path: PathBuf, +} + +fn load_lockfile>(path: P) -> Result { + let path = path.as_ref(); + let lockfile: PnpmLock = serde_yaml::from_str( + &fs::read_to_string(path).map_err(|e| map_io_to_fs_error(e, path.to_path_buf()))?, + ) + .map_err(|e| MoonError::Yaml(path.to_path_buf(), e))?; + + Ok(lockfile) +} + +fn write_lockfile(_path: &Path, _lockfile: &PnpmLock) -> Result<(), MoonError> { + Ok(()) // Do nothing +} + +#[cached(result)] +pub fn load_lockfile_dependencies(path: PathBuf) -> Result { + let mut deps: LockfileDependencyVersions = HashMap::new(); + + if let Some(lockfile) = PnpmLock::read(path)? { + // Dependencies are defined in the following formats: + // /p-limit/2.3.0 + // /jest/28.1.3_@types+node@18.0.6 + // /@jest/core/28.1.3 + // /@babel/plugin-transform-block-scoping/7.18.9_@babel+core@7.18.9 + for dep_locator in lockfile.packages.keys() { + // Remove the leading slash + let mut locator = &dep_locator[1..]; + + // Find an underscore and return the 1st portion + if locator.contains('_') { + if let Some(under_index) = locator.find('_') { + locator = &dep_locator[1..(under_index + 1)]; + } + } + + // Find the last slash before the version + if let Some(slash_index) = locator.rfind('/') { + let name = &locator[0..slash_index]; + let version = &locator[(slash_index + 1)..]; + + if let Some(versions) = deps.get_mut(name) { + versions.push(version.to_owned()); + } else { + deps.insert(name.to_owned(), vec![version.to_owned()]); + } + } + } + } + + Ok(deps) +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_fs::prelude::*; + use moon_utils::string_vec; + use pretty_assertions::assert_eq; + use serde_yaml::{Mapping, Number}; + + #[test] + fn parses_lockfile() { + let temp = assert_fs::TempDir::new().unwrap(); + + temp.child("pnpm-lock.yaml") + .write_str( + r#" +lockfileVersion: 5.4 + +importers: + + .: {} + +packages: + + /@ampproject/remapping/2.2.0: + resolution: {integrity: sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==} + engines: {node: '>=6.0.0'} + dependencies: + '@jridgewell/gen-mapping': 0.1.1 + '@jridgewell/trace-mapping': 0.3.14 + dev: true + + /@babel/plugin-syntax-async-generators/7.8.4_@babel+core@7.18.9: + resolution: {integrity: sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7 + '@babel/helper-plugin-utils': 7.18.9 + dev: true + + /array-union/2.1.0: + resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==} + engines: {node: '>=8'} + dev: true + + /solid-jest/0.2.0_@babel+core@7.18.9: + resolution: {integrity: sha512-1ILtAj+z6bh1vTvaDlcT8501vmkzkVZMk2aiexJy+XWTZ+sb9B7IWedvWadIhOwwL97fiW4eMmN6SrbaHjn12A==} + peerDependencies: + babel-preset-solid: ^1.0.0 + dependencies: + '@babel/preset-env': 7.18.9_@babel+core@7.18.9 + babel-jest: 27.5.1_@babel+core@7.18.9 + enhanced-resolve-jest: 1.1.0 + transitivePeerDependencies: + - '@babel/core' + - supports-color + dev: true +"#, + ) + .unwrap(); + + let lockfile = load_lockfile(temp.path().join("pnpm-lock.yaml")).unwrap(); + + assert_eq!( + lockfile, + PnpmLock { + lockfile_version: Value::Number(Number::from(5.4)), + importers: HashMap::from([(".".into(), Value::Mapping(Mapping::new()))]), + packages: HashMap::from([( + "/@ampproject/remapping/2.2.0".into(), + PnpmLockPackage { + dev: Some(true), + dependencies: Some(HashMap::from([ + ("@jridgewell/gen-mapping".to_owned(), Value::String("0.1.1".to_owned())), + ("@jridgewell/trace-mapping".to_owned(), Value::String("0.3.14".to_owned())) + ])), + engines: Some(HashMap::from([ + ("node".to_owned(), ">=6.0.0".to_owned()) + ])), + resolution: Some(HashMap::from([ + ("integrity".to_owned(), "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==".to_owned()) + ])), + ..PnpmLockPackage::default() + } + ), ( + "/@babel/plugin-syntax-async-generators/7.8.4_@babel+core@7.18.9".into(), + PnpmLockPackage { + dev: Some(true), + dependencies: Some(HashMap::from([ + ("@babel/core".to_owned(), Value::Number(Number::from(7))), + ("@babel/helper-plugin-utils".to_owned(), Value::String("7.18.9".to_owned())) + ])), + peer_dependencies: Some(HashMap::from([( + "@babel/core".to_owned(), + Value::String("^7.0.0-0".to_owned()) + )])), + resolution: Some(HashMap::from([ + ("integrity".to_owned(), "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==".to_owned()) + ])), + ..PnpmLockPackage::default() + } + ), ( + "/array-union/2.1.0".into(), + PnpmLockPackage { + dev: Some(true), + engines: Some(HashMap::from([ + ("node".to_owned(), ">=8".to_owned()) + ])), + resolution: Some(HashMap::from([ + ("integrity".to_owned(), "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==".to_owned()) + ])), + ..PnpmLockPackage::default() + } + ), ( + "/solid-jest/0.2.0_@babel+core@7.18.9".into(), + PnpmLockPackage { + dev: Some(true), + dependencies: Some(HashMap::from([ + ("babel-jest".to_owned(), Value::String("27.5.1_@babel+core@7.18.9".to_owned())), + ("@babel/preset-env".to_owned(), Value::String("7.18.9_@babel+core@7.18.9".to_owned())), + ("enhanced-resolve-jest".to_owned(), Value::String("1.1.0".to_owned())) + ])), + peer_dependencies: Some(HashMap::from([( + "babel-preset-solid".to_owned(), + Value::String("^1.0.0".to_owned()) + )])), + transitive_peer_dependencies: Some(string_vec!["@babel/core", "supports-color"]), + resolution: Some(HashMap::from([ + ("integrity".to_owned(), "sha512-1ILtAj+z6bh1vTvaDlcT8501vmkzkVZMk2aiexJy+XWTZ+sb9B7IWedvWadIhOwwL97fiW4eMmN6SrbaHjn12A==".to_owned()) + ])), + ..PnpmLockPackage::default() + } + )]), + ..PnpmLock::default() + } + ); + + assert_eq!( + load_lockfile_dependencies(temp.path().join("pnpm-lock.yaml")).unwrap(), + HashMap::from([ + ("array-union".to_owned(), string_vec!["2.1.0"]), + ("solid-jest".to_owned(), string_vec!["0.2.0"]), + ( + "@babel/plugin-syntax-async-generators".to_owned(), + string_vec!["7.8.4"] + ), + ("@ampproject/remapping".to_owned(), string_vec!["2.2.0"]), + ]) + ); + + temp.close().unwrap(); + } +} diff --git a/crates/lang-node/src/yarn.rs b/crates/lang-node/src/yarn.rs new file mode 100644 index 00000000000..0e671f3539d --- /dev/null +++ b/crates/lang-node/src/yarn.rs @@ -0,0 +1,211 @@ +use cached::proc_macro::cached; +use moon_error::{map_io_to_fs_error, MoonError}; +use moon_lang::config_cache; +use moon_lang::LockfileDependencyVersions; +use serde::{Deserialize, Serialize}; +use serde_yaml::Value; +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; + +config_cache!(YarnLock, "yarn.lock", load_lockfile, write_lockfile); + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct YarnLockDependency { + pub bin: Option>, + pub checksum: Option, + pub dependencies: Option>, + pub language_name: String, + pub link_type: String, + pub peer_dependencies: Option>, + pub peer_dependencies_meta: Option, + pub resolution: String, + pub version: String, + + #[serde(flatten)] + pub unknown: HashMap, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct YarnLockMetadata { + pub cache_key: Value, + pub version: Value, + + #[serde(flatten)] + pub unknown: HashMap, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct YarnLock { + #[serde(rename = "__metadata")] + pub metadata: YarnLockMetadata, + + #[serde(flatten)] + pub dependencies: HashMap, + + #[serde(skip)] + pub path: PathBuf, +} + +fn load_lockfile>(path: P) -> Result { + let path = path.as_ref(); + let content = + fs::read_to_string(path).map_err(|e| map_io_to_fs_error(e, path.to_path_buf()))?; + + let lockfile: YarnLock = + serde_yaml::from_str(&content).map_err(|e| MoonError::Yaml(path.to_path_buf(), e))?; + + Ok(lockfile) +} + +fn write_lockfile(_path: &Path, _lockfile: &YarnLock) -> Result<(), MoonError> { + Ok(()) // Do nothing +} + +#[cached(result)] +pub fn load_lockfile_dependencies(path: PathBuf) -> Result { + let mut deps: LockfileDependencyVersions = HashMap::new(); + + if let Some(lockfile) = YarnLock::read(path)? { + for dep in lockfile.dependencies.values() { + let name = if let Some(at_index) = dep.resolution.rfind('@') { + &dep.resolution[0..at_index] + } else { + &dep.resolution + }; + + if let Some(versions) = deps.get_mut(name) { + versions.push(dep.version.to_owned()); + } else { + deps.insert(name.to_owned(), vec![dep.version.to_owned()]); + } + } + } + + Ok(deps) +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_fs::prelude::*; + use moon_utils::string_vec; + use pretty_assertions::assert_eq; + use serde_yaml::Number; + + #[test] + fn parses_lockfile() { + let temp = assert_fs::TempDir::new().unwrap(); + + temp.child("yarn.lock").write_str(r#" +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 6 + cacheKey: 8 + +"@algolia/autocomplete-core@npm:1.5.2": + version: 1.5.2 + resolution: "@algolia/autocomplete-core@npm:1.5.2" + dependencies: + "@algolia/autocomplete-shared": 1.5.2 + checksum: a8ab49c689c7fe7782980af167dfd9bea0feb9fe9809d003da509096550852f48abff27b59e0bc9909a455fb998ff4b8a7ce45b7dd42cef42f675c81340a47e9 + languageName: node + linkType: hard + +"@babel/plugin-syntax-logical-assignment-operators@npm:^7.10.4, @babel/plugin-syntax-logical-assignment-operators@npm:^7.8.3": + version: 7.10.4 + resolution: "@babel/plugin-syntax-logical-assignment-operators@npm:7.10.4" + dependencies: + "@babel/helper-plugin-utils": ^7.10.4 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: aff33577037e34e515911255cdbb1fd39efee33658aa00b8a5fd3a4b903585112d037cce1cc9e4632f0487dc554486106b79ccd5ea63a2e00df4363f6d4ff886 + languageName: node + linkType: hard + +"eslint-scope@npm:5.1.1, eslint-scope@npm:^5.1.1": + version: 5.1.1 + resolution: "eslint-scope@npm:5.1.1" + dependencies: + esrecurse: ^4.3.0 + estraverse: 4 + checksum: 47e4b6a3f0cc29c7feedee6c67b225a2da7e155802c6ea13bbef4ac6b9e10c66cd2dcb987867ef176292bf4e64eccc680a49e35e9e9c669f4a02bac17e86abdb + languageName: node + linkType: hard +"#).unwrap(); + + let lockfile = load_lockfile(temp.path().join("yarn.lock")).unwrap(); + + assert_eq!( + lockfile, + YarnLock { + metadata: YarnLockMetadata { + cache_key: Value::Number(Number::from(8)), + version: Value::Number(Number::from(6)), + ..YarnLockMetadata::default() + }, + dependencies: HashMap::from([ + ("@algolia/autocomplete-core@npm:1.5.2".to_owned(), YarnLockDependency { + checksum: Some("a8ab49c689c7fe7782980af167dfd9bea0feb9fe9809d003da509096550852f48abff27b59e0bc9909a455fb998ff4b8a7ce45b7dd42cef42f675c81340a47e9".into()), + dependencies: Some(HashMap::from([ + ("@algolia/autocomplete-shared".to_owned(), Value::String("1.5.2".to_owned())) + ])), + language_name: "node".into(), + link_type: "hard".into(), + resolution: "@algolia/autocomplete-core@npm:1.5.2".into(), + version: "1.5.2".into(), + ..YarnLockDependency::default() + }), + ("@babel/plugin-syntax-logical-assignment-operators@npm:^7.10.4, @babel/plugin-syntax-logical-assignment-operators@npm:^7.8.3".to_owned(), YarnLockDependency { + checksum: Some("aff33577037e34e515911255cdbb1fd39efee33658aa00b8a5fd3a4b903585112d037cce1cc9e4632f0487dc554486106b79ccd5ea63a2e00df4363f6d4ff886".into()), + dependencies: Some(HashMap::from([ + ("@babel/helper-plugin-utils".to_owned(), Value::String("^7.10.4".to_owned())) + ])), + language_name: "node".into(), + link_type: "hard".into(), + peer_dependencies: Some(HashMap::from([ + ("@babel/core".to_owned(), Value::String("^7.0.0-0".to_owned())) + ])), + resolution: "@babel/plugin-syntax-logical-assignment-operators@npm:7.10.4".into(), + version: "7.10.4".into(), + ..YarnLockDependency::default() + }), + ("eslint-scope@npm:5.1.1, eslint-scope@npm:^5.1.1".to_owned(), YarnLockDependency { + checksum: Some("47e4b6a3f0cc29c7feedee6c67b225a2da7e155802c6ea13bbef4ac6b9e10c66cd2dcb987867ef176292bf4e64eccc680a49e35e9e9c669f4a02bac17e86abdb".into()), + dependencies: Some(HashMap::from([ + ("estraverse".to_owned(), Value::Number(Number::from(4))), + ("esrecurse".to_owned(), Value::String("^4.3.0".to_owned())) + ])), + language_name: "node".into(), + link_type: "hard".into(), + resolution: "eslint-scope@npm:5.1.1".into(), + version: "5.1.1".into(), + ..YarnLockDependency::default() + }) + ]), + ..YarnLock::default() + } + ); + + assert_eq!( + load_lockfile_dependencies(temp.path().join("yarn.lock")).unwrap(), + HashMap::from([ + ( + "@algolia/autocomplete-core".to_owned(), + string_vec!["1.5.2"] + ), + ( + "@babel/plugin-syntax-logical-assignment-operators".to_owned(), + string_vec!["7.10.4"] + ), + ("eslint-scope".to_owned(), string_vec!["5.1.1"]) + ]) + ); + + temp.close().unwrap(); + } +} diff --git a/crates/lang-node/src/yarn_classic.rs b/crates/lang-node/src/yarn_classic.rs new file mode 100644 index 00000000000..e7fe362c705 --- /dev/null +++ b/crates/lang-node/src/yarn_classic.rs @@ -0,0 +1,193 @@ +use cached::proc_macro::cached; +use moon_error::MoonError; +use moon_lang::config_cache; +use moon_lang::LockfileDependencyVersions; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{prelude::*, BufReader}; +use std::path::{Path, PathBuf}; + +config_cache!(YarnLock, "yarn.lock", load_lockfile, write_lockfile); + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct YarnLockDependency { + pub version: String, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct YarnLock { + pub dependencies: HashMap, + + #[serde(skip)] + pub path: PathBuf, +} + +// Package names are separated by commas in the following formats: +// "@babel/core@7.12.9": +// "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.16.0", "@babel/code-frame@^7.18.6", "@babel/code-frame@^7.8.3": +fn extract_package_name(line: &str) -> Option { + // Remove trailing colon + let names = &line[0..(line.len() - 1)]; + + for name in names.split(", ") { + let unquoted_name = if name.starts_with('"') { + &name[1..(name.len() - 1)] + } else { + name + }; + + if let Some(at_index) = unquoted_name.rfind('@') { + return Some(unquoted_name[0..at_index].to_owned()); + } + } + + None +} + +fn load_lockfile>(path: P) -> Result { + let path = path.as_ref(); + let file = File::open(path)?; + let reader = BufReader::new(file); + + let mut current_package = None; + let mut lockfile = YarnLock { + dependencies: HashMap::new(), + path: PathBuf::new(), + }; + + for line in reader.lines().flatten() { + if line.is_empty() || line.starts_with('#') { + continue; + } + + // Package name is the only line fully left aligned + if !line.starts_with(' ') { + current_package = Some(line[0..(line.len() - 1)].to_owned()); + + // Extract only the version and skip other fields + } else if line.starts_with(" version") { + if let Some(names) = current_package { + let version = line[11..(line.len() - 1)].to_owned(); + + lockfile + .dependencies + .insert(names, YarnLockDependency { version }); + + current_package = None; + } + } + } + + Ok(lockfile) +} + +fn write_lockfile(_path: &Path, _lockfile: &YarnLock) -> Result<(), MoonError> { + Ok(()) // Do nothing +} + +#[cached(result)] +pub fn load_lockfile_dependencies(path: PathBuf) -> Result { + let mut deps: LockfileDependencyVersions = HashMap::new(); + + if let Some(lockfile) = YarnLock::read(path)? { + for (names, dep) in lockfile.dependencies { + if let Some(name) = extract_package_name(&names) { + if let Some(versions) = deps.get_mut(&name) { + versions.push(dep.version.clone()); + } else { + deps.insert(name, vec![dep.version.clone()]); + } + } + } + } + + Ok(deps) +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_fs::prelude::*; + use moon_utils::string_vec; + use pretty_assertions::assert_eq; + + #[test] + fn parses_lockfile() { + let temp = assert_fs::TempDir::new().unwrap(); + + temp.child("yarn.lock").write_str(r#" +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@algolia/autocomplete-core@1.7.1": + version "1.7.1" + resolved "https://registry.yarnpkg.com/@algolia/autocomplete-core/-/autocomplete-core-1.7.1.tgz#025538b8a9564a9f3dd5bcf8a236d6951c76c7d1" + integrity sha512-eiZw+fxMzNQn01S8dA/hcCpoWCOCwcIIEUtHHdzN5TGB3IpzLbuhqFeTfh2OUhhgkE8Uo17+wH+QJ/wYyQmmzg== + dependencies: + "@algolia/autocomplete-shared" "1.7.1" + +"@babel/plugin-proposal-optional-chaining@^7.13.12", "@babel/plugin-proposal-optional-chaining@^7.18.9": + version "7.18.9" + resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.18.9.tgz#e8e8fe0723f2563960e4bf5e9690933691915993" + integrity sha512-v5nwt4IqBXihxGsW2QmCWMDS3B3bzGIk/EQVZz2ei7f3NJl8NzAJVvUmpDW5q1CRNY+Beb/k58UAH1Km1N411w== + dependencies: + "@babel/helper-plugin-utils" "^7.18.9" + "@babel/helper-skip-transparent-expression-wrappers" "^7.18.9" + "@babel/plugin-syntax-optional-chaining" "^7.8.3" + +repeat-string@^1.0.0, repeat-string@^1.5.4, repeat-string@^1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" + integrity sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w== +"#).unwrap(); + + let lockfile = load_lockfile(temp.path().join("yarn.lock")).unwrap(); + + assert_eq!( + lockfile, + YarnLock { + dependencies: HashMap::from([ + ( + "\"@algolia/autocomplete-core@1.7.1\"".into(), + YarnLockDependency { + version: "1.7.1".into() + } + ), + ( + "\"@babel/plugin-proposal-optional-chaining@^7.13.12\", \"@babel/plugin-proposal-optional-chaining@^7.18.9\"" + .into(), + YarnLockDependency { + version: "7.18.9".into() + } + ), + ( + "repeat-string@^1.0.0, repeat-string@^1.5.4, repeat-string@^1.6.1".into(), + YarnLockDependency { + version: "1.6.1".into() + } + ) + ]), + ..YarnLock::default() + } + ); + + assert_eq!( + load_lockfile_dependencies(temp.path().join("yarn.lock")).unwrap(), + HashMap::from([ + ( + "@algolia/autocomplete-core".to_owned(), + string_vec!["1.7.1"] + ), + ( + "@babel/plugin-proposal-optional-chaining".to_owned(), + string_vec!["7.18.9"] + ), + ("repeat-string".to_owned(), string_vec!["1.6.1"]) + ]) + ); + + temp.close().unwrap(); + } +} diff --git a/crates/lang/src/config.rs b/crates/lang/src/config.rs index a4865f15a5e..f2c1abc093e 100644 --- a/crates/lang/src/config.rs +++ b/crates/lang/src/config.rs @@ -1,7 +1,7 @@ #[macro_export] macro_rules! config_cache { ($struct:ident, $file:expr, $reader:ident, $writer:ident) => { - fn load_json(path: &Path) -> Result<$struct, MoonError> { + fn load_config_internal(path: &Path) -> Result<$struct, MoonError> { use moon_logger::{color, trace}; trace!( @@ -19,7 +19,7 @@ macro_rules! config_cache { // This merely exists to create the global cache! #[cached(sync_writes = true, result = true)] fn load_config(path: PathBuf) -> Result<$struct, MoonError> { - load_json(&path) + load_config_internal(&path) } impl $struct { @@ -90,7 +90,7 @@ macro_rules! config_cache { if let Some(item) = cache.cache_get(&path) { cfg = item.clone(); } else { - cfg = load_json(&path)?; + cfg = load_config_internal(&path)?; } func(&mut cfg)?; diff --git a/crates/lang/src/lib.rs b/crates/lang/src/lib.rs index 24ae8cd4e32..28bc6158211 100644 --- a/crates/lang/src/lib.rs +++ b/crates/lang/src/lib.rs @@ -2,6 +2,7 @@ mod config; mod errors; pub use errors::LangError; +use std::collections::HashMap; use std::fs; use std::path::Path; @@ -39,6 +40,8 @@ pub struct VersionManager { pub version_filename: StaticString, } +pub type LockfileDependencyVersions = HashMap>; + pub fn has_vendor_installed_dependencies>(dir: T, lang: &Language) -> bool { let vendor_path = dir.as_ref().join(lang.vendor_dir); diff --git a/crates/platform-node/src/actions/run_target.rs b/crates/platform-node/src/actions/run_target.rs index b35e7058778..4eebfcd1da3 100644 --- a/crates/platform-node/src/actions/run_target.rs +++ b/crates/platform-node/src/actions/run_target.rs @@ -1,6 +1,6 @@ use crate::hasher::NodeTargetHasher; use moon_action::{ActionContext, ProfileType}; -use moon_config::NodePackageManager; +use moon_config::{HasherOptimization, NodePackageManager}; use moon_error::MoonError; use moon_lang_node::{ node::{self, BinFile}, @@ -14,6 +14,7 @@ use moon_toolchain::{get_path_env_var, Executable}; use moon_utils::process::Command; use moon_utils::{path, string_vec}; use moon_workspace::{Workspace, WorkspaceError}; +use std::collections::HashMap; const LOG_TARGET: &str = "moon:platform-node:run-target"; @@ -155,19 +156,30 @@ pub async fn create_target_command( Ok(command) } -pub fn create_target_hasher( +pub async fn create_target_hasher( workspace: &Workspace, project: &Project, ) -> Result { let node = workspace.toolchain.get_node()?; let mut hasher = NodeTargetHasher::new(node.config.version.clone()); + let resolved_dependencies = if matches!( + workspace.config.hasher.optimization, + HasherOptimization::Accuracy + ) { + node.get_package_manager() + .get_resolved_depenencies(&project.root) + .await? + } else { + HashMap::new() + }; + if let Some(root_package) = PackageJson::read(&workspace.root)? { - hasher.hash_package_json(&root_package); + hasher.hash_package_json(&root_package, &resolved_dependencies); } if let Some(package) = PackageJson::read(&project.root)? { - hasher.hash_package_json(&package); + hasher.hash_package_json(&package, &resolved_dependencies); } if let Some(typescript_config) = &workspace.config.typescript { diff --git a/crates/platform-node/src/hasher.rs b/crates/platform-node/src/hasher.rs index b6d1275d99a..72f2ed722ca 100644 --- a/crates/platform-node/src/hasher.rs +++ b/crates/platform-node/src/hasher.rs @@ -1,5 +1,7 @@ use moon_hasher::{hash_btree, Digest, Hasher, Sha256}; +use moon_lang::LockfileDependencyVersions; use moon_lang_node::{package::PackageJson, tsconfig::TsConfigJson}; +use moon_utils::semver; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -36,17 +38,40 @@ impl NodeTargetHasher { } /// Hash `package.json` dependencies as version changes should bust the cache. - pub fn hash_package_json(&mut self, package: &PackageJson) { + pub fn hash_package_json( + &mut self, + package: &PackageJson, + resolved_deps: &LockfileDependencyVersions, + ) { + let copy_deps = |deps: &BTreeMap, hashed: &mut BTreeMap| { + 'outer: for (name, version_range) in deps { + if let Some(resolved_versions) = resolved_deps.get(name) { + if let Ok(version_req) = semver::VersionReq::parse(version_range) { + for resolved_version in resolved_versions { + if semver::satisfies_requirement(resolved_version, &version_req) { + hashed.insert(name.to_owned(), resolved_version.to_owned()); + + continue 'outer; + } + } + } + } + + // No match, just use the range itself + hashed.insert(name.to_owned(), version_range.to_owned()); + } + }; + if let Some(deps) = &package.dependencies { - self.package_dependencies.extend(deps.clone()); + copy_deps(deps, &mut self.package_dependencies); } if let Some(dev_deps) = &package.dev_dependencies { - self.package_dev_dependencies.extend(dev_deps.clone()); + copy_deps(dev_deps, &mut self.package_dev_dependencies); } if let Some(peer_deps) = &package.peer_dependencies { - self.package_peer_dependencies.extend(peer_deps.clone()); + copy_deps(peer_deps, &mut self.package_peer_dependencies); } } @@ -89,6 +114,7 @@ impl Hasher for NodeTargetHasher { mod tests { use super::*; use moon_hasher::to_hash_only; + use std::collections::HashMap; #[test] fn returns_default_hash() { @@ -120,21 +146,25 @@ mod tests { #[test] fn returns_same_hash_for_same_value_inserted() { + let resolved_deps = HashMap::new(); + let mut package1 = PackageJson::default(); package1.add_dependency("react", "17.0.0", true); let mut hasher1 = NodeTargetHasher::new(String::from("0.0.0")); - hasher1.hash_package_json(&package1); + hasher1.hash_package_json(&package1, &resolved_deps); let mut hasher2 = NodeTargetHasher::new(String::from("0.0.0")); - hasher2.hash_package_json(&package1); - hasher2.hash_package_json(&package1); + hasher2.hash_package_json(&package1, &resolved_deps); + hasher2.hash_package_json(&package1, &resolved_deps); assert_eq!(to_hash_only(&hasher1), to_hash_only(&hasher2)); } #[test] fn returns_same_hash_for_diff_order_insertion() { + let resolved_deps = HashMap::new(); + let mut package1 = PackageJson::default(); package1.add_dependency("react", "17.0.0", true); @@ -142,18 +172,20 @@ mod tests { package2.add_dependency("react-dom", "17.0.0", true); let mut hasher1 = NodeTargetHasher::new(String::from("0.0.0")); - hasher1.hash_package_json(&package2); - hasher1.hash_package_json(&package1); + hasher1.hash_package_json(&package2, &resolved_deps); + hasher1.hash_package_json(&package1, &resolved_deps); let mut hasher2 = NodeTargetHasher::new(String::from("0.0.0")); - hasher2.hash_package_json(&package1); - hasher2.hash_package_json(&package2); + hasher2.hash_package_json(&package1, &resolved_deps); + hasher2.hash_package_json(&package2, &resolved_deps); assert_eq!(to_hash_only(&hasher1), to_hash_only(&hasher2)); } #[test] fn returns_diff_hash_for_overwritten_value() { + let resolved_deps = HashMap::new(); + let mut package1 = PackageJson::default(); package1.add_dependency("react", "17.0.0", true); @@ -161,11 +193,11 @@ mod tests { package2.add_dependency("react", "18.0.0", true); let mut hasher1 = NodeTargetHasher::new(String::from("0.0.0")); - hasher1.hash_package_json(&package1); + hasher1.hash_package_json(&package1, &resolved_deps); let hash1 = to_hash_only(&hasher1); - hasher1.hash_package_json(&package2); + hasher1.hash_package_json(&package2, &resolved_deps); let hash2 = to_hash_only(&hasher1); @@ -178,31 +210,53 @@ mod tests { #[test] fn supports_all_dep_types() { + let resolved_deps = HashMap::new(); + let mut package = PackageJson::default(); package.add_dependency("moment", "10.0.0", true); let mut hasher1 = NodeTargetHasher::new(String::from("0.0.0")); - hasher1.hash_package_json(&package); + hasher1.hash_package_json(&package, &resolved_deps); let hash1 = to_hash_only(&hasher1); package.dev_dependencies = Some(BTreeMap::from([("eslint".to_owned(), "8.0.0".to_owned())])); let mut hasher2 = NodeTargetHasher::new(String::from("0.0.0")); - hasher2.hash_package_json(&package); + hasher2.hash_package_json(&package, &resolved_deps); let hash2 = to_hash_only(&hasher2); package.peer_dependencies = Some(BTreeMap::from([("react".to_owned(), "18.0.0".to_owned())])); let mut hasher3 = NodeTargetHasher::new(String::from("0.0.0")); - hasher3.hash_package_json(&package); + hasher3.hash_package_json(&package, &resolved_deps); let hash3 = to_hash_only(&hasher3); assert_ne!(hash1, hash2); assert_ne!(hash1, hash3); assert_ne!(hash2, hash3); } + + #[test] + fn uses_version_from_resolved_deps() { + let resolved_deps = HashMap::from([("prettier".to_owned(), vec!["2.1.3".to_owned()])]); + + let mut package = PackageJson::default(); + package.add_dependency("prettier", "^2.0.0", true); + package.add_dependency("rollup", "^2.0.0", true); + + let mut hasher = NodeTargetHasher::new(String::from("0.0.0")); + hasher.hash_package_json(&package, &resolved_deps); + + assert_eq!( + hasher.package_dependencies, + BTreeMap::from([ + ("prettier".to_owned(), "2.1.3".to_owned()), + ("rollup".to_owned(), "^2.0.0".to_owned()) + ]) + ) + } } mod tsconfig_json { diff --git a/crates/toolchain/src/pms/npm.rs b/crates/toolchain/src/pms/npm.rs index e6a8e6ec0ea..b4907fcc605 100644 --- a/crates/toolchain/src/pms/npm.rs +++ b/crates/toolchain/src/pms/npm.rs @@ -5,12 +5,14 @@ use crate::traits::{Executable, Installable, Lifecycle, PackageManager}; use crate::Toolchain; use async_trait::async_trait; use moon_config::NpmConfig; -use moon_lang_node::{node, NPM}; +use moon_lang::LockfileDependencyVersions; +use moon_lang_node::{node, npm, NPM}; use moon_logger::{color, debug, Logable}; -use moon_utils::is_ci; use moon_utils::process::Command; +use moon_utils::{fs, is_ci}; +use std::collections::HashMap; use std::env; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; pub struct NpmTool { bin_path: PathBuf, @@ -252,6 +254,20 @@ impl PackageManager for NpmTool { String::from(NPM.manifest_filename) } + async fn get_resolved_depenencies( + &self, + project_root: &Path, + ) -> Result { + let lockfile_path = match fs::find_upwards(NPM.lock_filenames[0], project_root) { + Some(path) => path, + None => { + return Ok(HashMap::new()); + } + }; + + Ok(npm::load_lockfile_dependencies(lockfile_path)?) + } + async fn install_dependencies(&self, toolchain: &Toolchain) -> Result<(), ToolchainError> { let mut args = vec!["install"]; diff --git a/crates/toolchain/src/pms/pnpm.rs b/crates/toolchain/src/pms/pnpm.rs index 470e73f603f..00fef68bfd6 100644 --- a/crates/toolchain/src/pms/pnpm.rs +++ b/crates/toolchain/src/pms/pnpm.rs @@ -5,11 +5,13 @@ use crate::traits::{Executable, Installable, Lifecycle, PackageManager}; use crate::Toolchain; use async_trait::async_trait; use moon_config::PnpmConfig; -use moon_lang_node::{node, PNPM}; +use moon_lang::LockfileDependencyVersions; +use moon_lang_node::{node, pnpm, PNPM}; use moon_logger::{color, debug, Logable}; -use moon_utils::is_ci; +use moon_utils::{fs, is_ci}; +use std::collections::HashMap; use std::env; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; pub struct PnpmTool { bin_path: PathBuf, @@ -179,6 +181,20 @@ impl PackageManager for PnpmTool { String::from(PNPM.manifest_filename) } + async fn get_resolved_depenencies( + &self, + project_root: &Path, + ) -> Result { + let lockfile_path = match fs::find_upwards(PNPM.lock_filenames[0], project_root) { + Some(path) => path, + None => { + return Ok(HashMap::new()); + } + }; + + Ok(pnpm::load_lockfile_dependencies(lockfile_path)?) + } + async fn install_dependencies(&self, toolchain: &Toolchain) -> Result<(), ToolchainError> { let mut args = vec!["install"]; let lockfile = toolchain.workspace_root.join(self.get_lock_filename()); diff --git a/crates/toolchain/src/pms/yarn.rs b/crates/toolchain/src/pms/yarn.rs index 3409d0c6952..898e6093ae1 100644 --- a/crates/toolchain/src/pms/yarn.rs +++ b/crates/toolchain/src/pms/yarn.rs @@ -5,11 +5,13 @@ use crate::traits::{Executable, Installable, Lifecycle, PackageManager}; use crate::Toolchain; use async_trait::async_trait; use moon_config::YarnConfig; -use moon_lang_node::{node, YARN}; +use moon_lang::LockfileDependencyVersions; +use moon_lang_node::{node, yarn, yarn_classic, YARN}; use moon_logger::{color, debug, Logable}; -use moon_utils::is_ci; +use moon_utils::{fs, is_ci}; +use std::collections::HashMap; use std::env; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; pub struct YarnTool { bin_path: PathBuf, @@ -243,6 +245,24 @@ impl PackageManager for YarnTool { String::from(YARN.manifest_filename) } + async fn get_resolved_depenencies( + &self, + project_root: &Path, + ) -> Result { + let lockfile_path = match fs::find_upwards(YARN.lock_filenames[0], project_root) { + Some(path) => path, + None => { + return Ok(HashMap::new()); + } + }; + + if self.is_v1() { + return Ok(yarn_classic::load_lockfile_dependencies(lockfile_path)?); + } + + Ok(yarn::load_lockfile_dependencies(lockfile_path)?) + } + async fn install_dependencies(&self, toolchain: &Toolchain) -> Result<(), ToolchainError> { let mut args = vec!["install"]; diff --git a/crates/toolchain/src/traits.rs b/crates/toolchain/src/traits.rs index b0b03ec0878..7c703663e89 100644 --- a/crates/toolchain/src/traits.rs +++ b/crates/toolchain/src/traits.rs @@ -2,10 +2,11 @@ use crate::errors::ToolchainError; use crate::helpers::get_path_env_var; use crate::Toolchain; use async_trait::async_trait; +use moon_lang::LockfileDependencyVersions; use moon_logger::{debug, Logable}; use moon_utils::process::Command; use moon_utils::{fs, is_offline}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; #[async_trait] pub trait Downloadable: Send + Sync + Logable { @@ -246,6 +247,13 @@ pub trait PackageManager: /// Return the name of the manifest. fn get_manifest_filename(&self) -> String; + /// Return a list of dependencies resolved to their latest version from the lockfile. + /// Dependencies are based on a manifest at the provided path. + async fn get_resolved_depenencies( + &self, + project_root: &Path, + ) -> Result; + /// Install dependencies for a defined manifest. async fn install_dependencies(&self, toolchain: &Toolchain) -> Result<(), ToolchainError>; diff --git a/crates/utils/src/semver.rs b/crates/utils/src/semver.rs index cb58a877fbd..555b9e49bf6 100644 --- a/crates/utils/src/semver.rs +++ b/crates/utils/src/semver.rs @@ -6,3 +6,19 @@ pub fn extract_major_version(version: &str) -> u64 { Err(_) => 0, } } + +pub fn satisfies_range(version: &str, range: &str) -> bool { + if let Ok(req) = VersionReq::parse(range) { + return satisfies_requirement(version, &req); + } + + false +} + +pub fn satisfies_requirement(version: &str, req: &VersionReq) -> bool { + if let Ok(ver) = Version::parse(version) { + return req.matches(&ver); + } + + false +} diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 6830040ee5a..a484e0c823c 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -10,6 +10,8 @@ #### 🚀 Updates +- Added a `hasher` setting to `.moon/workspace.yml`, for controlling aspects of smart hashing. +- Updated hashing to utilize the resolved version from the lockfile when applicable. - Updated the action runner to fail when an output is defined and the output does not exist after being ran. diff --git a/website/blog/2022-09-01_v0.13.md b/website/blog/2022-09-01_v0.13.md new file mode 100644 index 00000000000..b71f3758aa5 --- /dev/null +++ b/website/blog/2022-09-01_v0.13.md @@ -0,0 +1,31 @@ +--- +title: v0.13 - Hashing and toolchain improvements +description: This is my first post on Docusaurus 2. +slug: v0.13 +authors: [milesj] +tags: [hasher, toolchain] +hide_table_of_contents: false +--- + +With this release, we've landed some improvements to our smart hashing, and paved the road for +additional languagues. + + + +## Improved hashing accuracy + +Our [smart hashing layer](../docs/concepts/cache#hashing) is pretty powerful, but was not entirely +accurate. Up until now, when hashing a Node.js project, we'd include the `dependencies`, +`devDependencies`, and `peerDependencies` versions located within the project's `package.json` as +hash inputs. This was great, because if a dependency was explicitly upgraded in the `package.json`, +it would invalidate the previous hashes as the version number changed. + +However, what if the dependency was implicitly upgraded by another project, but the `package.json` +was not modified? These kind of transitive changes were currently ignored by moon, but no longer, as +moon will now resolve all `package.json` dependencies to the resolved version found in the root +lockfile (`package-lock.json`, `yarn.lock`, etc)! + +At moon, we believe in providing consumers with the ability to configure to their needs, and as +such, have added a new [`hasher`](../docs/config/workspace#hasher) setting to +[`.moon/workspace.yml`](../docs/config/workspace). This setting will allow you to choose between the +2 hashing patterns above! diff --git a/website/blog/authors.yml b/website/blog/authors.yml new file mode 100644 index 00000000000..c2839177e2b --- /dev/null +++ b/website/blog/authors.yml @@ -0,0 +1,5 @@ +milesj: + name: Miles Johnson + title: Founder, developer + url: https://github.com/milesj + image_url: https://pbs.twimg.com/profile_images/1532262885648281601/TQbEOiNd_400x400.jpg diff --git a/website/docs/config/workspace.mdx b/website/docs/config/workspace.mdx index 07bd4077a5b..e78796771c3 100644 --- a/website/docs/config/workspace.mdx +++ b/website/docs/config/workspace.mdx @@ -11,74 +11,6 @@ import VersionLabel from '@site/src/components/Docs/VersionLabel'; The `.moon/workspace.yml` file configures available projects and their locations, the toolchain, and the workspace development environment. -## `actionRunner` - -> `ActionRunnerConfig` - -Configures aspects of the action runner. - -### `cacheLifetime` - -> `string` - -The maximum lifetime of cached artifacts before they're marked as stale and automatically removed by -the action runner. Defaults to "7 days". This field requires an integer and a timeframe unit that -can be [parsed as a duration](https://docs.rs/humantime/2.1.0/humantime/fn.parse_duration.html). - -```yaml title=".moon/workspace.yml" {2} -actionRunner: - cacheLifetime: '24 hours' -``` - -### `implicitInputs` - -> `string[]` - -Defines task [`inputs`](./project#inputs) that are implicit inherited by _all_ tasks within the -workspace. This is extremely useful for the "changes to these files should always trigger a task" -scenario. - -Like `inputs`, file paths/globs defined here are relative from the inheriting project. -[Project and workspace relative file patterns](../concepts/file-pattern#project-relative) are -supported and encouraged. - -```yaml title=".moon/workspace.yml" {2-5} -actionRunner: - implicitInputs: - - 'package.json' - - '/.moon/project.yml' - - '/.moon/workspace.yml' -``` - -> When not defined, this setting defaults to the list in the example above. When this setting _is -> defined_, that list will be overwritten, so be sure to explicitly define them if you would like to -> retain that functionality. - -### `inheritColorsForPipedTasks` - -> `boolean` - -Force colors to be inherited from the current terminal for all tasks that are ran as a child process -and their output is piped to the action runner. Defaults to `true`. -[View more about color handling in moon](../commands/overview#colors). - -```yaml title=".moon/workspace.yml" {2} -actionRunner: - inheritColorsForPipedTasks: true -``` - -### `logRunningCommand` - -> `boolean` - -When enabled, will log the task's command, resolved arguments, and working directory when a target -is ran. Defaults to `false`. - -```yaml title=".moon/workspace.yml" {2} -actionRunner: - logRunningCommand: true -``` - ## `extends` > `string` @@ -154,6 +86,8 @@ projects: > Unlike packages in the JavaScript ecosystem, a moon project _does not_ require a `package.json`, > and is not coupled to Yarn workspaces (or similar architectures). +## Languages + ## `node` > `NodeConfig` @@ -488,6 +422,96 @@ Would result in the following `references` within both `tsconfig.json`s. } ``` +## Features + +## `actionRunner` + +> `ActionRunnerConfig` + +Configures aspects of the action runner. + +### `cacheLifetime` + +> `string` + +The maximum lifetime of cached artifacts before they're marked as stale and automatically removed by +the action runner. Defaults to "7 days". This field requires an integer and a timeframe unit that +can be [parsed as a duration](https://docs.rs/humantime/2.1.0/humantime/fn.parse_duration.html). + +```yaml title=".moon/workspace.yml" {2} +actionRunner: + cacheLifetime: '24 hours' +``` + +### `implicitInputs` + +> `string[]` + +Defines task [`inputs`](./project#inputs) that are implicit inherited by _all_ tasks within the +workspace. This is extremely useful for the "changes to these files should always trigger a task" +scenario. + +Like `inputs`, file paths/globs defined here are relative from the inheriting project. +[Project and workspace relative file patterns](../concepts/file-pattern#project-relative) are +supported and encouraged. + +```yaml title=".moon/workspace.yml" {2-5} +actionRunner: + implicitInputs: + - 'package.json' + - '/.moon/project.yml' + - '/.moon/workspace.yml' +``` + +> When not defined, this setting defaults to the list in the example above. When this setting _is +> defined_, that list will be overwritten, so be sure to explicitly define them if you would like to +> retain that functionality. + +### `inheritColorsForPipedTasks` + +> `boolean` + +Force colors to be inherited from the current terminal for all tasks that are ran as a child process +and their output is piped to the action runner. Defaults to `true`. +[View more about color handling in moon](../commands/overview#colors). + +```yaml title=".moon/workspace.yml" {2} +actionRunner: + inheritColorsForPipedTasks: true +``` + +### `logRunningCommand` + +> `boolean` + +When enabled, will log the task's command, resolved arguments, and working directory when a target +is ran. Defaults to `false`. + +```yaml title=".moon/workspace.yml" {2} +actionRunner: + logRunningCommand: true +``` + +## `hasher` + +> `HasherConfig` + +Configures aspects of smart hashing layer. + +### `optimization` + +Determines the optimization level to utilize when hashing content before running targets. + +- `accuracy` (default) - When hashing dependency versions, utilize the resolved value in the + lockfile. This requires parsing the lockfile, which may reduce performance. +- `performance` - When hashing dependency versions, utilize the value defined in the manifest. This + is typically a version range or requirement. + +```yaml title=".moon/workspace.yml" {2} +hasher: + optimization: 'performance' +``` + ## `vcs` > `VcsConfig` diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 14ac86252a1..85af91aae6e 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -42,12 +42,10 @@ const config = { sidebarPath: require.resolve('./sidebars.js'), editUrl: 'https://github.com/moonrepo/moon/tree/master/website', }, - // blog: { - // showReadingTime: true, - // // Please change this to your repo. - // editUrl: - // 'https://github.com/moonrepo/moon/tree/master/website', - // }, + blog: { + showReadingTime: true, + editUrl: 'https://github.com/moonrepo/moon/tree/master/website', + }, theme: { customCss: [ require.resolve('./src/css/theme.css'), @@ -90,11 +88,11 @@ const config = { position: 'left', label: 'Docs', }, - // { - // to: '/blog', - // label: 'Blog', - // position: 'left', - // }, + { + to: '/blog', + label: 'Blog', + position: 'left', + }, // { // to: 'api', // label: 'Packages',