diff --git a/Cargo.lock b/Cargo.lock index c155cc9b881..9040ec64626 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -93,9 +93,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ "anstyle", "anstyle-parse", @@ -131,9 +131,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -632,9 +632,9 @@ dependencies = [ [[package]] name = "cargo_toml" -version = "0.16.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a1f1117a8ff2f3547295da90f473c392d8d1107c90cea1ea82b1a544a97a4a" +checksum = "e3f9629bc6c4388ea699781dc988c2b99766d7679b151c81990b4fa1208fafd3" dependencies = [ "serde", "toml 0.8.2", @@ -800,9 +800,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.4" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" +checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" dependencies = [ "clap_builder", "clap_derive", @@ -810,9 +810,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.4" +version = "4.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" +checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" dependencies = [ "anstream", "anstyle", @@ -823,11 +823,11 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.4.1" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4110a1e6af615a9e6d0a36f805d5c99099f8bab9b8042f5bc1fa220a4a89e36f" +checksum = "e3ae8ba90b9d8b007efe66e55e48fb936272f5ca00349b5b0e89877520d35ea7" dependencies = [ - "clap 4.4.4", + "clap 4.4.6", ] [[package]] @@ -1158,7 +1158,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.4", + "clap 4.4.6", "criterion-plot", "futures", "is-terminal", @@ -3020,7 +3020,7 @@ dependencies = [ name = "moon_action_context" version = "0.1.0" dependencies = [ - "clap 4.4.4", + "clap 4.4.6", "moon_common", "moon_target", "rustc-hash", @@ -3205,7 +3205,7 @@ version = "1.14.5" dependencies = [ "bytes", "ci_env", - "clap 4.4.4", + "clap 4.4.6", "clap_complete", "clap_lex 0.5.1", "console", @@ -4392,9 +4392,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.7.3" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a4d085fd991ac8d5b05a147b437791b4260b76326baf0fc60cf7c9c27ecd33" +checksum = "c022f1e7b65d6a24c0dbbd5fb344c66881bc01f3e5ae74a1c8100f2f985d98a4" dependencies = [ "memchr", "thiserror", @@ -4403,9 +4403,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.3" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bee7be22ce7918f641a33f08e3f43388c7656772244e2bbb2477f44cc9021a" +checksum = "35513f630d46400a977c4cb58f78e1bfbe01434316e60c37d27b9ad6139c66d8" dependencies = [ "pest", "pest_generator", @@ -4413,9 +4413,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.3" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1511785c5e98d79a05e8a6bc34b4ac2168a0e3e92161862030ad84daa223141" +checksum = "bc9fc1b9e7057baba189b5c626e2d6f40681ae5b6eb064dc7c7834101ec8123a" dependencies = [ "pest", "pest_meta", @@ -4426,9 +4426,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.3" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42f0394d3123e33353ca5e1e89092e533d2cc490389f2bd6131c43c634ebc5f" +checksum = "1df74e9e7ec4053ceb980e7c0c8bd3594e977fde1af91daba9c928e8e8c6708d" dependencies = [ "once_cell", "pest", @@ -4661,9 +4661,9 @@ dependencies = [ [[package]] name = "proto_core" -version = "0.19.2" +version = "0.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c68fe1824ab8c19d63392f686a9f48e03425f2e687b272b421170867de3b19b" +checksum = "95344f1074fd611a4652e1f8d16be6991059c312831f40483bbe750a84558a09" dependencies = [ "cached", "extism", @@ -4692,9 +4692,9 @@ dependencies = [ [[package]] name = "proto_pdk_api" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b86ccfd83ad829a3666f84aacbd55e60cf0044608db9b1dfe890b80f0bafe9" +checksum = "ef5c2ebb2bebee2804153fabaffdf0ed19268d56b814fec5670f0207adde625b" dependencies = [ "anyhow", "semver", @@ -4834,13 +4834,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.5" +version = "1.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" dependencies = [ "aho-corasick 1.0.1", "memchr", - "regex-automata 0.3.8", + "regex-automata 0.3.9", "regex-syntax 0.7.5", ] @@ -4855,9 +4855,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" dependencies = [ "aho-corasick 1.0.1", "memchr", @@ -4887,9 +4887,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "base64", "bytes", @@ -4917,6 +4917,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls", @@ -5481,9 +5482,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "starbase" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dc2dc5364b3a7b113afd1f3d609edc1ae4c9f742f345949669c7ae5b9ac1944" +checksum = "0ce70d3d623e4e26202bdac9fedda7f6e91a12c12f98147eaf2e624c7525c457" dependencies = [ "async-trait", "chrono", @@ -5692,6 +5693,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "system-interface" version = "0.25.9" diff --git a/Cargo.toml b/Cargo.toml index bbe61df8608..3a3819e8f38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,8 +28,8 @@ cached = "0.46.0" chrono = { version = "0.4.31", features = ["serde"] } cd_env = "0.1.2" ci_env = "0.2.1" -clap = { version = "4.4.4", features = ["derive", "env", "wrap_help"] } -clap_complete = "4.4.1" +clap = { version = "4.4.6", features = ["derive", "env", "wrap_help"] } +clap_complete = "4.4.3" console = "0.15.7" criterion = { version = "0.5.1", features = ["async_tokio"] } miette = "5.10.0" @@ -39,10 +39,10 @@ pathdiff = "0.2.1" petgraph = { version = "0.6.4", default-features = false, features = [ "serde-1", ] } -proto_core = "0.19.2" +proto_core = "0.19.3" relative-path = { version = "1.9.0", features = ["serde"] } -regex = "1.9.5" -reqwest = { version = "0.11.20", default-features = false, features = [ +regex = "1.9.6" +reqwest = { version = "0.11.22", default-features = false, features = [ "rustls-tls-native-roots", # We don't use openssl but its required for musl builds "native-tls-vendored", @@ -60,7 +60,7 @@ semver = "1.0.19" serde = { version = "1.0.188", features = ["derive", "rc"] } serde_json = "1.0.107" serde_yaml = "0.9.25" -starbase = "0.2.7" +starbase = "0.2.8" starbase_archive = { version = "0.2.3", default-features = false, features = [ "tar-gz", ] } @@ -75,7 +75,7 @@ starbase_utils = { version = "0.3.4", default-features = false, features = [ "yaml", ] } tera = { version = "1.19.1", features = ["preserve_order"] } -thiserror = "1.0.48" +thiserror = "1.0.49" tokio = { version = "1.32.0", default-features = false, features = [ "process", "rt", diff --git a/crates/core/notifier/Cargo.toml b/crates/core/notifier/Cargo.toml index 1a7cc428d19..fa46d16da0e 100644 --- a/crates/core/notifier/Cargo.toml +++ b/crates/core/notifier/Cargo.toml @@ -17,4 +17,4 @@ serde = { workspace = true } serde_json = { workspace = true } starbase_styles = { workspace = true } tokio = { workspace = true } -uuid = { version = "1.4.1", features = ["v4"] } +uuid = { workspace = true } diff --git a/crates/core/test-utils/Cargo.toml b/crates/core/test-utils/Cargo.toml index 2be0ecf114e..a793a3fa654 100644 --- a/crates/core/test-utils/Cargo.toml +++ b/crates/core/test-utils/Cargo.toml @@ -10,7 +10,7 @@ moon_config = { path = "../../../nextgen/config" } assert_cmd = "2.0.12" assert_fs = "1.0.13" clean-path = "0.2.1" -insta = "1.32.0" +insta = "1.33.0" predicates = "3.0.4" pretty_assertions = "1.4.0" rustc-hash = { workspace = true } diff --git a/crates/rust/lang/Cargo.toml b/crates/rust/lang/Cargo.toml index 2bd2c2f35d1..9ae21e0b0a1 100644 --- a/crates/rust/lang/Cargo.toml +++ b/crates/rust/lang/Cargo.toml @@ -9,7 +9,7 @@ moon_lang = { path = "../../core/lang" } moon_logger = { path = "../../core/logger" } cached = { workspace = true } cargo-lock = "9.0.0" -cargo_toml = "0.16.2" +cargo_toml = "0.16.3" miette = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true } diff --git a/nextgen/hash/Cargo.toml b/nextgen/hash/Cargo.toml index 0d8c9f8c316..c4282bafd0a 100644 --- a/nextgen/hash/Cargo.toml +++ b/nextgen/hash/Cargo.toml @@ -12,7 +12,7 @@ moon_config = { version = "0.1.0", path = "../config" } miette = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -sha2 = "0.10.7" +sha2 = "0.10.8" starbase_utils = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } diff --git a/nextgen/query/Cargo.toml b/nextgen/query/Cargo.toml index ee70bb86985..a27804767ed 100644 --- a/nextgen/query/Cargo.toml +++ b/nextgen/query/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/moonrepo/moon" moon_common = { version = "0.1.0", path = "../common" } moon_config = { version = "0.1.0", path = "../config" } miette = { workspace = true } -pest = "2.7.3" -pest_derive = "2.7.3" +pest = "2.7.4" +pest_derive = "2.7.4" starbase_utils = { workspace = true } thiserror = { workspace = true } diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 1bfbbd6bd5c..fb7fbc7d05d 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -18,24 +18,6 @@ implicitly or explicitly depend on the other project) will now automatically mark that other project as a "peer" dependency. For example, "b" becomes a peer dependency for "a". - ```yaml - tasks: - build: - deps: ['b:build'] - - # Now internally becomes: - dependsOn: - - id: 'b' - scope: 'peer' - - tasks: - build: - deps: ['b:build'] - ``` - - We're marking this as a breaking change as this could subtly introduce cycles in the project graph - that weren't present before, and for Node.js projects, this may inject `peerDependencies`. - #### 🎉 Release - Rewrote the dependency graph from the ground-up: @@ -44,7 +26,7 @@ arbitrarily. - Cleaned up dependency chains between actions, greatly reducing the number of nodes in the graph. - Renamed `RunTarget` to `RunTask`, including interactive and persistent variants. -- Updated the action graph to iterate using a topological queue, which executes actions on-demand in +- Updated the action graph to process using a topological queue, which executes actions on-demand in the thread pool when they are ready (dependencies have been met). Previously, we would sort topologically _into batches_, which worked, but resulted in many threads uselessly waiting for an action to run, which was blocked waiting for the current batch to complete. @@ -56,8 +38,10 @@ #### 🚀 Updates -- Added a `moon action-graph` command and deprecated `moon dep-graph`. +- Added a `moon action-graph` command. - Added a `--dependents` argument to `moon action-graph`. +- Added the ability to skip non-`RunTask` actions using environment variables. +- Deprecated the `moon dep-graph` command. #### 🐞 Fixes @@ -67,6 +51,7 @@ #### ⚙️ Internal - Added in-memory caching to project graph file system lookup operations. +- Updated Rust to v1.72. ## 1.14.5 diff --git a/website/blog/2023-10-09_moon-v1.15.mdx b/website/blog/2023-10-09_moon-v1.15.mdx new file mode 100644 index 00000000000..6db633dcc32 --- /dev/null +++ b/website/blog/2023-10-09_moon-v1.15.mdx @@ -0,0 +1,143 @@ +--- +slug: moon-v1.15 +title: moon v1.15 - Next-generation action graph +authors: [milesj] +tags: [action, dependency, graph, pipeline, railway] +image: ./img/moon/v1.15.png +--- + +In this release, we've taken the new step in modernizing our action pipeline, by rewriting the +dependency graph. + + + +## Hello action graph, goodbye dependency graph + +For the past few months, we've been working on a rewrite of our action pipeline, which consists of +the project graph, dependency graph, task executor, process pipeline, and more. It's a slow process, +with many different pieces that must land in sequence, but we're almost done. The next step in this +process is the [introduction of the new action graph](/docs/how-it-works/action-graph), which +replaces the previous dependency graph. + +For the most part, the graphs work in a similar fashion, but since we rewrote it from the ground up, +we were able to resolve any discrepancies and performance issues. The biggest changes between the +new and old graphs are: + +- All actions now depend on the `SyncWorkspace` action, instead of this action running arbitrarily. +- Cleaned up dependency chains between actions, greatly reducing the number of nodes in the graph. +- Renamed `RunTarget` to `RunTask`, including interactive and persistent variants. +- And lastly, we ditched our batched task approach for a ready queue. Continue reading for more + information! + +### A new performant thread pool + +In the old dependency graph, when we'd execute a task, we'd order the graph topologically and then +group actions into batches (or buckets) based on their dependency chains. Batches would then be +executed in order within the thread pool. This approach worked well, but had one major flaw: it +wasn't as performant as could be. For example, if our thread pool size was 12, and a batch only had +2 tasks in it, what were the other 10 threads doing? Absolutely nothing. They were sitting idly, +waiting for a task. + +And now with the new action graph, we take full advantage of all threads in the pool. Instead of the +batched approach above, we now use a topological task-ready queue, where a thread without work (or +is waiting for work) can poll the graph for a new task to run. A task is considered ready to run if +it either has no dependencies, or all of its dependencies (in the chain) have been ran. + +For large graphs, this should result in a significant performance improvement! + +### Automatic peer dependencies (breaking) + +Because of these graph changes, we do have a minor "breaking change". Tasks that depend (via `deps`) +on other tasks from arbitrary projects (the parent project doesn't implicitly or explicitly depend +on the other project) will now automatically mark that other project as a "peer" dependency. For +example, "b" becomes a peer dependency for "a". + +```yaml title="a/moon.yml" +tasks: + build: + deps: ['b:build'] +``` + +Now internally becomes: + +```yaml title="a/moon.yml" +dependsOn: + - id: 'b' + scope: 'peer' + +tasks: + build: + deps: ['b:build'] +``` + +We're marking this as a breaking change as this could subtly introduce cycles in the project graph +that weren't present before, and for Node.js projects, this may inject `peerDependencies`. However, +this change was necessary to ensure accurate dependency chains in the graph. + +## New `moonrepo/setup-toolchain` GitHub action + +We've begun a process to deprecate the +[moonrepo/setup-moon-action](https://github.com/moonrepo/setup-moon-action) and +[moonrepo/setup-proto](https://github.com/moonrepo/setup-proto) GitHub actions, and instead, combine +and replace them with a new [moonrepo/setup-toolchain](https://github.com/moonrepo/setup-toolchain) +action. Why a new action instead of fixing the others? + +The biggest problem was that both previous actions shared about 90% of the same code, but were +slightly different in how they installed the binaries and cached the toolchain. It was was also +confusing for consumers to understand and know which action to use (because they shouldn't be used +together). + +To remedy this, we're prototyping the new +[moonrepo/setup-toolchain](https://github.com/moonrepo/setup-toolchain) action, which has been +working quite well. It aims to solve the following: + +- Installs `proto` globally so that installed tools can also be executed globally. +- Conditionally installs `moon` globally if the repository is using moon (attempts to detect a + `.moon` directory). +- Caches the toolchain (`~/.proto`) so subsequent runs are faster. +- Hashes `.prototools` and `.moon/toolchain.yml` files to generate a unique cache key. +- Cleans the toolchain before caching to remove unused or stale tools. +- Can auto-install tools when used. + +```diff +# ... +jobs: + ci: + name: CI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 +- - uses: moonrepo/setup-moon-action@v1 ++ - uses: moonrepo/setup-toolchain@v0 +``` + +## Now supported in Railway + +If you're a big fan of [Railway](https://railway.app/) (like we are), and you're deploying a Node.js +backed application, then you'll be happy to hear that Railway now officially and natively supports +moon! We spent some time over the past month +[integrating moon support into their Nixpacks architecture](https://nixpacks.com/docs/providers/node). + +To make use of this, set the `NIXPACKS_MOON_APP_NAME` environment variable to the name of your moon +project that you want to be deployed. This will then automatically run `moon run :build` and +`moon run :start` respectively. To customize the task names, you can set the +`NIXPACKS_MOON_BUILD_TASK` and `NIXPACKS_MOON_START_TASK` environment variables. + +:::info + +This is currently only supported for Node.js projects, but will be expanded to other languages in +the future! + +::: + +## Other changes + +View the [official release](https://github.com/moonrepo/moon/releases/tag/v1.15.0) for a full list +of changes. + +- Added a `moon action-graph` command. +- Added a `--dependents` argument to `moon action-graph`. +- Added the ability to skip non-`RunTask` actions using environment variables. +- Deprecated the `moon dep-graph` command. diff --git a/website/blog/img/moon/v1.15.png b/website/blog/img/moon/v1.15.png new file mode 100644 index 00000000000..22cf321649f Binary files /dev/null and b/website/blog/img/moon/v1.15.png differ diff --git a/website/docs/commands/action-graph.mdx b/website/docs/commands/action-graph.mdx index bc4e517d171..2b8c6710fa1 100644 --- a/website/docs/commands/action-graph.mdx +++ b/website/docs/commands/action-graph.mdx @@ -2,6 +2,10 @@ title: action-graph --- +import VersionLabel from '@site/src/components/Docs/VersionLabel'; + + + The `moon action-graph [target]` (or `moon ag`) command will generate and serve a visual graph of all actions and tasks within the workspace, known as the [action graph](../how-it-works/action-graph). In other tools, this is sometimes referred to as a diff --git a/website/docs/how-it-works/action-graph.mdx b/website/docs/how-it-works/action-graph.mdx index 3c9cb38d2ea..8c3af6798b6 100644 --- a/website/docs/how-it-works/action-graph.mdx +++ b/website/docs/how-it-works/action-graph.mdx @@ -95,5 +95,5 @@ that never exits. All persistent tasks are run in parallel as the last batch in ## What is the graph used for? -Without the action graph, tasks would not efficiently, or possibly at all! The graph helps to run -tasks in parallel, in the correct order, and to ensure a reliable outcome. +Without the action graph, tasks would not efficiently run, or possibly at all! The graph helps to +run tasks in parallel, in the correct order, and to ensure a reliable outcome. diff --git a/website/docs/install.mdx b/website/docs/install.mdx index 08b1dfb671c..ead8b814672 100644 --- a/website/docs/install.mdx +++ b/website/docs/install.mdx @@ -50,12 +50,8 @@ current session. To persist across sessions, update `PATH` manually. moon can be installed and managed in [proto's toolchain](/proto) using a TOML plugin. This will install moon to `~/.proto/tools/moon` and make the binary available at `~/.proto/bin`. -```toml title=".prototools" -[plugins] -moon = "source:https://raw.githubusercontent.com/moonrepo/moon/master/proto-plugin.toml" -``` - ```shell +proto add-plugin moon "source:https://raw.githubusercontent.com/moonrepo/moon/master/proto-plugin.toml" proto install moon ```