From d96aa9514b765223723b2c202b217ec73effb74d Mon Sep 17 00:00:00 2001 From: Miles Johnson Date: Sun, 24 Nov 2024 12:27:21 -0800 Subject: [PATCH] internal: Polish and fixes. (#1731) * Fix json output. * Update deps. * Fix tests. * Polish python. * Fix submodule dir issues. * Fix tests. * Support submodules. * Better submodules support. --- .github/workflows/rust.yml | 2 +- .moon/workspace.yml | 23 +- CHANGELOG.md | 17 +- Cargo.lock | 95 ++-- Cargo.toml | 9 +- crates/affected/src/affected_tracker.rs | 3 +- crates/api/Cargo.toml | 2 +- crates/app/Cargo.toml | 4 +- crates/app/src/commands/graph/action.rs | 2 +- crates/app/src/queries/projects.rs | 9 +- ...raph_test__action_graph__outputs_json.snap | 57 ++- crates/codegen/Cargo.toml | 2 +- crates/common/src/consts.rs | 2 +- crates/plugin/Cargo.toml | 2 +- crates/process/src/command_inspector.rs | 9 +- crates/remote/Cargo.toml | 4 +- crates/task-builder/src/tasks_builder.rs | 10 +- .../task-builder/tests/tasks_builder_test.rs | 74 +-- crates/task-expander/src/task_expander.rs | 2 +- crates/task-expander/tests/utils.rs | 2 +- crates/task/src/task.rs | 10 +- crates/vcs/Cargo.toml | 2 + crates/vcs/src/git.rs | 480 +++++++++++------- crates/vcs/src/git_submodule.rs | 63 +++ crates/vcs/src/lib.rs | 1 + crates/vcs/src/process_cache.rs | 15 + crates/vcs/src/touched_files.rs | 9 + crates/workspace/src/workspace_builder.rs | 10 - legacy/core/utils/Cargo.toml | 2 +- legacy/python/lang/src/pip_requirements.rs | 6 +- .../platform/src/actions/install_deps.rs | 12 +- legacy/python/platform/src/python_platform.rs | 1 - legacy/python/tool/src/python_tool.rs | 26 +- packages/types/src/project.ts | 5 +- 34 files changed, 617 insertions(+), 355 deletions(-) create mode 100644 crates/vcs/src/git_submodule.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 326753da80f..9b9bc839a71 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -90,7 +90,7 @@ jobs: with: auto-install: true cache: ${{ runner.os == 'Linux' }} - proto-version: '0.42.1' # Keep in sync + proto-version: '0.42.2' # Keep in sync - uses: mozilla-actions/sccache-action@v0.0.5 if: ${{ vars.ENABLE_SCCACHE == 'true' }} - name: Checking coverage status diff --git a/.moon/workspace.yml b/.moon/workspace.yml index a1da6f0790b..c49cd7311e1 100644 --- a/.moon/workspace.yml +++ b/.moon/workspace.yml @@ -35,15 +35,14 @@ docker: include: - '*.config.js' - '*.json' - -unstable_remote: - host: 'grpc://localhost:9092' - # mtls: - # caCert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem' - # clientCert: 'crates/remote/tests/__fixtures__/certs-local/client.pem' - # clientKey: 'crates/remote/tests/__fixtures__/certs-local/client.key' - # domain: 'localhost' - # tls: - # # assumeHttp2: true - # cert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem' - # # domain: 'localhost' +# unstable_remote: +# host: 'grpc://localhost:9092' +# mtls: +# caCert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem' +# clientCert: 'crates/remote/tests/__fixtures__/certs-local/client.pem' +# clientKey: 'crates/remote/tests/__fixtures__/certs-local/client.key' +# domain: 'localhost' +# tls: +# # assumeHttp2: true +# cert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem' +# # domain: 'localhost' diff --git a/CHANGELOG.md b/CHANGELOG.md index 57742019272..c2cbad5bcf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,20 +9,32 @@ - Refactored the `moon query tasks` command. - CLI options have been replaced with new task based options, instead of being project based. - Now utilizes the new task graph and affected tracker. +- The `moon project-graph --json` output no longer includes task data (the `tasks` field is an empty + map). + - Use `moon task-graph` to access this data, and piece things together using the project + `taskTargets` field. + - The `moon project --json` output pieces everything together automatically. #### 🚀 Updates -- Added unstable support for self-hosted remote caches, powered by the +- Added _unstable_ support for self-hosted remote caches, powered by the [Bazel Remote Execution API](https://github.com/bazelbuild/remote-apis). - Allows for 3rd-party implementations like [`bazel-remote`](https://github.com/buchgr/bazel-remote) to be used. - Currently supports the gRPC protocol, and will support HTTP in a later release. - Our moonbase product will be sunset in the future. +- Added Python tier 2 and 3 support. + - Will download and install Python into the toolchain when a `version` is configured. + - Will parse the `requirements.txt` to resolve and install dependencies. + - Added a `python` setting to `.moon/toolchain.yml`. + - Added a `toolchain.python` setting to `moon.yml`. + - Updated `moon bin` commands to support Python. - Added a new task graph, that enables new granular based functionality for task related features. - Added a new `moon task-graph` command. - Can now control the depth of upstream (dependencies) and downstream (dependents). - Affected information now tracks based on dependent graph connections. - Added `--upstream` and `--downstream` options to `moon query tasks`. +- Added basic support for Git submodules, and will now extract touched files from all submodules. - Added 7 new token variables: `$arch`, `$os`, `$osFamily`, `$vcsBranch`, `$vcsRepository`, `$vcsRevision`, `$workingDir` - Added a `rust.binstallVersion` setting to `.moon/toolchain.yml`. @@ -35,11 +47,12 @@ - Fixed `moon project-graph ` not including all dependencies/dependents. It was only showing direct relationships. +- Fixed an issue where touched file paths would include Git submodule directories and trigger hasher warnings. #### ⚙️ Internal - Updated dependencies. -- Updated proto to v0.42.1 (from 0.42.0). +- Updated proto to v0.42.2 (from 0.42.0). ## 1.29.4 diff --git a/Cargo.lock b/Cargo.lock index c69d8d65094..919897f3374 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1022,7 +1022,7 @@ dependencies = [ "encode_unicode", "lazy_static", "libc", - "unicode-width", + "unicode-width 0.1.14", "windows-sys 0.52.0", ] @@ -1792,9 +1792,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -1807,9 +1807,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -1817,15 +1817,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1834,9 +1834,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -1853,9 +1853,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -1864,21 +1864,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -2566,15 +2566,15 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.8" +version = "0.17.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281" dependencies = [ "console", - "instant", "number_prefix", "portable-atomic", - "unicode-width", + "unicode-width 0.2.0", + "web-time", ] [[package]] @@ -2591,7 +2591,7 @@ dependencies = [ "newline-converter", "once_cell", "unicode-segmentation", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -2606,15 +2606,6 @@ dependencies = [ "similar", ] -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - [[package]] name = "io-extras" version = "0.18.2" @@ -2993,7 +2984,7 @@ dependencies = [ "terminal_size 0.3.0", "textwrap", "thiserror", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -4521,6 +4512,7 @@ name = "moon_vcs" version = "0.0.1" dependencies = [ "async-trait", + "futures", "git-url-parse", "ignore", "miette", @@ -4533,6 +4525,7 @@ dependencies = [ "semver", "serde", "starbase_sandbox", + "starbase_utils", "thiserror", "tokio", "tracing", @@ -4740,9 +4733,9 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "open" -version = "5.3.0" +version = "5.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a877bf6abd716642a53ef1b89fb498923a4afca5c754f9050b4d081c05c4b3" +checksum = "3ecd52f0b8d15c40ce4820aa251ed5de032e5d91fab27f7db2f40d42a8bdf69c" dependencies = [ "is-wsl", "libc", @@ -5239,18 +5232,18 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60caa6738c7369b940c3d49246a8d1749323674c65cb13010134f5c9bad5b519" +checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" dependencies = [ "prost", ] [[package]] name = "proto_core" -version = "0.43.5" +version = "0.43.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "febd80c3f5b8e51a8ba76f383a3ab40554f7984413d348c5a28d1d9496f1deed" +checksum = "2f2174cde8813bde1e0141e89118cd504fa63217c9943e7cacde9618877a250f" dependencies = [ "convert_case", "indexmap 2.6.0", @@ -5297,9 +5290,9 @@ dependencies = [ [[package]] name = "proto_pdk_api" -version = "0.24.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf5c5a1c6364a67055cc5f4a140ae6bd0f261119056317a970d82c00fc983ed2" +checksum = "b3a7751f23621badfad473b22bd1c28b7122afc20476354f0ac43581d3641a10" dependencies = [ "rustc-hash 2.0.0", "schematic", @@ -5775,9 +5768,9 @@ dependencies = [ [[package]] name = "scc" -version = "2.2.4" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d25269dd3a12467afe2e510f69fb0b46b698e5afb296b59f2145259deaf8e8" +checksum = "66b202022bb57c049555430e11fc22fea12909276a80a4c3d368da36ac1d88ed" dependencies = [ "sdd", "serde", @@ -5923,9 +5916,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "indexmap 2.6.0", "itoa", @@ -6598,7 +6591,7 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" dependencies = [ "smawk", "unicode-linebreak", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -7059,6 +7052,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -7681,7 +7680,7 @@ dependencies = [ "bumpalo", "leb128", "memchr", - "unicode-width", + "unicode-width 0.1.14", "wasm-encoder 0.218.0", ] diff --git a/Cargo.toml b/Cargo.toml index 8c3a25a1285..3e661f2f616 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,8 +32,11 @@ compact_str = { version = "0.8.0", default-features = false, features = [ "serde", ] } console = "0.15.8" +convert_case = "0.6.0" dirs = "5.0.1" +futures = "0.3.31" indexmap = "2.6.0" +md5 = "0.7.0" miette = "7.2.0" once_cell = "1.20.1" pathdiff = "0.2.2" @@ -51,14 +54,14 @@ reqwest = { version = "0.12.9", default-features = false, features = [ "native-tls-vendored", ] } rustc-hash = "2.0.0" -scc = "2.2.4" +scc = "2.2.5" schematic = { version = "0.17.6", default-features = false, features = [ "schema", ] } serial_test = "3.2.0" semver = "1.0.23" serde = { version = "1.0.215", features = ["derive"] } -serde_json = "1.0.128" +serde_json = "1.0.133" serde_yaml = "0.9.34" sha2 = "0.10.8" starbase = { version = "0.9.4" } @@ -92,7 +95,7 @@ uuid = { version = "1.11.0", features = ["v4"] } # proto/plugin related extism = "=1.8.0" extism-pdk = "1.3.0" -proto_core = "0.43.5" +proto_core = "0.43.6" proto_installer = "0.7.1" system_env = "0.6.1" version_spec = "0.7.0" diff --git a/crates/affected/src/affected_tracker.rs b/crates/affected/src/affected_tracker.rs index b5d1ace41a8..b74e06abf43 100644 --- a/crates/affected/src/affected_tracker.rs +++ b/crates/affected/src/affected_tracker.rs @@ -290,7 +290,8 @@ impl<'app> AffectedTracker<'app> { return Ok(Some(AffectedBy::AlreadyMarked)); } - if task.metadata.empty_inputs { + // inputs: [] + if task.state.empty_inputs { return Ok(Some(AffectedBy::AlwaysAffected)); } diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 5b2c54a482a..6d90f1503fb 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -12,7 +12,7 @@ moon_env = { path = "../env" } moon_time = { path = "../time" } cd_env = { workspace = true } ci_env = { workspace = true } -md5 = "0.7.0" +md5 = { workspace = true } miette = { workspace = true } proto_core = { workspace = true } reqwest = { workspace = true, features = ["json", "multipart", "stream"] } diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index 8ecd6a2f28e..44f4bada659 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -71,7 +71,7 @@ tokio = { workspace = true } tracing = { workspace = true } # Visualizer -open = "5.3.0" +open = "5.3.1" petgraph = { workspace = true } tera = { workspace = true } tiny_http = "0.12.0" @@ -79,7 +79,7 @@ tiny_http = "0.12.0" # TODO remove console = { workspace = true } dialoguer = { version = "0.11.0", default-features = false } -indicatif = "0.17.8" +indicatif = "0.17.9" moon_lang = { path = "../../legacy/core/lang" } moon_tool = { path = "../../legacy/core/tool" } moon_platform = { path = "../../legacy/core/platform" } diff --git a/crates/app/src/commands/graph/action.rs b/crates/app/src/commands/graph/action.rs index 93f10fe3bc7..a8841f1b973 100644 --- a/crates/app/src/commands/graph/action.rs +++ b/crates/app/src/commands/graph/action.rs @@ -59,7 +59,7 @@ pub async fn action_graph(session: CliSession, args: ActionGraphArgs) -> AppResu let graph_info = action_graph_repr(&action_graph).await; if args.json { - println!("{}", json::format(&graph_info, false)?); + println!("{}", json::format(&graph_info, true)?); return Ok(None); } diff --git a/crates/app/src/queries/projects.rs b/crates/app/src/queries/projects.rs index 858e369b652..0c68d1159bf 100644 --- a/crates/app/src/queries/projects.rs +++ b/crates/app/src/queries/projects.rs @@ -1,7 +1,7 @@ use super::convert_to_regex; use moon_affected::Affected; use moon_project::Project; -use moon_workspace_graph::WorkspaceGraph; +use moon_workspace_graph::{GraphConnections, WorkspaceGraph}; use serde::{Deserialize, Serialize}; use std::sync::Arc; use tracing::debug; @@ -51,7 +51,10 @@ fn load_with_regex( let type_regex = convert_to_regex("type", &options.type_of)?; let mut filtered = vec![]; - for project in workspace_graph.get_all_projects()? { + for project_id in workspace_graph.projects.get_node_keys() { + // Include tasks for JSON output + let project = workspace_graph.get_project_with_tasks(project_id)?; + if let Some(regex) = &id_regex { if !regex.is_match(&project.id) { continue; @@ -109,7 +112,7 @@ fn load_with_regex( } } - filtered.push(project); + filtered.push(Arc::new(project)); } Ok(filtered) diff --git a/crates/cli/tests/snapshots/action_graph_test__action_graph__outputs_json.snap b/crates/cli/tests/snapshots/action_graph_test__action_graph__outputs_json.snap index e3e873f46c0..4468167fe0e 100644 --- a/crates/cli/tests/snapshots/action_graph_test__action_graph__outputs_json.snap +++ b/crates/cli/tests/snapshots/action_graph_test__action_graph__outputs_json.snap @@ -2,4 +2,59 @@ source: crates/cli/tests/action_graph_test.rs expression: assert.output() --- -{"nodes":[{"id":0,"label":"SyncWorkspace"},{"id":1,"label":"SetupToolchain(node:18.0.0)"},{"id":3,"label":"SyncProject(node, basic)"},{"id":4,"label":"RunTask(basic:lint)"},{"id":2,"label":"InstallWorkspaceDeps(node:18.0.0)"}],"edges":[{"id":"1 -> 0","label":"","source":1,"target":0},{"id":"2 -> 1","label":"","source":2,"target":1},{"id":"3 -> 1","label":"","source":3,"target":1},{"id":"4 -> 2","label":"","source":4,"target":2},{"id":"4 -> 3","label":"","source":4,"target":3}]} +{ + "nodes": [ + { + "id": 0, + "label": "SyncWorkspace" + }, + { + "id": 1, + "label": "SetupToolchain(node:18.0.0)" + }, + { + "id": 3, + "label": "SyncProject(node, basic)" + }, + { + "id": 4, + "label": "RunTask(basic:lint)" + }, + { + "id": 2, + "label": "InstallWorkspaceDeps(node:18.0.0)" + } + ], + "edges": [ + { + "id": "1 -> 0", + "label": "", + "source": 1, + "target": 0 + }, + { + "id": "2 -> 1", + "label": "", + "source": 2, + "target": 1 + }, + { + "id": "3 -> 1", + "label": "", + "source": 3, + "target": 1 + }, + { + "id": "4 -> 2", + "label": "", + "source": 4, + "target": 2 + }, + { + "id": "4 -> 3", + "label": "", + "source": 4, + "target": 3 + } + ] +} diff --git a/crates/codegen/Cargo.toml b/crates/codegen/Cargo.toml index 4ad61065430..bed6c3d146a 100644 --- a/crates/codegen/Cargo.toml +++ b/crates/codegen/Cargo.toml @@ -15,7 +15,7 @@ moon_env = { path = "../env" } moon_process = { path = "../process" } moon_time = { path = "../time" } content_inspector = "0.2.4" -convert_case = "0.6.0" +convert_case = { workspace = true } miette = { workspace = true } once_cell = { workspace = true } regex = { workspace = true } diff --git a/crates/common/src/consts.rs b/crates/common/src/consts.rs index ffdd87d6402..bd753043425 100644 --- a/crates/common/src/consts.rs +++ b/crates/common/src/consts.rs @@ -6,4 +6,4 @@ pub const BIN_NAME: &str = "moon"; pub const CONFIG_DIRNAME: &str = ".moon"; -pub const PROTO_CLI_VERSION: &str = "0.42.1"; +pub const PROTO_CLI_VERSION: &str = "0.42.2"; diff --git a/crates/plugin/Cargo.toml b/crates/plugin/Cargo.toml index 046b6b58f5f..28556321421 100644 --- a/crates/plugin/Cargo.toml +++ b/crates/plugin/Cargo.toml @@ -15,7 +15,7 @@ moon_pdk_api = { version = "0.0.10", path = "../pdk-api" } moon_target = { path = "../target" } moon_workspace_graph = { path = "../workspace-graph" } async-trait = { workspace = true } -convert_case = "0.6.0" +convert_case = { workspace = true } extism = { workspace = true } miette = { workspace = true } proto_core = { workspace = true } diff --git a/crates/process/src/command_inspector.rs b/crates/process/src/command_inspector.rs index 6514216e023..2f8b8c4ffcc 100644 --- a/crates/process/src/command_inspector.rs +++ b/crates/process/src/command_inspector.rs @@ -140,9 +140,14 @@ impl<'cmd> CommandInspector<'cmd> { let line = self.get_command_line(); format!( - "{}{}", + "{}::{}::{}", line.command.join(OsStr::new(" ")).to_string_lossy(), - line.input.to_string_lossy() + line.input.to_string_lossy(), + self.command + .cwd + .as_ref() + .and_then(|cwd| cwd.as_os_str().to_str()) + .unwrap_or_default() ) } diff --git a/crates/remote/Cargo.toml b/crates/remote/Cargo.toml index 0e1e5da0120..2a352251789 100644 --- a/crates/remote/Cargo.toml +++ b/crates/remote/Cargo.toml @@ -13,7 +13,7 @@ bazel-remote-apis = "0.10.0" chrono = { workspace = true } filetime = "0.2.25" miette = { workspace = true } -prost-types = "0.13.2" +prost-types = "0.13.3" reqwest = { workspace = true } rustc-hash = { workspace = true } scc = { workspace = true } @@ -21,7 +21,7 @@ sha2 = { workspace = true } starbase_utils = { workspace = true, features = ["glob"] } thiserror = { workspace = true } tokio = { workspace = true } -tonic = { version = "0.12.2", default-features = false, features = [ +tonic = { version = "0.12.3", default-features = false, features = [ "channel", "gzip", "tls", diff --git a/crates/task-builder/src/tasks_builder.rs b/crates/task-builder/src/tasks_builder.rs index 15f32b98bc4..da747a85e43 100644 --- a/crates/task-builder/src/tasks_builder.rs +++ b/crates/task-builder/src/tasks_builder.rs @@ -297,8 +297,8 @@ impl<'proj> TasksBuilder<'proj> { task.preset = preset; task.options = self.build_task_options(id, preset)?; - task.metadata.local_only = is_local; - task.metadata.root_level = is_root_level_source(self.project_source); + task.state.local_only = is_local; + task.state.root_level = is_root_level_source(self.project_source); // Aggregate all values that are inherited from the global task configs, // and should always be included in the task, regardless of merge strategy. @@ -402,14 +402,14 @@ impl<'proj> TasksBuilder<'proj> { "Task has explicitly disabled inputs", ); - task.metadata.empty_inputs = true; - } else if self.context.monorepo && task.metadata.root_level { + task.state.empty_inputs = true; + } else if self.context.monorepo && task.state.root_level { trace!( task_target = target.as_str(), "Task is a root-level project in a monorepo, defaulting to no inputs", ); - task.metadata.empty_inputs = true; + task.state.empty_inputs = true; } else { trace!( task_target = target.as_str(), diff --git a/crates/task-builder/tests/tasks_builder_test.rs b/crates/task-builder/tests/tasks_builder_test.rs index ae34ae3d382..952ab6b02f7 100644 --- a/crates/task-builder/tests/tasks_builder_test.rs +++ b/crates/task-builder/tests/tasks_builder_test.rs @@ -129,7 +129,7 @@ mod tasks_builder { ] ); assert_eq!(build.outputs, vec![OutputPath::ProjectFile("out".into())]); - assert!(!build.metadata.local_only); + assert!(!build.state.local_only); let run = tasks.get("local-run").unwrap(); @@ -142,7 +142,7 @@ mod tasks_builder { ] ); assert_eq!(run.outputs, vec![]); - assert!(run.metadata.local_only); + assert!(run.state.local_only); let test = tasks.get("local-test").unwrap(); @@ -154,7 +154,7 @@ mod tasks_builder { InputPath::WorkspaceGlob(".moon/*.yml".into()), ] ); - assert!(!test.metadata.local_only); + assert!(!test.state.local_only); } #[tokio::test] @@ -173,7 +173,7 @@ mod tasks_builder { ] ); assert_eq!(build.outputs, vec![OutputPath::ProjectFile("out".into())]); - assert!(!build.metadata.local_only); + assert!(!build.state.local_only); let run = tasks.get("local-run").unwrap(); @@ -186,7 +186,7 @@ mod tasks_builder { ] ); assert_eq!(run.outputs, vec![]); - assert!(run.metadata.local_only); + assert!(run.state.local_only); } #[tokio::test] @@ -702,7 +702,7 @@ mod tasks_builder { use super::*; fn is_local(task: &Task) { - assert!(task.metadata.local_only); + assert!(task.state.local_only); assert!(!task.options.cache); assert_eq!(task.options.output_style, Some(TaskOutputStyle::Stream)); assert!(task.options.persistent); @@ -726,22 +726,22 @@ mod tasks_builder { let cache = tasks.get("override-cache").unwrap(); - assert!(cache.metadata.local_only); + assert!(cache.state.local_only); assert!(cache.options.cache); let style = tasks.get("override-style").unwrap(); - assert!(style.metadata.local_only); + assert!(style.state.local_only); assert_eq!(style.options.output_style, Some(TaskOutputStyle::Hash)); let persistent = tasks.get("override-persistent").unwrap(); - assert!(persistent.metadata.local_only); + assert!(persistent.state.local_only); assert!(!persistent.options.persistent); let ci = tasks.get("override-ci").unwrap(); - assert!(ci.metadata.local_only); + assert!(ci.state.local_only); assert!(ci.options.run_in_ci); } @@ -752,11 +752,11 @@ mod tasks_builder { let build = tasks.get("global-build").unwrap(); - assert!(build.metadata.local_only); + assert!(build.state.local_only); let run = tasks.get("global-run").unwrap(); - assert!(!run.metadata.local_only); + assert!(!run.state.local_only); } } @@ -831,7 +831,7 @@ mod tasks_builder { InputPath::WorkspaceGlob(".moon/*.yml".into()), ] ); - assert!(!task.metadata.empty_inputs); + assert!(!task.state.empty_inputs); let task = tasks.get("empty-inputs").unwrap(); @@ -839,7 +839,7 @@ mod tasks_builder { task.inputs, vec![InputPath::WorkspaceGlob(".moon/*.yml".into())] ); - assert!(task.metadata.empty_inputs); + assert!(task.state.empty_inputs); let task = tasks.get("with-inputs").unwrap(); @@ -850,7 +850,7 @@ mod tasks_builder { InputPath::WorkspaceGlob(".moon/*.yml".into()), ] ); - assert!(!task.metadata.empty_inputs); + assert!(!task.state.empty_inputs); } #[tokio::test] @@ -864,8 +864,8 @@ mod tasks_builder { task.inputs, vec![InputPath::WorkspaceGlob(".moon/*.yml".into())] ); - assert!(task.metadata.empty_inputs); - assert!(task.metadata.root_level); + assert!(task.state.empty_inputs); + assert!(task.state.root_level); let task = tasks.get("empty-inputs").unwrap(); @@ -873,8 +873,8 @@ mod tasks_builder { task.inputs, vec![InputPath::WorkspaceGlob(".moon/*.yml".into())] ); - assert!(task.metadata.empty_inputs); - assert!(task.metadata.root_level); + assert!(task.state.empty_inputs); + assert!(task.state.root_level); let task = tasks.get("with-inputs").unwrap(); @@ -885,8 +885,8 @@ mod tasks_builder { InputPath::WorkspaceGlob(".moon/*.yml".into()) ] ); - assert!(!task.metadata.empty_inputs); - assert!(task.metadata.root_level); + assert!(!task.state.empty_inputs); + assert!(task.state.root_level); } #[tokio::test] @@ -903,8 +903,8 @@ mod tasks_builder { InputPath::WorkspaceGlob(".moon/*.yml".into()) ] ); - assert!(!task.metadata.empty_inputs); - assert!(task.metadata.root_level); + assert!(!task.state.empty_inputs); + assert!(task.state.root_level); let task = tasks.get("empty-inputs").unwrap(); @@ -912,8 +912,8 @@ mod tasks_builder { task.inputs, vec![InputPath::WorkspaceGlob(".moon/*.yml".into())] ); - assert!(task.metadata.empty_inputs); - assert!(task.metadata.root_level); + assert!(task.state.empty_inputs); + assert!(task.state.root_level); let task = tasks.get("with-inputs").unwrap(); @@ -924,8 +924,8 @@ mod tasks_builder { InputPath::WorkspaceGlob(".moon/*.yml".into()) ] ); - assert!(!task.metadata.empty_inputs); - assert!(task.metadata.root_level); + assert!(!task.state.empty_inputs); + assert!(task.state.root_level); } #[tokio::test] @@ -943,7 +943,7 @@ mod tasks_builder { InputPath::WorkspaceGlob(".moon/*.yml".into()), ] ); - assert!(!task.metadata.empty_inputs); + assert!(!task.state.empty_inputs); let task = tasks.get("global-test").unwrap(); @@ -954,7 +954,7 @@ mod tasks_builder { InputPath::WorkspaceGlob(".moon/*.yml".into()), ] ); - assert!(!task.metadata.empty_inputs); + assert!(!task.state.empty_inputs); let task = tasks.get("global-run").unwrap(); @@ -962,7 +962,7 @@ mod tasks_builder { task.inputs, vec![InputPath::WorkspaceGlob(".moon/*.yml".into())] ); - assert!(task.metadata.empty_inputs); + assert!(task.state.empty_inputs); } } @@ -1290,7 +1290,7 @@ mod tasks_builder { // inherited assert_eq!(task.inputs.len(), 2); - assert!(task.metadata.empty_inputs); + assert!(task.state.empty_inputs); let task = tasks.get("outputs").unwrap(); @@ -1318,7 +1318,7 @@ mod tasks_builder { // inherited assert_eq!(task.inputs.len(), 2); - assert!(task.metadata.empty_inputs); + assert!(task.state.empty_inputs); let task = tasks.get("all").unwrap(); @@ -1364,7 +1364,7 @@ mod tasks_builder { InputPath::WorkspaceFile("global/tasks/tag-merge.yml".into()), ] ); - assert!(!task.metadata.empty_inputs); + assert!(!task.state.empty_inputs); let task = tasks.get("outputs").unwrap(); @@ -1410,7 +1410,7 @@ mod tasks_builder { InputPath::WorkspaceFile("global/tasks/tag-merge.yml".into()), ] ); - assert!(!task.metadata.empty_inputs); + assert!(!task.state.empty_inputs); let task = tasks.get("all").unwrap(); @@ -1762,7 +1762,7 @@ mod tasks_builder { InputPath::WorkspaceGlob(".moon/*.yml".into()), ] ); - assert!(!task.metadata.empty_inputs); + assert!(!task.state.empty_inputs); } #[tokio::test] @@ -1779,7 +1779,7 @@ mod tasks_builder { InputPath::WorkspaceGlob(".moon/*.yml".into()), ] ); - assert!(task.metadata.empty_inputs); + assert!(task.state.empty_inputs); } #[tokio::test] @@ -1797,7 +1797,7 @@ mod tasks_builder { InputPath::WorkspaceGlob(".moon/*.yml".into()), ] ); - assert!(!task.metadata.empty_inputs); + assert!(!task.state.empty_inputs); } #[tokio::test] diff --git a/crates/task-expander/src/task_expander.rs b/crates/task-expander/src/task_expander.rs index 20352fe43f6..1f0727ed76b 100644 --- a/crates/task-expander/src/task_expander.rs +++ b/crates/task-expander/src/task_expander.rs @@ -49,7 +49,7 @@ impl<'graph> TaskExpander<'graph> { self.expand_command(&mut task)?; } - task.metadata.expanded = true; + task.state.expanded = true; Ok(task) } diff --git a/crates/task-expander/tests/utils.rs b/crates/task-expander/tests/utils.rs index f3c96ee4562..2be8bf6a147 100644 --- a/crates/task-expander/tests/utils.rs +++ b/crates/task-expander/tests/utils.rs @@ -78,7 +78,7 @@ pub fn create_project_with_tasks(workspace_root: &Path, id: &str) -> Project { }; if task_id == "dev" { - task.metadata.local_only = true; + task.state.local_only = true; task.options.cache = false; task.options.persistent = true; } diff --git a/crates/task/src/task.rs b/crates/task/src/task.rs index 96ea0f62546..e89328fd6cb 100644 --- a/crates/task/src/task.rs +++ b/crates/task/src/task.rs @@ -16,7 +16,7 @@ use std::path::{Path, PathBuf}; cacheable!( #[derive(Clone, Debug, Default, Eq, PartialEq)] - pub struct TaskInternalMetadata { + pub struct TaskState { // Inputs were configured explicitly as `[]` pub empty_inputs: bool, @@ -59,8 +59,6 @@ cacheable!( #[serde(skip_serializing_if = "FxHashSet::is_empty")] pub input_globs: FxHashSet, - pub metadata: TaskInternalMetadata, - pub options: TaskOptions, pub outputs: Vec, @@ -79,6 +77,8 @@ cacheable!( #[serde(skip_serializing_if = "Option::is_none")] pub script: Option, + pub state: TaskState, + pub target: Target, #[serde(rename = "type")] @@ -195,7 +195,7 @@ impl Task { /// Return true if the task has been expanded. pub fn is_expanded(&self) -> bool { - self.metadata.expanded + self.state.expanded } /// Return true if an internal task. @@ -210,7 +210,7 @@ impl Task { /// Return true if a local only task. pub fn is_local(&self) -> bool { - self.metadata.local_only + self.state.local_only } /// Return true if the task is a "no operation" and does nothing. diff --git a/crates/vcs/Cargo.toml b/crates/vcs/Cargo.toml index 7efb0274640..8c7d9dd4650 100644 --- a/crates/vcs/Cargo.toml +++ b/crates/vcs/Cargo.toml @@ -12,6 +12,7 @@ publish = false moon_common = { path = "../common" } moon_process = { path = "../process" } async-trait = { workspace = true } +futures = { workspace = true } git-url-parse = "0.4.5" ignore = "0.4.23" miette = { workspace = true } @@ -21,6 +22,7 @@ rustc-hash = { workspace = true } scc = { workspace = true } semver = { workspace = true } serde = { workspace = true } +starbase_utils = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/crates/vcs/src/git.rs b/crates/vcs/src/git.rs index bfa5f9033e0..00ef9ed8e32 100644 --- a/crates/vcs/src/git.rs +++ b/crates/vcs/src/git.rs @@ -1,3 +1,4 @@ +use crate::git_submodule::*; use crate::process_cache::ProcessCache; use crate::touched_files::TouchedFiles; use crate::vcs::Vcs; @@ -99,9 +100,6 @@ pub struct Git { /// Root of the `.git` directory. pub git_root: PathBuf, - /// Ignore rules derived from a root `.gitignore` file. - ignore: Option, - /// Run and cache `git` commands. pub process: ProcessCache, @@ -113,6 +111,13 @@ pub struct Git { /// If in a git worktree, the root of the worktree (the `.git` file). pub worktree_root: Option, + + /// Ignore rules derived from a root `.gitignore` file. + ignore: Option, + + /// Map of submodules within the repository. + /// The root is also considered a module to keep things easy. + modules: BTreeMap, } impl Git { @@ -202,6 +207,19 @@ impl Git { ); } + // Load .gitmodules + let modules_path = repository_root.join(".gitmodules"); + let mut modules = BTreeMap::from_iter([("(root)".into(), GitModule::default())]); + + if modules_path.exists() { + debug!( + modules_file = ?modules_path, + "Loading submodules from .gitmodules", + ); + + modules.extend(parse_gitmodules_file(&modules_path)?); + } + Ok(Git { default_branch: Arc::new(default_branch.as_ref().to_owned()), ignore, @@ -215,6 +233,7 @@ impl Git { process: ProcessCache::new("git", workspace_root), git_root, worktree_root, + modules, }) } @@ -284,6 +303,246 @@ impl Git { Ok(self.default_branch.clone()) } + #[instrument(skip(self))] + async fn exec_diff( + &self, + module: &GitModule, + base_revision: &str, + revision: &str, + ) -> miette::Result { + let base = self.get_merge_base(base_revision, revision).await?; + + let output = self + .process + .run_command( + self.process.create_command_in_dir( + [ + "--no-pager", + "diff", + "--name-status", + "--no-color", + "--relative", + "--ignore-submodules", + // We use this option so that file names with special characters + // are displayed as-is and are not quoted/escaped + "-z", + base.as_ref().map(|b| b.as_str()).unwrap_or(base_revision), + ], + module.path.as_str(), + ), + false, + ) + .await?; + + if output.is_empty() { + return Ok(TouchedFiles::default()); + } + + let mut added = FxHashSet::default(); + let mut deleted = FxHashSet::default(); + let mut modified = FxHashSet::default(); + let mut staged = FxHashSet::default(); + let mut unstaged = FxHashSet::default(); + let mut last_status = "A"; + + // Lines AND statuses are terminated by a NUL byte + // X\0file\0 + // X000\0file\0 + // X000\0file\0renamed_file\0 + for line in output.split('\0') { + if line.is_empty() { + continue; + } + + // X\0 + // X000\0 + if DIFF_SCORE_PATTERN.is_match(line) || DIFF_PATTERN.is_match(line) { + last_status = &line[0..1]; + continue; + } + + let x = last_status.chars().next().unwrap_or_default(); + let file = module.path.join(self.to_workspace_relative_path(line)); + + match x { + 'A' | 'C' => { + added.insert(file.clone()); + staged.insert(file.clone()); + } + 'D' => { + deleted.insert(file.clone()); + staged.insert(file.clone()); + } + 'M' | 'R' | 'T' => { + modified.insert(file.clone()); + staged.insert(file.clone()); + } + 'U' => { + unstaged.insert(file.clone()); + } + _ => {} + } + } + + Ok(TouchedFiles { + added, + deleted, + modified, + staged, + unstaged, + untracked: FxHashSet::default(), + }) + } + + #[instrument(skip(self))] + async fn exec_ls_files( + &self, + module: &GitModule, + dir: &str, + ) -> miette::Result> { + let mut args = vec![ + "ls-files", + "--full-name", + "--cached", + "--modified", + "--others", // Includes untracked + "--exclude-standard", + ]; + + if self.is_version_supported(">=2.31.0").await? { + args.push("--deduplicate"); + } + + if !dir.is_empty() { + args.push(dir); + } + + let output = self + .process + .run_command( + self.process + .create_command_in_dir(args, module.path.as_str()), + false, + ) + .await?; + + let paths = output + .split('\n') + .filter_map(|file| { + let path = module.path.join(self.to_workspace_relative_path(file)); + + // Do not include directories + if self.process.root.join(path.as_str()).is_file() { + Some(path) + } else { + None + } + }) + .collect::>(); + + Ok(paths) + } + + // https://git-scm.com/docs/git-status#_short_format + #[instrument(skip(self))] + async fn exec_status(&self, module: &GitModule) -> miette::Result { + let output = self + .process + .run_command( + self.process.create_command_in_dir( + [ + "status", + "--porcelain", + "--untracked-files", + "--ignore-submodules", + // We use this option so that file names with special characters + // are displayed as-is and are not quoted/escaped + "-z", + ], + module.path.as_str(), + ), + false, + ) + .await?; + + if output.is_empty() { + return Ok(TouchedFiles::default()); + } + + let mut added = FxHashSet::default(); + let mut deleted = FxHashSet::default(); + let mut modified = FxHashSet::default(); + let mut untracked = FxHashSet::default(); + let mut staged = FxHashSet::default(); + let mut unstaged = FxHashSet::default(); + + // Lines are terminated by a NUL byte: + // XY file\0 + // XY file\0orig_file\0 + for line in output.split('\0') { + if line.is_empty() { + continue; + } + + // orig_file\0 + if !STATUS_PATTERN.is_match(line) { + continue; + } + + // XY file\0 + let mut chars = line.chars(); + let x = chars.next().unwrap_or_default(); + let y = chars.next().unwrap_or_default(); + let file = module + .path + .join(self.to_workspace_relative_path(&line[3..])); + + match x { + 'A' | 'C' => { + added.insert(file.clone()); + staged.insert(file.clone()); + } + 'D' => { + deleted.insert(file.clone()); + staged.insert(file.clone()); + } + 'M' | 'R' => { + modified.insert(file.clone()); + staged.insert(file.clone()); + } + _ => {} + } + + match y { + 'A' | 'C' => { + added.insert(file.clone()); + unstaged.insert(file.clone()); + } + 'D' => { + deleted.insert(file.clone()); + unstaged.insert(file.clone()); + } + 'M' | 'R' => { + modified.insert(file.clone()); + unstaged.insert(file.clone()); + } + '?' => { + untracked.insert(file.clone()); + } + _ => {} + } + } + + Ok(TouchedFiles { + added, + deleted, + modified, + staged, + unstaged, + untracked, + }) + } + fn to_workspace_relative_path(&self, value: &str) -> WorkspaceRelativePathBuf { let file = WorkspaceRelativePathBuf::from(value); @@ -395,26 +654,26 @@ impl Vcs for Git { #[instrument(skip(self))] async fn get_file_tree(&self, dir: &str) -> miette::Result> { - let mut args = vec![ - "ls-files", - "--full-name", - "--cached", - "--modified", - "--others", // Includes untracked - "--exclude-standard", - dir, - ]; - - if self.is_version_supported(">=2.31.0").await? { - args.push("--deduplicate"); + // Check to see if the requested dir is within a submodule + if let Some(module) = self + .modules + .values() + .find(|module| !module.is_root() && dir.starts_with(module.path.as_str())) + { + return self + .exec_ls_files( + module, + dir.strip_prefix(module.path.as_str()).unwrap_or_default(), + ) + .await; } - let output = self.process.run(args, true).await?; + // If not, then check against the root + if let Some(module) = self.modules.values().find(|module| module.is_root()) { + return self.exec_ls_files(module, dir).await; + } - Ok(output - .split('\n') - .map(|file| self.to_workspace_relative_path(file)) - .collect::>()) + Ok(vec![]) } async fn get_hooks_dir(&self) -> miette::Result { @@ -475,98 +734,18 @@ impl Vcs for Git { Err(GitError::ExtractRepoSlugFailed.into()) } - // https://git-scm.com/docs/git-status#_short_format - #[instrument(skip(self))] async fn get_touched_files(&self) -> miette::Result { - let output = self - .process - .run( - [ - "status", - "--porcelain", - "--untracked-files", - // We use this option so that file names with special characters - // are displayed as-is and are not quoted/escaped - "-z", - ], - false, - ) - .await?; - - if output.is_empty() { - return Ok(TouchedFiles::default()); - } + let mut touched_files = TouchedFiles::default(); - let mut added = FxHashSet::default(); - let mut deleted = FxHashSet::default(); - let mut modified = FxHashSet::default(); - let mut untracked = FxHashSet::default(); - let mut staged = FxHashSet::default(); - let mut unstaged = FxHashSet::default(); - - // Lines are terminated by a NUL byte: - // XY file\0 - // XY file\0orig_file\0 - for line in output.split('\0') { - if line.is_empty() { - continue; - } - - // orig_file\0 - if !STATUS_PATTERN.is_match(line) { - continue; - } - - // XY file\0 - let mut chars = line.chars(); - let x = chars.next().unwrap_or_default(); - let y = chars.next().unwrap_or_default(); - let file = self.to_workspace_relative_path(&line[3..]); - - match x { - 'A' | 'C' => { - added.insert(file.clone()); - staged.insert(file.clone()); - } - 'D' => { - deleted.insert(file.clone()); - staged.insert(file.clone()); - } - 'M' | 'R' => { - modified.insert(file.clone()); - staged.insert(file.clone()); - } - _ => {} - } - - match y { - 'A' | 'C' => { - added.insert(file.clone()); - unstaged.insert(file.clone()); - } - 'D' => { - deleted.insert(file.clone()); - unstaged.insert(file.clone()); - } - 'M' | 'R' => { - modified.insert(file.clone()); - unstaged.insert(file.clone()); - } - '?' => { - untracked.insert(file.clone()); - } - _ => {} - } + for result in futures::future::try_join_all( + self.modules.values().map(|module| self.exec_status(module)), + ) + .await? + { + touched_files.merge(result); } - Ok(TouchedFiles { - added, - deleted, - modified, - staged, - unstaged, - untracked, - }) + Ok(touched_files) } async fn get_touched_files_against_previous_revision( @@ -602,84 +781,19 @@ impl Vcs for Git { base_revision: &str, revision: &str, ) -> miette::Result { - let base = self.get_merge_base(base_revision, revision).await?; - - let output = self - .process - .run( - [ - "--no-pager", - "diff", - "--name-status", - "--no-color", - "--relative", - // We use this option so that file names with special characters - // are displayed as-is and are not quoted/escaped - "-z", - base.as_ref().map(|b| b.as_str()).unwrap_or(base_revision), - ], - false, - ) - .await?; - - if output.is_empty() { - return Ok(TouchedFiles::default()); - } + let mut touched_files = TouchedFiles::default(); - let mut added = FxHashSet::default(); - let mut deleted = FxHashSet::default(); - let mut modified = FxHashSet::default(); - let mut staged = FxHashSet::default(); - let mut unstaged = FxHashSet::default(); - let mut last_status = "A"; - - // Lines AND statuses are terminated by a NUL byte - // X\0file\0 - // X000\0file\0 - // X000\0file\0renamed_file\0 - for line in output.split('\0') { - if line.is_empty() { - continue; - } - - // X\0 - // X000\0 - if DIFF_SCORE_PATTERN.is_match(line) || DIFF_PATTERN.is_match(line) { - last_status = &line[0..1]; - continue; - } - - let x = last_status.chars().next().unwrap_or_default(); - let file = self.to_workspace_relative_path(line); - - match x { - 'A' | 'C' => { - added.insert(file.clone()); - staged.insert(file.clone()); - } - 'D' => { - deleted.insert(file.clone()); - staged.insert(file.clone()); - } - 'M' | 'R' | 'T' => { - modified.insert(file.clone()); - staged.insert(file.clone()); - } - 'U' => { - unstaged.insert(file.clone()); - } - _ => {} - } + for result in futures::future::try_join_all( + self.modules + .values() + .map(|module| self.exec_diff(module, base_revision, revision)), + ) + .await? + { + touched_files.merge(result); } - Ok(TouchedFiles { - added, - deleted, - modified, - staged, - unstaged, - untracked: FxHashSet::default(), - }) + Ok(touched_files) } async fn get_version(&self) -> miette::Result { diff --git a/crates/vcs/src/git_submodule.rs b/crates/vcs/src/git_submodule.rs new file mode 100644 index 00000000000..74fea813c67 --- /dev/null +++ b/crates/vcs/src/git_submodule.rs @@ -0,0 +1,63 @@ +use moon_common::path::RelativePathBuf; +use rustc_hash::FxHashMap; +use starbase_utils::fs; +use std::path::Path; + +#[derive(Debug, Default)] +pub struct GitModule { + pub branch: Option, + pub path: RelativePathBuf, + pub url: String, +} + +impl GitModule { + pub fn is_root(&self) -> bool { + self.path.as_str().is_empty() + } +} + +pub fn parse_gitmodules_file(path: &Path) -> miette::Result> { + let mut modules = FxHashMap::default(); + let mut current_module_name = None; + let mut current_module = GitModule::default(); + let contents = fs::read_file(path)?; + + fn clean_line(line: &str) -> String { + line.replace("=", "").replace("\"", "").trim().to_owned() + } + + for line in contents.lines() { + let line = line.trim(); + + if line.starts_with("[submodule") { + if current_module_name.is_some() { + modules.insert(current_module_name.take().unwrap(), current_module); + current_module = GitModule::default(); + } + + let name = line + .replace("[submodule", "") + .replace("\"", "") + .replace("]", "") + .trim() + .to_owned(); + + current_module_name = Some(name); + } else if let Some(value) = line.strip_prefix("branch") { + current_module.branch = Some(clean_line(value)); + } else if let Some(value) = line.strip_prefix("path") { + current_module.path = RelativePathBuf::from(clean_line(value)); + } else if let Some(value) = line.strip_prefix("url") { + current_module.url = clean_line(value); + } + } + + if current_module_name.is_some() { + modules.insert(current_module_name.take().unwrap(), current_module); + } + + Ok(modules + .into_iter() + .filter(|(_, module)| !module.path.as_str().is_empty()) + .collect()) +} diff --git a/crates/vcs/src/lib.rs b/crates/vcs/src/lib.rs index 453ee7e9f4b..d98280b6043 100644 --- a/crates/vcs/src/lib.rs +++ b/crates/vcs/src/lib.rs @@ -1,4 +1,5 @@ mod git; +mod git_submodule; mod process_cache; mod touched_files; mod vcs; diff --git a/crates/vcs/src/process_cache.rs b/crates/vcs/src/process_cache.rs index 019f853b7b4..d5459cce8f7 100644 --- a/crates/vcs/src/process_cache.rs +++ b/crates/vcs/src/process_cache.rs @@ -42,6 +42,21 @@ impl ProcessCache { command } + pub fn create_command_in_dir(&self, args: I, dir: &str) -> Command + where + I: IntoIterator, + A: AsRef, + { + let mut command = self.create_command(args); + + // Run in a directory to support submodules + if !dir.is_empty() && dir != "." { + command.cwd(self.root.join(dir)); + } + + command + } + pub async fn run(&self, args: I, trim: bool) -> miette::Result> where I: IntoIterator, diff --git a/crates/vcs/src/touched_files.rs b/crates/vcs/src/touched_files.rs index 53865496b69..a706ffab1fe 100644 --- a/crates/vcs/src/touched_files.rs +++ b/crates/vcs/src/touched_files.rs @@ -27,6 +27,15 @@ impl TouchedFiles { files.extend(&self.unstaged); files } + + pub fn merge(&mut self, other: TouchedFiles) { + self.added.extend(other.added); + self.deleted.extend(other.deleted); + self.modified.extend(other.modified); + self.untracked.extend(other.untracked); + self.staged.extend(other.staged); + self.unstaged.extend(other.unstaged); + } } #[derive(Clone, Copy, Debug, Deserialize, Default, Eq, PartialEq, Serialize)] diff --git a/crates/workspace/src/workspace_builder.rs b/crates/workspace/src/workspace_builder.rs index 0ed4240f8eb..0c60294d695 100644 --- a/crates/workspace/src/workspace_builder.rs +++ b/crates/workspace/src/workspace_builder.rs @@ -290,11 +290,6 @@ impl<'app> WorkspaceBuilder<'app> { } // Not loaded, build the project - trace!( - project_id = id.as_str(), - "Project does not exist in the project graph, attempting to load", - ); - let mut project = self.build_project(&id).await?; cycle.insert(id.clone()); @@ -474,11 +469,6 @@ impl<'app> WorkspaceBuilder<'app> { } // Not loaded, resolve the task - trace!( - task_target = target.as_str(), - "Task does not exist in the task graph, attempting to load", - ); - let (_, project_index) = self .internal_load_project(target.get_project_id().unwrap(), &mut FxHashSet::default()) .await?; diff --git a/legacy/core/utils/Cargo.toml b/legacy/core/utils/Cargo.toml index 556d973a6d3..c14231d00eb 100644 --- a/legacy/core/utils/Cargo.toml +++ b/legacy/core/utils/Cargo.toml @@ -12,7 +12,7 @@ chrono = { workspace = true } clean-path = "0.2.1" dunce = "1.0.5" humantime = "2.1.0" -md5 = "0.7.0" +md5 = { workspace = true } miette = { workspace = true } once_cell = { workspace = true } pathdiff = { workspace = true } diff --git a/legacy/python/lang/src/pip_requirements.rs b/legacy/python/lang/src/pip_requirements.rs index 9bef4a5be5b..c581227c6d2 100644 --- a/legacy/python/lang/src/pip_requirements.rs +++ b/legacy/python/lang/src/pip_requirements.rs @@ -23,10 +23,8 @@ pub fn load_lockfile_dependencies(path: PathBuf) -> miette::Result, - #[allow(dead_code)] pub workspace_root: PathBuf, } diff --git a/legacy/python/tool/src/python_tool.rs b/legacy/python/tool/src/python_tool.rs index 1b62a57ee85..28dcaff599c 100644 --- a/legacy/python/tool/src/python_tool.rs +++ b/legacy/python/tool/src/python_tool.rs @@ -1,6 +1,6 @@ use moon_config::PythonConfig; use moon_console::{Checkpoint, Console}; -use moon_logger::{debug, map_list}; +use moon_logger::debug; use moon_process::Command; use moon_tool::{ async_trait, get_proto_env_vars, get_proto_paths, load_tool_plugin, prepend_path_env_var, @@ -10,32 +10,19 @@ use moon_toolchain::RuntimeReq; use proto_core::flow::install::InstallOptions; use proto_core::{Id, ProtoEnvironment, Tool as ProtoTool, UnresolvedVersionSpec}; use rustc_hash::FxHashMap; -use starbase_styles::color; use std::path::PathBuf; use std::sync::Arc; use std::{ffi::OsStr, path::Path}; use tracing::instrument; -const LOG_TARGET: &str = "moon:python-tool"; - pub fn get_python_tool_paths(python_tool: &PythonTool, working_dir: &Path) -> Vec { - let venv_python = working_dir.join(python_tool.config.venv_name.clone()); + let venv_python = working_dir.join(&python_tool.config.venv_name); - let paths = if venv_python.exists() { - vec![ - venv_python.join("Scripts").clone(), - venv_python.join("bin").clone(), - ] + if venv_python.exists() { + vec![venv_python.join("Scripts"), venv_python.join("bin")] } else { get_proto_paths(&python_tool.proto_env) - }; - - debug!( - target: LOG_TARGET, - "Proto Env {} ", - map_list(&paths, |c| color::label(c.display().to_string())), - ); - paths + } } pub struct PythonTool { @@ -151,12 +138,15 @@ impl Tool for PythonTool { } } } + self.tool.locate_globals_dirs().await?; + Ok(installed) } async fn teardown(&mut self) -> miette::Result<()> { self.tool.teardown().await?; + Ok(()) } } diff --git a/packages/types/src/project.ts b/packages/types/src/project.ts index 41dcae86929..ba712b6d31c 100644 --- a/packages/types/src/project.ts +++ b/packages/types/src/project.ts @@ -51,7 +51,7 @@ export interface TaskOptions { windowsShell: TaskWindowsShell | null; } -export interface TaskMetadata { +export interface TaskState { emptyInputs: boolean; expanded: boolean; localOnly: boolean; @@ -68,12 +68,13 @@ export interface Task { inputFiles: string[]; inputGlobs: string[]; inputVars: string[]; - metadata: TaskMetadata; options: TaskOptions; outputs: string[]; outputFiles: string[]; outputGlobs: string[]; platform: PlatformType; + script: string | null; + state: TaskState; target: string; type: TaskType; }