diff --git a/.gitignore b/.gitignore index 251f1c9cecc..60975177810 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /target node_modules/ .nx +bazel-remote # Yarn .yarn/* diff --git a/.moon/workspace.yml b/.moon/workspace.yml index 377549da3a5..a1da6f0790b 100644 --- a/.moon/workspace.yml +++ b/.moon/workspace.yml @@ -35,3 +35,15 @@ docker: include: - '*.config.js' - '*.json' + +unstable_remote: + host: 'grpc://localhost:9092' + # mtls: + # caCert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem' + # clientCert: 'crates/remote/tests/__fixtures__/certs-local/client.pem' + # clientKey: 'crates/remote/tests/__fixtures__/certs-local/client.key' + # domain: 'localhost' + # tls: + # # assumeHttp2: true + # cert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem' + # # domain: 'localhost' diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e82de68c7b..57742019272 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,12 @@ #### 🚀 Updates -- Resolved the `strictProjectIds` experiment and you can no longer reference the original ID. -- Resolved the `disallowRunInCiMismatch` experiment and you can no longer have a CI based task - depend on a non-CI based task. +- Added unstable support for self-hosted remote caches, powered by the + [Bazel Remote Execution API](https://github.com/bazelbuild/remote-apis). + - Allows for 3rd-party implementations like + [`bazel-remote`](https://github.com/buchgr/bazel-remote) to be used. + - Currently supports the gRPC protocol, and will support HTTP in a later release. + - Our moonbase product will be sunset in the future. - Added a new task graph, that enables new granular based functionality for task related features. - Added a new `moon task-graph` command. - Can now control the depth of upstream (dependencies) and downstream (dependents). @@ -24,6 +27,9 @@ `$vcsRevision`, `$workingDir` - Added a `rust.binstallVersion` setting to `.moon/toolchain.yml`. - Updated Pkl configurations to support `read()` for environment variables. +- Resolved the `strictProjectIds` experiment and you can no longer reference the original ID. +- Resolved the `disallowRunInCiMismatch` experiment and you can no longer have a CI based task + depend on a non-CI based task. #### 🐞 Fixes diff --git a/Cargo.lock b/Cargo.lock index 627e152d4a0..c69d8d65094 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,6 +371,28 @@ dependencies = [ "wasm-bindgen-futures", ] +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "async-task" version = "4.7.1" @@ -400,6 +422,53 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 1.0.1", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -447,6 +516,18 @@ dependencies = [ "regex", ] +[[package]] +name = "bazel-remote-apis" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31bbf354d9565230c677c9f8fa60dff7427d8b5a5a49477a716ae13882679959" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tonic-build", +] + [[package]] name = "binstall-tar" version = "0.4.42" @@ -699,7 +780,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" dependencies = [ "heck 0.4.1", - "indexmap", + "indexmap 2.6.0", "log", "proc-macro2", "quote", @@ -1896,7 +1977,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" dependencies = [ "fallible-iterator", - "indexmap", + "indexmap 2.6.0", "stable_deref_trait", ] @@ -1982,13 +2063,19 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.13.2" @@ -2192,6 +2279,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -2217,6 +2305,19 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper 1.4.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.6.0" @@ -2442,6 +2543,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.6.0" @@ -2832,6 +2943,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "maybe-owned" version = "0.3.4" @@ -3030,6 +3147,7 @@ dependencies = [ "moon_console", "moon_notifier", "moon_project", + "moon_remote", "moon_task", "moon_toolchain", "moon_toolchain_plugin", @@ -3061,6 +3179,7 @@ dependencies = [ "moon_platform", "moon_process", "moon_project", + "moon_remote", "moon_task_runner", "moon_time", "moon_toolchain_plugin", @@ -3168,6 +3287,7 @@ dependencies = [ "moon_python_platform", "moon_python_tool", "moon_query", + "moon_remote", "moon_rust_lang", "moon_rust_platform", "moon_rust_tool", @@ -3412,7 +3532,7 @@ name = "moon_config" version = "0.0.10" dependencies = [ "httpmock", - "indexmap", + "indexmap 2.6.0", "miette", "moon_common", "moon_config", @@ -3987,6 +4107,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "moon_remote" +version = "0.0.1" +dependencies = [ + "async-trait", + "bazel-remote-apis", + "chrono", + "filetime", + "miette", + "moon_action", + "moon_common", + "moon_config", + "prost-types", + "reqwest", + "rustc-hash 2.0.0", + "scc", + "sha2", + "starbase_utils", + "thiserror", + "tokio", + "tonic", + "tracing", +] + [[package]] name = "moon_rust_lang" version = "0.0.1" @@ -4208,6 +4352,7 @@ dependencies = [ "moon_platform", "moon_process", "moon_project", + "moon_remote", "moon_task", "moon_task_hasher", "moon_test_utils2", @@ -4452,6 +4597,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" + [[package]] name = "native-tls" version = "0.2.12" @@ -4501,7 +4652,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5b1935bec0b4fb49c42f50bd6822a3171cbf55a0d9cd1810715baa2f319a1d7" dependencies = [ - "indexmap", + "indexmap 2.6.0", "rustc-hash 2.0.0", "semver", "serde", @@ -4577,7 +4728,7 @@ checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "crc32fast", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.6.0", "memchr", ] @@ -4799,7 +4950,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.6.0", "serde", "serde_derive", ] @@ -4857,6 +5008,26 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" +[[package]] +name = "pin-project" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -4983,6 +5154,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +dependencies = [ + "proc-macro2", + "syn 2.0.87", +] + [[package]] name = "proc-macro-crate" version = "3.2.0" @@ -5022,6 +5203,27 @@ dependencies = [ "prost-derive", ] +[[package]] +name = "prost-build" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8650aabb6c35b860610e9cff5dc1af886c9e25073b7b1712a68972af4281302" +dependencies = [ + "bytes", + "heck 0.5.0", + "itertools 0.13.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.87", + "tempfile", +] + [[package]] name = "prost-derive" version = "0.13.3" @@ -5035,6 +5237,15 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "prost-types" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60caa6738c7369b940c3d49246a8d1749323674c65cb13010134f5c9bad5b519" +dependencies = [ + "prost", +] + [[package]] name = "proto_core" version = "0.43.5" @@ -5042,7 +5253,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "febd80c3f5b8e51a8ba76f383a3ab40554f7984413d348c5a28d1d9496f1deed" dependencies = [ "convert_case", - "indexmap", + "indexmap 2.6.0", "miette", "minisign-verify", "once_cell", @@ -5367,7 +5578,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.1", "system-configuration", "tokio", "tokio-native-tls", @@ -5599,7 +5810,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b13cc48a6f0f9cdb16cfa88f7a104e1e77483b03e154b0de40c68af41462d0db" dependencies = [ "garde", - "indexmap", + "indexmap 2.6.0", "markdown", "miette", "reqwest", @@ -5636,7 +5847,7 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "768ff9d8765442119bfb73b7808607f76f75fb2987932637d43b144d6ad5e3fe" dependencies = [ - "indexmap", + "indexmap 2.6.0", "rpkl", "semver", "serde", @@ -5716,7 +5927,7 @@ version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ - "indexmap", + "indexmap 2.6.0", "itoa", "memchr", "ryu", @@ -5770,7 +5981,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -6203,6 +6414,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "sync_wrapper" version = "1.0.1" @@ -6511,6 +6728,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.12" @@ -6551,13 +6779,87 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", "winnow", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "flate2", + "h2", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "rustls-native-certs 0.8.0", + "rustls-pemfile", + "socket2", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -6656,7 +6958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9901b6b03c576f11b614000e04b5416649faf695a2ccb236ef3b76518533331f" dependencies = [ "clean-path", - "indexmap", + "indexmap 2.6.0", "rustc-hash 2.0.0", "serde", "serde_json", @@ -7089,7 +7391,7 @@ dependencies = [ "ahash", "bitflags 2.6.0", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.6.0", "semver", "serde", ] @@ -7122,7 +7424,7 @@ dependencies = [ "fxprof-processed-profile", "gimli 0.28.1", "hashbrown 0.14.5", - "indexmap", + "indexmap 2.6.0", "ittapi", "libc", "libm", @@ -7246,7 +7548,7 @@ dependencies = [ "cranelift-bitset", "cranelift-entity", "gimli 0.28.1", - "indexmap", + "indexmap 2.6.0", "log", "object", "postcard", @@ -7357,7 +7659,7 @@ checksum = "8f88e49a9b81746ec0cede5505e40a4012c92cb5054cd7ef4300dc57c36f26b1" dependencies = [ "anyhow", "heck 0.4.1", - "indexmap", + "indexmap 2.6.0", "wit-parser", ] @@ -7794,7 +8096,7 @@ checksum = "ceeb0424aa8679f3fcf2d6e3cfa381f3d6fa6179976a2c05a6249dd2bb426716" dependencies = [ "anyhow", "id-arena", - "indexmap", + "indexmap 2.6.0", "log", "semver", "serde", @@ -7969,7 +8271,7 @@ dependencies = [ "crossbeam-utils", "displaydoc", "flate2", - "indexmap", + "indexmap 2.6.0", "memchr", "thiserror", "zopfli", diff --git a/Cargo.toml b/Cargo.toml index 1245a698907..8c3a25a1285 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,17 +1,17 @@ [workspace] resolver = "2" members = [ - "crates/*", - "legacy/core/*", + "crates/*", + "legacy/core/*", - # Languages - "legacy/bun/*", - "legacy/deno/*", - "legacy/javascript/*", - "legacy/node/*", - "legacy/rust/*", - "legacy/system/*", - "legacy/typescript/*", + # Languages + "legacy/bun/*", + "legacy/deno/*", + "legacy/javascript/*", + "legacy/node/*", + "legacy/rust/*", + "legacy/system/*", + "legacy/typescript/*", ] exclude = ["tests/fixtures", "wasm/test-plugin"] default-members = ["crates/cli"] @@ -24,12 +24,12 @@ chrono = { version = "0.4.38", features = ["serde"] } cd_env = "0.2.0" ci_env = "0.3.0" clap = { version = "4.5.21", default-features = false, features = [ - "std", - "error-context", + "std", + "error-context", ] } clap_complete = "4.5.38" compact_str = { version = "0.8.0", default-features = false, features = [ - "serde", + "serde", ] } console = "0.15.8" dirs = "5.0.1" @@ -38,51 +38,52 @@ miette = "7.2.0" once_cell = "1.20.1" pathdiff = "0.2.2" petgraph = { version = "0.6.5", default-features = false, features = [ - "serde-1", + "serde-1", ] } relative-path = { version = "1.9.3" } regex = { version = "1.11.0", default-features = false, features = [ - "std", - "perf", + "std", + "perf", ] } reqwest = { version = "0.12.9", default-features = false, features = [ - "rustls-tls-native-roots", - # We don't use openssl but its required for musl builds - "native-tls-vendored", + "rustls-tls-native-roots", + # We don't use openssl but its required for musl builds + "native-tls-vendored", ] } rustc-hash = "2.0.0" scc = "2.2.4" schematic = { version = "0.17.6", default-features = false, features = [ - "schema", + "schema", ] } serial_test = "3.2.0" semver = "1.0.23" serde = { version = "1.0.215", features = ["derive"] } serde_json = "1.0.128" serde_yaml = "0.9.34" +sha2 = "0.10.8" starbase = { version = "0.9.4" } starbase_archive = { version = "0.8.9", default-features = false, features = [ - "miette", - "tar-gz", + "miette", + "tar-gz", ] } starbase_events = "0.6.4" starbase_sandbox = "0.7.6" starbase_shell = "0.5.10" starbase_styles = { version = "0.4.4", features = ["relative-path"] } starbase_utils = { version = "0.8.12", default-features = false, features = [ - "editor-config", - "miette", + "editor-config", + "miette", ] } tera = { version = "1.20.0", features = ["preserve_order"] } thiserror = "1.0.64" tokio = { version = "1.41.1", default-features = false, features = [ - "macros", - "process", - "rt-multi-thread", - "rt", - "signal", - "time", - "tracing", + "macros", + "process", + "rt-multi-thread", + "rt", + "signal", + "time", + "tracing", ] } tokio-util = "0.7.12" tracing = "0.1.40" diff --git a/crates/action-pipeline/Cargo.toml b/crates/action-pipeline/Cargo.toml index 249782f1738..afd6525b845 100644 --- a/crates/action-pipeline/Cargo.toml +++ b/crates/action-pipeline/Cargo.toml @@ -17,6 +17,7 @@ moon_common = { path = "../common" } moon_console = { path = "../console" } moon_notifier = { path = "../notifier" } moon_project = { path = "../project" } +moon_remote = { path = "../remote" } moon_task = { path = "../task" } moon_toolchain = { path = "../toolchain" } moon_toolchain_plugin = { path = "../toolchain-plugin" } diff --git a/crates/action-pipeline/src/action_pipeline.rs b/crates/action-pipeline/src/action_pipeline.rs index 31b66152384..e4163d41fdd 100644 --- a/crates/action-pipeline/src/action_pipeline.rs +++ b/crates/action-pipeline/src/action_pipeline.rs @@ -5,6 +5,7 @@ use crate::job_dispatcher::JobDispatcher; use crate::subscribers::cleanup_subscriber::CleanupSubscriber; use crate::subscribers::console_subscriber::ConsoleSubscriber; use crate::subscribers::moonbase_subscriber::MoonbaseSubscriber; +use crate::subscribers::remote_subscriber::RemoteSubscriber; use crate::subscribers::reports_subscriber::ReportsSubscriber; use crate::subscribers::webhooks_subscriber::WebhooksSubscriber; use moon_action::{Action, ActionNode}; @@ -351,6 +352,8 @@ impl ActionPipeline { )) .await; + self.emitter.subscribe(RemoteSubscriber).await; + debug!("Subscribing run reports and estimates"); self.emitter diff --git a/crates/action-pipeline/src/subscribers/mod.rs b/crates/action-pipeline/src/subscribers/mod.rs index 5bea7535a23..cca661ce6ec 100644 --- a/crates/action-pipeline/src/subscribers/mod.rs +++ b/crates/action-pipeline/src/subscribers/mod.rs @@ -1,5 +1,6 @@ pub mod cleanup_subscriber; pub mod console_subscriber; pub mod moonbase_subscriber; +pub mod remote_subscriber; pub mod reports_subscriber; pub mod webhooks_subscriber; diff --git a/crates/action-pipeline/src/subscribers/remote_subscriber.rs b/crates/action-pipeline/src/subscribers/remote_subscriber.rs new file mode 100644 index 00000000000..8104d5ce677 --- /dev/null +++ b/crates/action-pipeline/src/subscribers/remote_subscriber.rs @@ -0,0 +1,22 @@ +use crate::event_emitter::{Event, Subscriber}; +use async_trait::async_trait; +use moon_remote::RemoteService; +use tracing::debug; + +#[derive(Default)] +pub struct RemoteSubscriber; + +#[async_trait] +impl Subscriber for RemoteSubscriber { + async fn on_emit<'data>(&mut self, event: &Event<'data>) -> miette::Result<()> { + if matches!(event, Event::PipelineCompleted { .. }) { + if let Some(session) = RemoteService::session() { + debug!("Waiting for in-flight remote service requests to finish"); + + session.wait_for_requests().await; + } + } + + Ok(()) + } +} diff --git a/crates/action/src/operation.rs b/crates/action/src/operation.rs index 191b104b807..8cfbf3842b4 100644 --- a/crates/action/src/operation.rs +++ b/crates/action/src/operation.rs @@ -83,6 +83,19 @@ impl Operation { .unwrap_or_else(|| "unknown failure".into()) } + pub fn label(&self) -> &str { + match &self.meta { + OperationMeta::NoOperation => "NoOperation", + OperationMeta::OutputHydration(_) => "OutputHydration", + OperationMeta::ProcessExecution(_) => "ProcessExecution", + OperationMeta::SyncOperation(_) => "SyncOperation", + OperationMeta::TaskExecution(_) => "TaskExecution", + OperationMeta::ArchiveCreation => "ArchiveCreation", + OperationMeta::HashGeneration(_) => "HashGeneration", + OperationMeta::MutexAcquisition => "MutexAcquisition", + } + } + pub fn finish(&mut self, status: ActionStatus) { self.finished_at = Some(now_timestamp()); self.status = status; diff --git a/crates/actions/Cargo.toml b/crates/actions/Cargo.toml index 9926a0a6304..0a75a55f1bb 100644 --- a/crates/actions/Cargo.toml +++ b/crates/actions/Cargo.toml @@ -17,6 +17,7 @@ moon_hash = { path = "../hash" } moon_pdk_api = { path = "../pdk-api" } moon_process = { path = "../process" } moon_project = { path = "../project" } +moon_remote = { path = "../remote" } moon_task_runner = { path = "../task-runner" } moon_time = { path = "../time" } moon_toolchain_plugin = { path = "../toolchain-plugin" } diff --git a/crates/actions/src/actions/install_deps.rs b/crates/actions/src/actions/install_deps.rs index 603f4f8794a..43581e7e79b 100644 --- a/crates/actions/src/actions/install_deps.rs +++ b/crates/actions/src/actions/install_deps.rs @@ -244,7 +244,7 @@ async fn hash_manifests( } } - let hash = app_context.cache_engine.hash.save_manifest(hasher)?; + let (hash, _) = app_context.cache_engine.hash.save_manifest(hasher)?; operation.meta.set_hash(&hash); operation.finish(ActionStatus::Passed); diff --git a/crates/actions/src/actions/sync_workspace.rs b/crates/actions/src/actions/sync_workspace.rs index dd0dccab794..37857a75ed0 100644 --- a/crates/actions/src/actions/sync_workspace.rs +++ b/crates/actions/src/actions/sync_workspace.rs @@ -7,6 +7,7 @@ use moon_action::{Action, ActionStatus, Operation}; use moon_action_context::ActionContext; use moon_app_context::AppContext; use moon_common::color; +use moon_remote::RemoteService; use moon_toolchain_plugin::ToolchainRegistry; use moon_workspace_graph::WorkspaceGraph; use std::sync::Arc; @@ -21,6 +22,13 @@ pub async fn sync_workspace( workspace_graph: WorkspaceGraph, toolchain_registry: Arc, ) -> miette::Result { + // Connect to the remote service in this action, + // as it always runs before tasks, and we don't need it + // for non-pipeline related features! + if let Some(remote_config) = &app_context.workspace_config.remote { + RemoteService::connect(remote_config, &app_context.workspace_root).await?; + } + if should_skip_action("MOON_SKIP_SYNC_WORKSPACE").is_some() { debug!( "Skipping workspace sync because {} is set", diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index b4a4ecc581a..8ecd6a2f28e 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -26,6 +26,7 @@ moon_plugin = { path = "../plugin" } moon_project = { path = "../project" } moon_project_graph = { path = "../project-graph" } moon_query = { path = "../query" } +moon_remote = { path = "../remote" } moon_task = { path = "../task" } moon_task_graph = { path = "../task-graph" } moon_toolchain = { path = "../toolchain" } diff --git a/crates/app/src/session.rs b/crates/app/src/session.rs index 7c03b4c5ee1..8260f0f0881 100644 --- a/crates/app/src/session.rs +++ b/crates/app/src/session.rs @@ -4,7 +4,6 @@ use crate::components::*; use crate::systems::*; use async_trait::async_trait; use moon_action_graph::ActionGraphBuilder; -use moon_api::Moonbase; use moon_app_context::AppContext; use moon_cache::CacheEngine; use moon_common::{is_ci, is_test_env}; @@ -39,7 +38,6 @@ pub struct CliSession { // Components pub config_loader: ConfigLoader, pub console: Console, - pub moonbase: Option>, pub moon_env: Arc, pub proto_env: Arc, @@ -71,7 +69,6 @@ impl CliSession { config_loader: ConfigLoader::default(), console: Console::new(cli.quiet), extension_registry: OnceCell::new(), - moonbase: None, moon_env: Arc::new(MoonEnvironment::default()), project_graph: OnceCell::new(), proto_env: Arc::new(ProtoEnvironment::default()), @@ -277,7 +274,7 @@ impl AppSession for CliSession { if !is_test_env() && is_ci() { let vcs = self.get_vcs_adapter()?; - self.moonbase = startup::signin_to_moonbase(&vcs).await?; + startup::signin_to_moonbase(&vcs).await?; } Ok(None) diff --git a/crates/cache/src/cache_engine.rs b/crates/cache/src/cache_engine.rs index 209ba8bc3f3..240da9f7604 100644 --- a/crates/cache/src/cache_engine.rs +++ b/crates/cache/src/cache_engine.rs @@ -141,7 +141,7 @@ impl CacheEngine { let name = fs::file_name(&path); let mut state = self.state.load_state::(&name)?; - let hash = self.hash.save_manifest_without_hasher(&name, data)?; + let (hash, _) = self.hash.save_manifest_without_hasher(&name, data)?; if hash != state.data.last_hash { let result = op().await?; diff --git a/crates/cache/src/hash_engine.rs b/crates/cache/src/hash_engine.rs index c017f6131e6..412881f2f17 100644 --- a/crates/cache/src/hash_engine.rs +++ b/crates/cache/src/hash_engine.rs @@ -44,22 +44,24 @@ impl HashEngine { self.hashes_dir.join(format!("{hash}.json")) } - pub fn save_manifest(&self, mut hasher: ContentHasher) -> miette::Result { + pub fn save_manifest(&self, mut hasher: ContentHasher) -> miette::Result<(String, usize)> { let hash = hasher.generate_hash()?; let path = self.get_manifest_path(&hash); debug!(label = hasher.label, manifest = ?path, "Saving hash manifest"); - fs::write_file(&path, hasher.serialize()?)?; + let data = hasher.serialize()?; - Ok(hash) + fs::write_file(&path, data)?; + + Ok((hash, data.len())) } pub fn save_manifest_without_hasher( &self, label: &str, content: T, - ) -> miette::Result { + ) -> miette::Result<(String, usize)> { let mut hasher = ContentHasher::new(label); hasher.hash_content(content)?; diff --git a/crates/cache/tests/hash_engine_test.rs b/crates/cache/tests/hash_engine_test.rs index bd8fa27338d..0545bbf335f 100644 --- a/crates/cache/tests/hash_engine_test.rs +++ b/crates/cache/tests/hash_engine_test.rs @@ -23,7 +23,7 @@ fn saves_manifest() { }) .unwrap(); - let hash = engine.save_manifest(hasher).unwrap(); + let (hash, _) = engine.save_manifest(hasher).unwrap(); assert_eq!( hash, @@ -48,7 +48,7 @@ fn saves_manifest_without_hasher() { let sandbox = create_empty_sandbox(); let engine = HashEngine::new(sandbox.path()).unwrap(); - let hash = engine + let (hash, _) = engine .save_manifest_without_hasher( "test", Content { diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 77b1e588803..7cf3e50ec1e 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -30,7 +30,18 @@ fn get_version() -> String { } fn get_tracing_modules() -> Vec { - let mut modules = string_vec!["moon", "proto", "schematic", "starbase", "warpgate"]; + let mut modules = string_vec![ + "moon", + "proto", + "schematic", + "starbase", + "warpgate", + // Remote testing + // "h2", + // "hyper", + // "tonic", + // "rustls", + ]; if env::var("MOON_DEBUG_WASM").is_ok() { modules.push("extism".into()); diff --git a/crates/config/src/shapes/poly.rs b/crates/config/src/shapes/poly.rs index cd115a024d5..0e8185f87ec 100644 --- a/crates/config/src/shapes/poly.rs +++ b/crates/config/src/shapes/poly.rs @@ -11,6 +11,12 @@ cacheable!( } ); +impl Default for OneOrMany { + fn default() -> Self { + Self::Many(vec![]) + } +} + impl OneOrMany { pub fn to_list(&self) -> Vec { match self { diff --git a/crates/config/src/workspace/mod.rs b/crates/config/src/workspace/mod.rs index 1bb963b5bc7..bcb9eee79d8 100644 --- a/crates/config/src/workspace/mod.rs +++ b/crates/config/src/workspace/mod.rs @@ -6,6 +6,7 @@ mod generator_config; mod hasher_config; mod notifier_config; mod plugins_config; +mod remote_config; mod runner_config; mod vcs_config; @@ -17,5 +18,6 @@ pub use generator_config::*; pub use hasher_config::*; pub use notifier_config::*; pub use plugins_config::*; +pub use remote_config::*; pub use runner_config::*; pub use vcs_config::*; diff --git a/crates/config/src/workspace/remote_config.rs b/crates/config/src/workspace/remote_config.rs new file mode 100644 index 00000000000..800a54128d5 --- /dev/null +++ b/crates/config/src/workspace/remote_config.rs @@ -0,0 +1,96 @@ +use crate::portable_path::FilePath; +use schematic::{validate, Config, ValidateError, ValidateResult}; + +fn path_is_required( + value: &FilePath, + _data: &D, + _context: &C, + _finalize: bool, +) -> ValidateResult { + if value.as_str().is_empty() { + return Err(ValidateError::new("path must not be empty")); + } + + Ok(()) +} + +/// Configures the action cache (AC) and content addressable cache (CAS). +#[derive(Clone, Config, Debug)] +pub struct RemoteCacheConfig { + #[setting(default = "moon-outputs")] + pub instance_name: String, +} + +/// Configures for server-only authentication with TLS. +#[derive(Clone, Config, Debug)] +pub struct RemoteTlsConfig { + /// If true, assume that the server supports HTTP/2, + /// even if it doesn't provide protocol negotiation via ALPN. + pub assume_http2: bool, + + /// A file path, relative from the workspace root, to the + /// certificate authority PEM encoded X509 certificate. + #[setting(validate = path_is_required)] + pub cert: FilePath, + + /// The domain name in which to verify the TLS certificate. + pub domain: Option, +} + +/// Configures for both server and client authentication with mTLS. +#[derive(Clone, Config, Debug)] +pub struct RemoteMtlsConfig { + /// If true, assume that the server supports HTTP/2, + /// even if it doesn't provide protocol negotiation via ALPN. + pub assume_http2: bool, + + /// A file path, relative from the workspace root, to the + /// certificate authority PEM encoded X509 certificate. + #[setting(validate = path_is_required)] + pub ca_cert: FilePath, + + /// A file path, relative from the workspace root, to the + /// client's PEM encoded X509 certificate. + #[setting(validate = path_is_required)] + pub client_cert: FilePath, + + /// A file path, relative from the workspace root, to the + /// client's PEM encoded X509 private key. + #[setting(validate = path_is_required)] + pub client_key: FilePath, + + /// The domain name in which to verify the TLS certificate. + pub domain: Option, +} + +/// Configures the remote service, powered by the Bazel Remote Execution API. +#[derive(Clone, Config, Debug)] +pub struct RemoteConfig { + /// Configures the action cache (AC) and content addressable cache (CAS). + #[setting(nested)] + pub cache: RemoteCacheConfig, + + /// The remote host to connect and send requests to. + /// Supports gRPC protocols. + #[setting(validate = validate::not_empty)] + pub host: String, + + /// Connect to the host using server and client authentication with mTLS. + /// This takes precedence over normal TLS. + #[setting(nested)] + pub mtls: Option, + + /// Connect to the host using server-only authentication with TLS. + #[setting(nested)] + pub tls: Option, +} + +impl RemoteConfig { + pub fn is_localhost(&self) -> bool { + self.host.contains("localhost") || self.host.contains("0.0.0.0") + } + + pub fn is_secure(&self) -> bool { + self.tls.is_some() || self.mtls.is_some() + } +} diff --git a/crates/config/src/workspace_config.rs b/crates/config/src/workspace_config.rs index 3aeb4e026a6..8c4bbc33d81 100644 --- a/crates/config/src/workspace_config.rs +++ b/crates/config/src/workspace_config.rs @@ -145,6 +145,10 @@ pub struct WorkspaceConfig { #[setting(nested, validate = validate_projects)] pub projects: WorkspaceProjects, + /// Configures aspects of the remote service. + #[setting(nested, rename = "unstable_remote")] + pub remote: Option, + /// Configures aspects of the task runner (also known as the action pipeline). #[setting(nested)] pub runner: RunnerConfig, diff --git a/crates/hash/Cargo.toml b/crates/hash/Cargo.toml index e418a66cd75..8887fb6f161 100644 --- a/crates/hash/Cargo.toml +++ b/crates/hash/Cargo.toml @@ -11,7 +11,7 @@ publish = false [dependencies] miette = { workspace = true } serde = { workspace = true } -sha2 = "0.10.8" +sha2 = { workspace = true } starbase_utils = { workspace = true, features = ["json"] } tracing = { workspace = true } diff --git a/crates/remote/Cargo.toml b/crates/remote/Cargo.toml new file mode 100644 index 00000000000..0e1e5da0120 --- /dev/null +++ b/crates/remote/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "moon_remote" +version = "0.0.1" +edition = "2021" +publish = false + +[dependencies] +moon_action = { path = "../action" } +moon_common = { path = "../common" } +moon_config = { path = "../config" } +async-trait = { workspace = true } +bazel-remote-apis = "0.10.0" +chrono = { workspace = true } +filetime = "0.2.25" +miette = { workspace = true } +prost-types = "0.13.2" +reqwest = { workspace = true } +rustc-hash = { workspace = true } +scc = { workspace = true } +sha2 = { workspace = true } +starbase_utils = { workspace = true, features = ["glob"] } +thiserror = { workspace = true } +tokio = { workspace = true } +tonic = { version = "0.12.2", default-features = false, features = [ + "channel", + "gzip", + "tls", + "tls-native-roots", +] } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/crates/remote/src/fs_digest.rs b/crates/remote/src/fs_digest.rs new file mode 100644 index 00000000000..80a4df0dd88 --- /dev/null +++ b/crates/remote/src/fs_digest.rs @@ -0,0 +1,242 @@ +// Note: Don't use `starbase_utils::fs` as it spams the logs far too much! + +use bazel_remote_apis::build::bazel::remote::execution::v2::{ + Digest, NodeProperties, OutputDirectory, OutputFile, OutputSymlink, +}; +use chrono::NaiveDateTime; +use moon_common::path::{PathExt, WorkspaceRelativePathBuf}; +use prost_types::Timestamp; +use sha2::{Digest as Sha256Digest, Sha256}; +use starbase_utils::fs::FsError; +use starbase_utils::glob; +use std::path::PathBuf; +use std::{ + fs::{self, Metadata}, + path::Path, + time::{SystemTime, UNIX_EPOCH}, +}; +use tracing::instrument; + +pub struct Blob { + pub bytes: Vec, + pub digest: Digest, +} + +impl Blob { + pub fn new(bytes: Vec) -> Self { + Self { + digest: create_digest(&bytes), + bytes, + } + } +} + +pub fn create_digest(bytes: &[u8]) -> Digest { + let mut hasher = Sha256::default(); + hasher.update(bytes); + + Digest { + hash: format!("{:x}", hasher.finalize()), + size_bytes: bytes.len() as i64, + } +} + +pub fn create_timestamp(time: SystemTime) -> Option { + time.duration_since(UNIX_EPOCH) + .ok() + .map(|duration| Timestamp { + seconds: duration.as_secs() as i64, + nanos: duration.subsec_nanos() as i32, + }) +} + +pub fn create_timestamp_from_naive(time: NaiveDateTime) -> Option { + let utc = time.and_utc(); + + Some(Timestamp { + seconds: utc.timestamp(), + nanos: utc.timestamp_subsec_nanos() as i32, + }) +} + +#[cfg(unix)] +fn is_file_executable(_path: &Path, props: &NodeProperties) -> bool { + props.unix_mode.is_some_and(|mode| mode & 0o111 != 0) +} + +#[cfg(windows)] +fn is_file_executable(path: &Path, _props: &NodeProperties) -> bool { + path.extension().is_some_and(|ext| ext == "exe") +} + +pub fn compute_node_properties(metadata: &Metadata) -> NodeProperties { + let mut props = NodeProperties::default(); + + if let Ok(time) = metadata.modified() { + props.mtime = create_timestamp(time); + } + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + props.unix_mode = Some(metadata.permissions().mode()); + } + + props +} + +#[derive(Default)] +pub struct OutputDigests { + pub blobs: Vec, + pub dirs: Vec, + pub files: Vec, + pub symlinks: Vec, +} + +impl OutputDigests { + pub fn insert_relative_path( + &mut self, + rel_path: WorkspaceRelativePathBuf, + workspace_root: &Path, + ) -> miette::Result<()> { + self.insert_path(rel_path.to_path(workspace_root), workspace_root) + } + + pub fn insert_path(&mut self, abs_path: PathBuf, workspace_root: &Path) -> miette::Result<()> { + // https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto#L1233 + let path_to_string = |inner_path: &Path| { + let outer_path = inner_path.relative_to(workspace_root).unwrap().to_string(); + + if let Some(stripped) = outer_path.strip_prefix('/') { + stripped.to_owned() + } else { + outer_path + } + }; + + let map_read_error = |error| FsError::Read { + path: abs_path.clone(), + error: Box::new(error), + }; + + if abs_path.is_symlink() { + let link = fs::read_link(&abs_path).map_err(map_read_error)?; + let metadata = fs::metadata(&abs_path).map_err(map_read_error)?; + let props = compute_node_properties(&metadata); + + self.symlinks.push(OutputSymlink { + path: path_to_string(&abs_path), + target: path_to_string(&link), + node_properties: Some(props), + }); + } else if abs_path.is_file() { + let bytes = fs::read(&abs_path).map_err(map_read_error)?; + let digest = create_digest(&bytes); + let metadata = fs::metadata(&abs_path).map_err(map_read_error)?; + let props = compute_node_properties(&metadata); + + self.files.push(OutputFile { + path: path_to_string(&abs_path), + digest: Some(digest.clone()), + is_executable: is_file_executable(&abs_path, &props), + contents: vec![], + node_properties: Some(props), + }); + + self.blobs.push(Blob { digest, bytes }); + } else if abs_path.is_dir() { + // TODO use the REAPI directory types + for abs_file in glob::walk_files(abs_path, ["**/*"])? { + self.insert_path(abs_file, workspace_root)?; + } + } + + Ok(()) + } +} + +#[instrument] +pub fn compute_digests_for_outputs( + paths: Vec, + workspace_root: &Path, +) -> miette::Result { + let mut result = OutputDigests::default(); + + for path in paths { + result.insert_relative_path(path, workspace_root)?; + } + + Ok(result) +} + +fn apply_node_properties(path: &Path, props: &NodeProperties) -> miette::Result<()> { + if let Some(mtime) = &props.mtime { + filetime::set_file_mtime( + path, + filetime::FileTime::from_unix_time(mtime.seconds, mtime.nanos as u32), + ) + .map_err(|error| FsError::Write { + path: path.to_owned(), + error: Box::new(error), + })?; + } + + #[cfg(unix)] + if let Some(mode) = &props.unix_mode { + use std::os::unix::fs::PermissionsExt; + + fs::set_permissions(path, fs::Permissions::from_mode(*mode)).map_err(|error| { + FsError::Perms { + path: path.to_path_buf(), + error: Box::new(error), + } + })?; + } + + Ok(()) +} + +pub fn write_output_file( + output_path: PathBuf, + bytes: Vec, + file: &OutputFile, +) -> miette::Result<()> { + fs::write(&output_path, bytes).map_err(|error| FsError::Write { + path: output_path.clone(), + error: Box::new(error), + })?; + + if let Some(props) = &file.node_properties { + apply_node_properties(&output_path, props)?; + } + + Ok(()) +} + +pub fn link_output_file( + from_path: PathBuf, + to_path: PathBuf, + link: &OutputSymlink, +) -> miette::Result<()> { + // Windows requires admin privileges to create soft/hard links, + // so just copy for now... annoying... definitely revisit! + #[cfg(windows)] + fs::copy(&from_path, &to_path).map_err(|error| FsError::Copy { + from: from_path.clone(), + to: to_path.clone(), + error: Box::new(error), + })?; + + #[cfg(unix)] + std::os::unix::fs::symlink(&from_path, &to_path).map_err(|error| FsError::Create { + path: to_path.clone(), + error: Box::new(error), + })?; + + if let Some(props) = &link.node_properties { + apply_node_properties(&to_path, props)?; + } + + Ok(()) +} diff --git a/crates/remote/src/grpc_remote_client.rs b/crates/remote/src/grpc_remote_client.rs new file mode 100644 index 00000000000..5342621393d --- /dev/null +++ b/crates/remote/src/grpc_remote_client.rs @@ -0,0 +1,372 @@ +use crate::fs_digest::Blob; +use crate::grpc_tls::*; +use crate::remote_client::RemoteClient; +use crate::remote_error::RemoteError; +use bazel_remote_apis::build::bazel::remote::execution::v2::{ + action_cache_client::ActionCacheClient, batch_update_blobs_request, + capabilities_client::CapabilitiesClient, compressor, + content_addressable_storage_client::ContentAddressableStorageClient, digest_function, + ActionResult, BatchReadBlobsRequest, BatchUpdateBlobsRequest, Digest, GetActionResultRequest, + GetCapabilitiesRequest, ServerCapabilities, UpdateActionResultRequest, +}; +use moon_common::color; +use moon_config::RemoteConfig; +use std::{error::Error, path::Path}; +use tonic::{ + transport::{Channel, Endpoint}, + Code, +}; +use tracing::{trace, warn}; + +fn map_transport_error(error: tonic::transport::Error) -> RemoteError { + RemoteError::ConnectFailed { + error: Box::new(error), + } +} + +fn map_status_error(error: tonic::Status) -> RemoteError { + match error.source() { + Some(src) => RemoteError::CallFailedViaSource { + error: src.to_string(), + }, + None => RemoteError::CallFailed { + error: Box::new(error), + }, + } +} + +#[derive(Default)] +pub struct GrpcRemoteClient { + channel: Option, + instance_name: String, +} + +#[async_trait::async_trait] +impl RemoteClient for GrpcRemoteClient { + async fn connect_to_host( + &mut self, + config: &RemoteConfig, + workspace_root: &Path, + ) -> miette::Result<()> { + let host = &config.host; + + trace!( + instance = &config.cache.instance_name, + "Connecting to gRPC host {} {}", + color::url(host), + if config.mtls.is_some() { + "(with mTLS)" + } else if config.tls.is_some() { + "(with TLS)" + } else { + "(insecure)" + } + ); + + let mut endpoint = Endpoint::from_shared(host.to_owned()) + .map_err(map_transport_error)? + .user_agent("moon") + .map_err(map_transport_error)? + .keep_alive_while_idle(true); + + if let Some(mtls) = &config.mtls { + endpoint = endpoint + .tls_config(create_mtls_config(mtls, workspace_root)?) + .map_err(map_transport_error)?; + } else if let Some(tls) = &config.tls { + endpoint = endpoint + .tls_config(create_tls_config(tls, workspace_root)?) + .map_err(map_transport_error)?; + } + + if config.is_localhost() { + endpoint = endpoint.origin( + format!( + "{}://localhost", + if config.is_secure() { "https" } else { "http" } + ) + .parse() + .unwrap(), + ); + } + + self.channel = Some(endpoint.connect().await.map_err(map_transport_error)?); + self.instance_name = config.cache.instance_name.clone(); + + Ok(()) + } + + // https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto#L452 + async fn load_capabilities(&self) -> miette::Result { + let mut client = CapabilitiesClient::new(self.channel.clone().unwrap()); + + trace!("Loading remote execution API capabilities from gRPC server"); + + let response = client + .get_capabilities(GetCapabilitiesRequest { + instance_name: self.instance_name.clone(), + }) + .await + .map_err(map_status_error)?; + + Ok(response.into_inner()) + } + + // https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto#L170 + async fn get_action_result(&self, digest: &Digest) -> miette::Result> { + let mut client = ActionCacheClient::new(self.channel.clone().unwrap()); + + trace!(hash = &digest.hash, "Checking for a cached action result"); + + match client + .get_action_result(GetActionResultRequest { + instance_name: self.instance_name.clone(), + action_digest: Some(digest.to_owned()), + inline_stderr: true, + inline_stdout: true, + digest_function: digest_function::Value::Sha256 as i32, + ..Default::default() + }) + .await + { + Ok(response) => { + let result = response.into_inner(); + + trace!( + hash = &digest.hash, + files = result.output_files.len(), + links = result.output_symlinks.len(), + dirs = result.output_directories.len(), + exit_code = result.exit_code, + "Cache hit on action result" + ); + + Ok(Some(result)) + } + Err(status) => { + if matches!(status.code(), Code::NotFound) { + trace!(hash = &digest.hash, "Cache miss on action result"); + + Ok(None) + } else { + Err(map_status_error(status).into()) + } + } + } + } + + // https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto#L193 + async fn update_action_result( + &self, + digest: &Digest, + result: ActionResult, + ) -> miette::Result> { + let mut client = ActionCacheClient::new(self.channel.clone().unwrap()); + + trace!( + hash = &digest.hash, + files = result.output_files.len(), + links = result.output_symlinks.len(), + dirs = result.output_directories.len(), + exit_code = result.exit_code, + "Caching action result" + ); + + match client + .update_action_result(UpdateActionResultRequest { + instance_name: self.instance_name.clone(), + action_digest: Some(digest.to_owned()), + action_result: Some(result), + digest_function: digest_function::Value::Sha256 as i32, + ..Default::default() + }) + .await + { + Ok(response) => { + trace!(hash = &digest.hash, "Cached action result"); + + Ok(Some(response.into_inner())) + } + Err(status) => { + let code = status.code(); + + if matches!(code, Code::InvalidArgument | Code::FailedPrecondition) { + warn!( + code = ?code, + "Failed to cache action result: {}", + status.message() + ); + + Ok(None) + } else if matches!(code, Code::ResourceExhausted) { + warn!( + code = ?code, + "Remote service is out of storage space: {}", + status.message() + ); + + Ok(None) + } else { + Err(map_status_error(status).into()) + } + } + } + } + + // https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto#L403 + async fn batch_read_blobs( + &self, + digest: &Digest, + blob_digests: Vec, + ) -> miette::Result> { + let mut client = ContentAddressableStorageClient::new(self.channel.clone().unwrap()); + + trace!( + hash = &digest.hash, + "Downloading {} output blobs", + blob_digests.len() + ); + + let response = match client + .batch_read_blobs(BatchReadBlobsRequest { + acceptable_compressors: vec![compressor::Value::Identity as i32], + instance_name: self.instance_name.clone(), + digests: blob_digests, + digest_function: digest_function::Value::Sha256 as i32, + }) + .await + { + Ok(res) => res, + Err(status) => { + return if matches!(status.code(), Code::InvalidArgument) { + warn!( + hash = &digest.hash, + "Attempted to download more blobs than the allowed limit" + ); + + Ok(vec![]) + } else { + Err(map_status_error(status).into()) + }; + } + }; + + let mut blobs = vec![]; + let mut total_count = 0; + + for download in response.into_inner().responses { + if let Some(status) = download.status { + if status.code != 0 { + warn!( + details = ?status.details, + "Failed to download blob: {}", + status.message + ); + } + } + + if let Some(digest) = download.digest { + blobs.push(Blob { + digest, + bytes: download.data, + }); + } + + total_count += 1; + } + + trace!( + hash = &digest.hash, + "Downloaded {} of {} output blobs", + blobs.len(), + total_count + ); + + Ok(blobs) + } + + // https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto#L379 + async fn batch_update_blobs( + &self, + digest: &Digest, + blobs: Vec, + ) -> miette::Result>> { + let mut client = ContentAddressableStorageClient::new(self.channel.clone().unwrap()); + + trace!( + hash = &digest.hash, + "Uploading {} output blobs", + blobs.len() + ); + + let response = match client + .batch_update_blobs(BatchUpdateBlobsRequest { + instance_name: self.instance_name.clone(), + requests: blobs + .into_iter() + .map(|blob| batch_update_blobs_request::Request { + digest: Some(blob.digest), + data: blob.bytes, + compressor: compressor::Value::Identity as i32, + }) + .collect(), + digest_function: digest_function::Value::Sha256 as i32, + }) + .await + { + Ok(res) => res, + Err(status) => { + let code = status.code(); + + return if matches!(code, Code::InvalidArgument) { + warn!( + hash = &digest.hash, + "Attempted to upload more blobs than the allowed limit" + ); + + Ok(vec![]) + } else if matches!(code, Code::ResourceExhausted) { + warn!( + code = ?code, + "Remote service exhausted resource: {}", + status.message() + ); + + Ok(vec![]) + } else { + Err(map_status_error(status).into()) + }; + } + }; + + let mut digests = vec![]; + let mut uploaded_count = 0; + + for upload in response.into_inner().responses { + if let Some(status) = upload.status { + if status.code != 0 { + warn!( + details = ?status.details, + "Failed to upload blob: {}", + status.message + ); + } + } + + if upload.digest.is_some() { + uploaded_count += 1; + } + + digests.push(upload.digest); + } + + trace!( + hash = &digest.hash, + "Uploaded {} of {} output blobs", + uploaded_count, + digests.len() + ); + + Ok(digests) + } +} diff --git a/crates/remote/src/grpc_tls.rs b/crates/remote/src/grpc_tls.rs new file mode 100644 index 00000000000..b0c04437018 --- /dev/null +++ b/crates/remote/src/grpc_tls.rs @@ -0,0 +1,75 @@ +use moon_config::{RemoteMtlsConfig, RemoteTlsConfig}; +use starbase_utils::fs; +use std::path::Path; +use tonic::transport::{Certificate, ClientTlsConfig, Identity}; +use tracing::trace; + +// TLS server: +// - `*.pem` file +// - `*.key` file +// TLS client: +// - `*.pem` file +// - domain name +// mTLS client: +// - cert authority `*.pem` file +// - client `*.pem` file +// - client `*.key` file +// - domain name + +// https://github.com/hyperium/tonic/blob/master/examples/src/tls/client.rs +pub fn create_tls_config( + config: &RemoteTlsConfig, + workspace_root: &Path, +) -> miette::Result { + let cert = workspace_root.join(&config.cert); + + trace!( + cert = ?cert, + domain = &config.domain, + http2 = config.assume_http2, + "Configuring TLS", + ); + + let mut tls = ClientTlsConfig::new() + .with_enabled_roots() + .ca_certificate(Certificate::from_pem(fs::read_file_bytes(cert)?)); + + if let Some(domain) = &config.domain { + tls = tls.domain_name(domain.to_owned()); + } + + Ok(tls.assume_http2(config.assume_http2)) +} + +// https://github.com/hyperium/tonic/blob/master/examples/src/tls_client_auth/client.rs +pub fn create_mtls_config( + config: &RemoteMtlsConfig, + workspace_root: &Path, +) -> miette::Result { + let client_cert = workspace_root.join(&config.client_cert); + let client_key = workspace_root.join(&config.client_key); + let ca_cert = workspace_root.join(&config.ca_cert); + + trace!( + client_cert = ?client_cert, + client_key = ?client_key, + ca_cert = ?ca_cert, + domain = &config.domain, + http2 = config.assume_http2, + "Configuring mTLS", + ); + + let mut mtls = ClientTlsConfig::new() + .with_enabled_roots() + .ca_certificate(Certificate::from_pem(fs::read_file_bytes(ca_cert)?)) + .identity(Identity::from_pem( + fs::read_file_bytes(client_cert)?, + fs::read_file_bytes(client_key)?, + )); + + if let Some(domain) = &config.domain { + mtls = mtls.domain_name(domain.to_owned()); + } + + Ok(mtls.assume_http2(config.assume_http2)) +} diff --git a/crates/remote/src/lib.rs b/crates/remote/src/lib.rs new file mode 100644 index 00000000000..25e0cf25959 --- /dev/null +++ b/crates/remote/src/lib.rs @@ -0,0 +1,20 @@ +mod fs_digest; +mod grpc_remote_client; +mod grpc_tls; +mod remote_client; +mod remote_error; +mod remote_service; + +pub use bazel_remote_apis::build::bazel::remote::execution::v2::Digest; +pub use fs_digest::*; +pub use remote_error::*; +pub use remote_service::*; + +// TODO: +// - HTTP(S) client +// - Other digest functions besides sha256 +// - Compression formats (only identity right now) +// - Proper error handling +// - Directory blob types +// - Write/read bytestream for large blobs +// - TLS/mTLS issues diff --git a/crates/remote/src/remote_client.rs b/crates/remote/src/remote_client.rs new file mode 100644 index 00000000000..c83a9fbbd55 --- /dev/null +++ b/crates/remote/src/remote_client.rs @@ -0,0 +1,37 @@ +use crate::fs_digest::Blob; +use bazel_remote_apis::build::bazel::remote::execution::v2::{ + ActionResult, Digest, ServerCapabilities, +}; +use moon_config::RemoteConfig; +use std::path::Path; + +#[async_trait::async_trait] +pub trait RemoteClient: Send + Sync { + async fn connect_to_host( + &mut self, + config: &RemoteConfig, + workspace_root: &Path, + ) -> miette::Result<()>; + + async fn load_capabilities(&self) -> miette::Result; + + async fn get_action_result(&self, digest: &Digest) -> miette::Result>; + + async fn update_action_result( + &self, + digest: &Digest, + result: ActionResult, + ) -> miette::Result>; + + async fn batch_read_blobs( + &self, + digest: &Digest, + blob_digests: Vec, + ) -> miette::Result>; + + async fn batch_update_blobs( + &self, + digest: &Digest, + blobs: Vec, + ) -> miette::Result>>; +} diff --git a/crates/remote/src/remote_error.rs b/crates/remote/src/remote_error.rs new file mode 100644 index 00000000000..5d981cdea0e --- /dev/null +++ b/crates/remote/src/remote_error.rs @@ -0,0 +1,31 @@ +use miette::Diagnostic; +use thiserror::Error; + +#[derive(Error, Debug, Diagnostic)] +pub enum RemoteError { + #[diagnostic(code(remote::grpc::call_failed))] + #[error("Failed to make gRPC call.")] + CallFailed { + #[source] + error: Box, + }, + + #[diagnostic(code(remote::grpc::call_failed))] + #[error("Failed to make gRPC call: {error}")] + CallFailedViaSource { error: String }, + + #[diagnostic(code(remote::grpc::connect_failed))] + #[error("Failed to connect to gRPC host.")] + ConnectFailed { + #[source] + error: Box, + }, + + #[diagnostic(code(remote::http::no_support))] + #[error("The HTTP based remote service is currently not supported, use gRPC instead.")] + NoHttpClient, + + #[diagnostic(code(remote::unsupported_protocol))] + #[error("Unknown remote host protocol, only gRPC is supported.")] + UnknownHostProtocol, +} diff --git a/crates/remote/src/remote_service.rs b/crates/remote/src/remote_service.rs new file mode 100644 index 00000000000..367b2f0ca02 --- /dev/null +++ b/crates/remote/src/remote_service.rs @@ -0,0 +1,553 @@ +use crate::fs_digest::*; +use crate::grpc_remote_client::GrpcRemoteClient; +use crate::remote_client::RemoteClient; +use crate::RemoteError; +use bazel_remote_apis::build::bazel::remote::execution::v2::{ + digest_function, ActionResult, Digest, ExecutedActionMetadata, ServerCapabilities, +}; +use miette::IntoDiagnostic; +use moon_action::Operation; +use moon_common::{color, is_ci}; +use moon_config::RemoteConfig; +use rustc_hash::FxHashMap; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock}; +use std::time::SystemTime; +use tokio::sync::RwLock; +use tokio::task::{JoinHandle, JoinSet}; +use tracing::{debug, info, instrument, trace, warn}; + +static INSTANCE: OnceLock> = OnceLock::new(); + +pub struct RemoteService { + pub config: RemoteConfig, + pub workspace_root: PathBuf, + + action_results: scc::HashMap, + cache_enabled: bool, + capabilities: ServerCapabilities, + client: Arc>, + upload_requests: Arc>>>, +} + +impl RemoteService { + pub fn session() -> Option> { + INSTANCE.get().cloned() + } + + #[instrument] + pub async fn connect(config: &RemoteConfig, workspace_root: &Path) -> miette::Result<()> { + if is_ci() && config.is_localhost() { + debug!( + host = &config.host, + "Remote service is configured with a localhost endpoint, but we are in a CI environment; disabling service", + ); + + return Ok(()); + } + + info!( + docs = "https://github.com/bazelbuild/remote-apis", + "Remote service, powered by the Bazel Remote Execution API, is currently unstable" + ); + info!("Please report any issues to GitHub or Discord"); + + let mut client = + if config.host.starts_with("http://") || config.host.starts_with("https://") { + return Err(RemoteError::NoHttpClient.into()); + } else if config.host.starts_with("grpc://") || config.host.starts_with("grpcs://") { + Box::new(GrpcRemoteClient::default()) + } else { + return Err(RemoteError::UnknownHostProtocol.into()); + }; + + client.connect_to_host(config, workspace_root).await?; + + let mut instance = Self { + action_results: scc::HashMap::default(), + capabilities: client.load_capabilities().await?, + cache_enabled: false, + client: Arc::new(client), + config: config.to_owned(), + upload_requests: Arc::new(RwLock::new(vec![])), + workspace_root: workspace_root.to_owned(), + }; + + instance.validate_capabilities()?; + + let _ = INSTANCE.set(Arc::new(instance)); + + Ok(()) + } + + pub fn validate_capabilities(&mut self) -> miette::Result<()> { + let host = &self.config.host; + let mut enabled = true; + + if let Some(cap) = &self.capabilities.cache_capabilities { + let sha256_fn = digest_function::Value::Sha256 as i32; + + if !cap.digest_functions.contains(&sha256_fn) { + enabled = false; + + warn!( + host, + "Remote service does not support SHA256 digests, which is required by moon" + ); + } + + if let Some(ac_cap) = &cap.action_cache_update_capabilities { + if !ac_cap.update_enabled { + enabled = false; + + warn!( + host, + "Remote service does not support caching of actions, which is required by moon" + ); + } + } + } else { + enabled = false; + + warn!( + host, + "Remote service does not support caching, disabling in moon" + ); + } + + self.cache_enabled = enabled; + + // TODO check low_api_version/high_api_version + + Ok(()) + } + + pub fn get_max_batch_size(&self) -> i64 { + self.capabilities + .cache_capabilities + .as_ref() + .and_then(|cap| { + if cap.max_batch_total_size_bytes == 0 { + None + } else { + Some(cap.max_batch_total_size_bytes) + } + }) + // grpc limit: 4mb - buffer + .unwrap_or(4194304 - (1024 * 10)) + } + + #[instrument(skip(self))] + pub async fn is_operation_cached(&self, digest: &Digest) -> miette::Result { + if !self.cache_enabled { + return Ok(false); + } + + if self.action_results.contains_async(&digest.hash).await { + return Ok(true); + } + + if let Some(result) = self.client.get_action_result(digest).await? { + let _ = self + .action_results + .insert_async(digest.hash.clone(), result) + .await; + + return Ok(true); + } + + Ok(false) + } + + #[instrument(skip(self, operation))] + pub async fn save_operation( + &self, + digest: &Digest, + operation: &Operation, + ) -> miette::Result<()> { + if !self.cache_enabled || operation.has_failed() { + return Ok(()); + } + + let operation_label = operation.label().to_owned(); + + debug!( + hash = &digest.hash, + "Caching {} operation", + color::muted_light(&operation_label) + ); + + let result = self.create_action_result_from_operation(operation, None)?; + let digest = digest.to_owned(); + let client = Arc::clone(&self.client); + + self.upload_requests + .write() + .await + .push(tokio::spawn(async move { + if let Err(error) = client.update_action_result(&digest, result).await { + warn!( + hash = &digest.hash, + "Failed to cache {} operation: {}", + color::muted_light(operation_label), + color::muted_light(error.to_string()), + ); + } + })); + + Ok(()) + } + + #[instrument(skip(self, operation, outputs))] + pub async fn save_operation_with_outputs( + &self, + digest: &Digest, + operation: &Operation, + mut outputs: OutputDigests, + ) -> miette::Result<()> { + if !self.cache_enabled || operation.has_failed() { + return Ok(()); + } + + let operation_label = operation.label().to_owned(); + + debug!( + hash = &digest.hash, + "Caching {} operation with outputs", + color::muted_light(&operation_label) + ); + + let mut result = self.create_action_result_from_operation(operation, Some(&mut outputs))?; + result.output_files = outputs.files; + result.output_symlinks = outputs.symlinks; + result.output_directories = outputs.dirs; + + let digest = digest.to_owned(); + let client = Arc::clone(&self.client); + let max_size = self.get_max_batch_size(); + + self.upload_requests + .write() + .await + .push(tokio::spawn(async move { + if !outputs.blobs.is_empty() { + if let Some(metadata) = &mut result.execution_metadata { + metadata.output_upload_start_timestamp = + create_timestamp(SystemTime::now()); + } + + let upload_result = batch_upload_blobs( + client.clone(), + digest.clone(), + outputs.blobs, + max_size as usize, + ) + .await; + + if upload_result.is_err() || upload_result.is_ok_and(|res| !res) { + return; + } + + if let Some(metadata) = &mut result.execution_metadata { + metadata.output_upload_completed_timestamp = + create_timestamp(SystemTime::now()); + } + } + + if let Err(error) = client.update_action_result(&digest, result).await { + warn!( + hash = &digest.hash, + "Failed to cache {} operation: {}", + color::muted_light(operation_label), + color::muted_light(error.to_string()), + ); + } + })); + + Ok(()) + } + + #[instrument(skip(self, operation))] + pub async fn restore_operation( + &self, + digest: &Digest, + operation: &mut Operation, + ) -> miette::Result<()> { + if !self.cache_enabled { + return Ok(()); + } + + let Some(result) = self.action_results.get_async(&digest.hash).await else { + return Ok(()); + }; + + let operation_label = operation.label().to_owned(); + let has_outputs = !result.output_files.is_empty() + || !result.output_symlinks.is_empty() + || !result.output_directories.is_empty(); + + if has_outputs { + debug!( + hash = &digest.hash, + "Restoring {} operation with outputs", + color::muted_light(&operation_label) + ); + } else { + debug!( + hash = &digest.hash, + "Restoring {} operation", + color::muted_light(&operation_label) + ); + } + + if let Some(output) = operation.get_output_mut() { + output.exit_code = Some(result.exit_code); + + if !result.stderr_raw.is_empty() { + output.set_stderr(String::from_utf8_lossy(&result.stderr_raw).into()); + } + + if !result.stdout_raw.is_empty() { + output.set_stdout(String::from_utf8_lossy(&result.stdout_raw).into()); + } + } + + batch_download_blobs( + self.client.clone(), + digest, + &result, + &self.workspace_root, + self.get_max_batch_size() as usize, + ) + .await?; + + debug!( + hash = &digest.hash, + "Restored {} operation", + color::muted_light(&operation_label) + ); + + Ok(()) + } + + #[instrument(skip(self))] + pub async fn wait_for_requests(&self) { + let mut requests = self.upload_requests.write().await; + + for future in requests.drain(0..) { + // We can ignore the errors because we handle them in + // the tasks above by logging to the console + let _ = future.await; + } + } + + fn create_action_result_from_operation( + &self, + operation: &Operation, + outputs: Option<&mut OutputDigests>, + ) -> miette::Result { + let mut result = ActionResult { + execution_metadata: Some(ExecutedActionMetadata { + worker: "moon".into(), + execution_start_timestamp: create_timestamp_from_naive(operation.started_at), + execution_completed_timestamp: operation + .finished_at + .and_then(create_timestamp_from_naive), + ..Default::default() + }), + ..Default::default() + }; + + if let Some(exec) = operation.get_output() { + result.exit_code = exec.exit_code.unwrap_or_default(); + + if let Some(outputs) = outputs { + if let Some(stderr) = &exec.stderr { + let blob = Blob::new(stderr.as_bytes().to_owned()); + + result.stderr_digest = Some(blob.digest.clone()); + outputs.blobs.push(blob); + } + + if let Some(stdout) = &exec.stdout { + let blob = Blob::new(stdout.as_bytes().to_owned()); + + result.stdout_digest = Some(blob.digest.clone()); + outputs.blobs.push(blob); + } + } + } + + Ok(result) + } +} + +async fn batch_upload_blobs( + client: Arc>, + digest: Digest, + blobs: Vec, + max_size: usize, +) -> miette::Result { + let blob_groups = partition_into_groups(blobs, max_size, |blob| blob.bytes.len()); + + if blob_groups.is_empty() { + return Ok(false); + } + + let group_total = blob_groups.len(); + let mut set = JoinSet::default(); + + for (group_index, group) in blob_groups.into_iter() { + let client = Arc::clone(&client); + let digest = digest.to_owned(); + + if group_total > 1 { + trace!( + hash = &digest.hash, + blobs = group.items.len(), + size = group.size, + max_size, + "Batching blobs upload (group {} of {})", + group_index + 1, + group_total + ); + } + + set.spawn(async move { + if let Err(error) = client.batch_update_blobs(&digest, group.items).await { + warn!( + hash = &digest.hash, + group = group_index + 1, + "Failed to upload blobs: {}", + color::muted_light(error.to_string()), + ); + + return false; + } + + true + }); + } + + let results = set.join_all().await; + + Ok(results.into_iter().all(|passed| passed)) +} + +async fn batch_download_blobs( + client: Arc>, + digest: &Digest, + result: &ActionResult, + workspace_root: &Path, + max_size: usize, +) -> miette::Result<()> { + let mut file_map = FxHashMap::default(); + let mut digests = vec![]; + + // TODO support directories + for file in &result.output_files { + if let Some(digest) = &file.digest { + file_map.insert(&digest.hash, file); + digests.push(digest.to_owned()); + } + } + + let digest_groups = partition_into_groups(digests, max_size, |dig| dig.size_bytes as usize); + + if digest_groups.is_empty() { + return Ok(()); + } + + let group_total = digest_groups.len(); + let mut set = JoinSet::>>::default(); + + for (group_index, group) in digest_groups.into_iter() { + let client = Arc::clone(&client); + let digest = digest.to_owned(); + + if group_total > 1 { + trace!( + hash = &digest.hash, + blobs = group.items.len(), + size = group.size, + max_size, + "Batching blobs download (group {} of {})", + group_index + 1, + group_total + ); + } + + set.spawn(async move { client.batch_read_blobs(&digest, group.items).await }); + } + + while let Some(res) = set.join_next().await { + for blob in res.into_diagnostic()?? { + if let Some(file) = file_map.get(&blob.digest.hash) { + write_output_file(workspace_root.join(&file.path), blob.bytes, file)?; + } + } + } + + // Create symlinks after blob files have been written, + // as the link target may reference one of these outputs + for link in &result.output_symlinks { + link_output_file( + workspace_root.join(&link.target), + workspace_root.join(&link.path), + link, + )?; + } + + Ok(()) +} + +struct Partition { + pub items: Vec, + pub size: usize, +} + +fn partition_into_groups( + items: Vec, + max_size: usize, + get_size: impl Fn(&T) -> usize, +) -> BTreeMap> { + let mut groups = BTreeMap::>::default(); + + for item in items { + let item_size = get_size(&item); + let mut index_to_use = -1; + + if item_size >= max_size { + warn!( + size = item_size, + max_size, + "Encountered a blob larger than the max size; this is currently not supported until we support the ByteStream API; aborting" + ); + + return BTreeMap::default(); + } + + // Try and find a partition that this item can go into + for (index, group) in &groups { + if group.size + item_size < max_size { + index_to_use = *index; + break; + } + } + + // If no partition available, create a new one + if index_to_use == -1 { + index_to_use = groups.len() as i32; + } + + let entry = groups.entry(index_to_use).or_insert_with(|| Partition { + items: vec![], + size: 0, + }); + entry.size += item_size; + entry.items.push(item); + } + + groups +} diff --git a/crates/remote/tests/__fixtures__/certs-local/README.md b/crates/remote/tests/__fixtures__/certs-local/README.md new file mode 100644 index 00000000000..dc295bbe913 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/README.md @@ -0,0 +1 @@ +These certs are generated from the `scripts/data/generateCerts.sh` file! diff --git a/crates/remote/tests/__fixtures__/certs-local/ca.crt b/crates/remote/tests/__fixtures__/certs-local/ca.crt new file mode 100644 index 00000000000..2d235085aa7 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/ca.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIFCzCCAvOgAwIBAgIUHSBLheiEsREQXE2y7k0be9x9Z+gwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI0MTEyMzAyMjkzOFoYDzMwMDUw +MTI1MDIyOTM4WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCiQHwtgPknqDqAd4UYa9oZeTtBYpeTvMmNce5+RUnZ +f9cXJYYEgK4lOeM1yJuvBC3jvrzGco3tZ7qZn3qBfpGTENID6QZxKowCJwFMsrjz +rwXXDGCkI+Lk6Ztz3ssH9g9Py9nrt9Q292o1fBgNeO5LF4eiy425IutjFETYeY6L +cJlSoRFl4ehz85jWG/VFMeqm52CQ9Y1x4K+VtPU7zmR93SlEmLSQcsVdKDTEPKNh +4SDzFEA5q5zssYxUAcMO8pWC2wZUT8PgnoMRCXl3JWp/0BhfDlyrarz/QvB9qy1p +WOWb07DX+wdNXyxGuZZaMyJzwVZPBrVxgLHRt72podXM8rJ5wzJlwsaTEPCL2sEJ +V4mHXuOYOBevxJnf0TMNP1FvUIwQCvZXVCUS6pQQ5KCC512b7Vof84FQ93AnBucI +OZml59mBHTHqPV7qldbcnljH9Si+cwlxsAj4BjK1qDJBvWpQeo1YqN1P+y4p8Xhb +r4A+mE0Up1BkVp9yyvLN4AX4F/FTLfnrGq4woGpnihAz8+OoB2eciXFkycnqSDh2 +GpfbaH0TveFQ+A9wInbFdvzcbp8hAvhCiJUl/Rez1xCZrMIH9llcI9/OrqGiFP6Z +b1BzTZlN9QwToOshCBPc81lFTChqMyj1+4D91x/89Jru6Ky1gpTyyA3Y4c/B60EL +mwIDAQABo1MwUTAdBgNVHQ4EFgQUdo2ZshddYrN+ZEC2me7xbbm6z18wHwYDVR0j +BBgwFoAUdo2ZshddYrN+ZEC2me7xbbm6z18wDwYDVR0TAQH/BAUwAwEB/zANBgkq +hkiG9w0BAQsFAAOCAgEAAckUB1ZOnitfGriLAG/sZeZWNuz/xRwd4NOQFf1vd1ea +wmF3W9VzkPWiB3xEj4y/4TM7TFzuP/qSexndytphr7NiQGwcHdBrpaEikRmIJokZ +xAiGLkM00zgdGaFHBYj9in7QdNTGq3fkcfhiQ0JVN/PWuW0EoF0oThNWeSdFZwJW +tQ6Fmhfd7GmWlEkilrgd95pNZDQXhhA2dwKCTLVFM0AeehpIHm0cKWzs5k0VV8+R +gnVDT58U2XXyElHjPG8qtA2pbSRoEjlH6c8OhbUugcFTwm7WI2cZ/f0K595BVS1/ +FzmKbOdjz6P45uV90cjbBUpiaWcCmo8u3lSQWaRc3LH06wKkAr1Ci4vbXcTOyKhU +zVaCR1LL2Ij9u9QFikiCdskIY1rksJ0v+95KUsrlcUWWAZKtfwuAWdZH23LMom/Q +sUvWItYGbXmhsPXccAny3e/sqzP/w7yuEaeZDV0C4b7QDsSQO/mFf4weXfGyG/p1 +EGwTB2nfZ7h9KyD5R/HcduauR27udP/cJs/xCUxVHqF+wo5O2lTJfcZ6tnS79xpx +6K+ZhaunYsUaig9WVhMMiQx2PSzi5yXG6DeII102Be3ORelnUdRJEMwjDZxRJaH4 +Tor2PueQzJ5nIGxNuKHy3ZRQehJiKmXutzPgJBMj/WsCoMekidBIF24Tlhc/Y5M= +-----END CERTIFICATE----- diff --git a/crates/remote/tests/__fixtures__/certs-local/ca.key b/crates/remote/tests/__fixtures__/certs-local/ca.key new file mode 100644 index 00000000000..105d733ca27 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/ca.key @@ -0,0 +1,54 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIJpDBWBgkqhkiG9w0BBQ0wSTAxBgkqhkiG9w0BBQwwJAQQFZW1akSKquHsGxUf +7S8cMwICCAAwDAYIKoZIhvcNAgkFADAUBggqhkiG9w0DBwQIp2xCj/cyKjMEgglI +KwYzDVDHKJkLQ+i5Ggds9v7bQvtX1ChNf9lbvaD2/WSTWaYdCSONqtgviCqTfpnp +71bItC4yp7IO4B0SVIQVA8wyRXO7fUw3zMThyra47tShqPHb9jfG51N2bw0PgO9z +ecGv43yTcXAKJbXMX8P5Q6okhUiVLPM0GPiuGGZ2TkEaO6aGP81/YqnYn3GCHauF +idlbZWi4i96hvhHk5W/nL5NKmhzdANe/vLiDvFuoxjaurkcmYcIvnK3Lpy6pGqFm +m+/NFR2ftNsKtm6QKEQxr38r6rNz+ncCR45c2jQiGMC3wGLUuXtEMu8pgdRcYgRP +EX1ALwvyj1WXKFFJp9tSpe6MCXJHcHLZhiPiT/YBQ6GCSgP4tqblEnFA9yniTz6i +a21Ss1DPFesAOaQj3/WmU8xejzKM+/zle4VnRn4lK7CaOSl08J25OoBnJAiELHWo +aSJVUfVfWjGUETKEodlyXB2OkzxFuJb0bK+XxxkBkfHB7SCEJk6l/5CTESGS1hlW +KKRi3Xs6m90FrHvJutfKR0eKjHwxyuiYlf82Rt+APDgtRlGxRc0TKysYXnB+NL5Z +28yG8/1ynDSlirtfnpXxB3qWN7hKii/HNuU4v5gzPpFjaXUtgvkDOrmbVXPG0h9R +YJcZbgI2ZasId06C0H6/mT1WhWy/KURzTJTClfNQFjrb/lXXKMhG00L5wczB1dtp +m49CgOLNIVdcfwUs31tStUdAyfAr7hc6Ui9C5j47KVBqt7c6rRoVnGYTDemm16Zl +EIxy0KJt2Gf/uvKuxkelL3Oy3ogTTyAOS1QPy6nUYfqJG0HNmX2UpZS2Wx4mwUcO +22kTRrvYGAKb/3zOYxXsNwzBF8nW5nqCABc/+rzY0jU/f2HMpNeIJMf7OPA4av1a +m4tmseGdCucqg240MQaz96mYA6ld7+7Wp1AsLFv3O4XGj3CUeq2MFV3qyKg6GZAR +VomGLZUgYNWxF/kQ669HNt/Nr1YF3LkKCrgCVghKkMQ/Hmmlgy6hWqrr/jIcZ8Zh +icsHe9IWkLwwWvpJKarW0X9slp/BJbCpd9aJ8IYj9ohmhVid4SqjSNozMDlT7CZ7 ++XDqQ3Oe+wBYKhtvi99l2GsTJi/l/87scydsxKPMciR66UDmbSw3T84+8tNhq+1Y +I0o84zVsfgymG/wtk/LXcKAUBsh6n5HoIImWAL6vuZfYzytd6lwjA+GwGG7LJaqL +pP6QmGrF6+yU+sLkTFsaP7CM2t06bUMMJnmwRNA6PpOOQ+/mCwKv50kVmkTa8aBr +g8hhe9/DHg/LEql3tfHt5+eRRCNa9wI8iKHeKRhKl8uqWgCRVHkccvvU6Qs8xx94 +sNB3r9e4Blb2MU6clDx7jrBRi39/jQQxNJoNBhMwkua3h2mFQwT7rG8t7mYdeQQu ++UIHPyVhTHLXeo8h3kC770HNkCiQo+REIlVjhOJRsSJ0+s7hS5XX95nDKWOZyNA6 +9ct49T1rH1JBy6UKSSn5a7TuvFdIOKlRV9dYC7109s+hTuGXfS75dnb16LsSkVMe +w98mPxwTzG9VC9QpBbSs112ZFr6ES2hcUZknlMpxxDeZ/BcD7OQc8IUrm2Erzvwk ++qX9PWGqq7cxeIOVmnloDYF4A2RkO4toA2zXUdRwWoj/ll7yopP4NfAaCPaH4RUz +Otu4UsPFh+jc4Ku0VRsRb9/AG2RqoNhwVbzUCaqzXQepXi+JcliPlHDnbh5oUYlE +XuZGd03Zhb0HTpdyZWhhX1LE2TBnSYBEYJhl6ct0HKnIf97kayM6sCzv9JYvUbcp +vTHdqIrd2c9mJVS5rdBIR6dbjpYj7HvR3KZzT3sc84b0pW6CRxfYVNrgMG+d9zny +hHt6O5b1l5PuW2PeOPDsXdnSiL6BQF6txA122UPNbjRTxsKticqI+0ibM88dsTEX +n4GPCpXoSi4w+XkISa38EqgD/f0Zk09OlPQUDAqTBjTix0rKC7/BMPupcuXFVUXD +zpQ8tZdwV/xy1JBM2pn1oBMkw/Yt44x7StNPV5V6ehuIJ3PzFfo5vjy2eZHDqlZh +QGMZIxFIKhiKqMSNXEwDHOei9kDAKH2gqlxkIkocIgJxb9UDvcu06b1ysoZZzvWt +hdCJmmt/Q+BebrBN+8mBfXYNjYp57oIMoEtwkC4M4zfcxd/Lo3p/EMUgouVlyFNr +YMfjlsJ90gYpZQ3BqfAd+4cTP0+9IdGFpeA+MYVPdKP7OX1KWYgEmj0+pocPJnHR +OBF/uKH3qxLVp7azWrVBB5RrlyFrqEiAl8LTmo3RBz5j6rrNZcaquZR/VCKsL+CP +4INQaDflk68NSCKAmarVuIsnS3qcTgomEHjK7nOy85rZmBvBlpt48Bqbo5XpXanF +XMdaZgJmLVdcwxx550460HsqI0NWx4EybV0UlLpMEyKUmLNlF+fzIB6rABn4eC/T +DeuFHR3pgMDIoI5uhAYiMg5vxztGWHarr3scVbR3JupNrkf/YBaMKPq1EdJrBTif +Wk5p2XU4YrTCeUHX8uKaeV9RUzVRwib29sJbxTOQfx3UbLXILczvp90DzImuYrz1 +H5uF94ZUaOX55tgGkZrYu/u/TkC7PPvU7vHdt9REAMFS8y3CNkU+oiOMLlQxW9k4 +Ru/sajoJJpsL5DIKjWO57lu76s7MeD/bmghIRsebodZnRZuTumCQph1z2Dt4Xsm4 +T/8UGmcaitYHz7gNEglkgN5iV6glLhnTdTmJP6GFRgzXyXmKrENcEuEeknhmnjzJ +JnMnagN0WXDrF/WV4Y3fS1vlcrpwfT3Ip8OlfssuhKTYklpjoJj/InmE+tW3KDh4 +5zWfDet5tera1EltPIPTeudwwtr4RlmadJhwrJRtSlViLCSQ0QvJNL0Z38hKnhst +y2tf1LYGohW0VMmSVn55JlDbi3X1aQ7aLAajKCJMsuqGITh0chLRTI9MjlNH+BHT +O4RuV3jsxaKTiGbX21tjqjUfbzvCRQsJIs2O4r1O6TrDxJ2b/bucumeml6SiKZPr +EtLh5ucg8u/YKvp6sxfFq0Deu+XpQpE8vLI6sWSGLzFMtppOdm62TP+5LKP5EK+d +/PkkrOYCPYRhAtC0bk8vRrEhuRwEgK8WTSCS1y+QHifbxfJumubHrDz7lyk9+XvZ +KsbVKZtdM9TK+URrCietawTnWchN+S2I +-----END ENCRYPTED PRIVATE KEY----- diff --git a/crates/remote/tests/__fixtures__/certs-local/ca.pem b/crates/remote/tests/__fixtures__/certs-local/ca.pem new file mode 100644 index 00000000000..a71384cc9d9 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/ca.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCiQHwtgPknqDqA +d4UYa9oZeTtBYpeTvMmNce5+RUnZf9cXJYYEgK4lOeM1yJuvBC3jvrzGco3tZ7qZ +n3qBfpGTENID6QZxKowCJwFMsrjzrwXXDGCkI+Lk6Ztz3ssH9g9Py9nrt9Q292o1 +fBgNeO5LF4eiy425IutjFETYeY6LcJlSoRFl4ehz85jWG/VFMeqm52CQ9Y1x4K+V +tPU7zmR93SlEmLSQcsVdKDTEPKNh4SDzFEA5q5zssYxUAcMO8pWC2wZUT8PgnoMR +CXl3JWp/0BhfDlyrarz/QvB9qy1pWOWb07DX+wdNXyxGuZZaMyJzwVZPBrVxgLHR +t72podXM8rJ5wzJlwsaTEPCL2sEJV4mHXuOYOBevxJnf0TMNP1FvUIwQCvZXVCUS +6pQQ5KCC512b7Vof84FQ93AnBucIOZml59mBHTHqPV7qldbcnljH9Si+cwlxsAj4 +BjK1qDJBvWpQeo1YqN1P+y4p8Xhbr4A+mE0Up1BkVp9yyvLN4AX4F/FTLfnrGq4w +oGpnihAz8+OoB2eciXFkycnqSDh2GpfbaH0TveFQ+A9wInbFdvzcbp8hAvhCiJUl +/Rez1xCZrMIH9llcI9/OrqGiFP6Zb1BzTZlN9QwToOshCBPc81lFTChqMyj1+4D9 +1x/89Jru6Ky1gpTyyA3Y4c/B60ELmwIDAQABAoICADRVnnhC+JrNDYmwg5K7/w5m +ZzGQ7plttIlWLXo3OTnZnXRO2yqKoyFIybvoaCcMrwyd0mAkRRlFcw/oO+iW/be1 +Hji6qiRId/6dvDKUF0oqszSJPobTOHVj0IuQWmbH/Gpds33vvpi4N17nFw8Jabt4 +7HwbMix8UfaUbBxzIQJjIGFhqK33LzZvZWXygKuADVMmoKRBQA1yTtB4HP3cddeP +RadlpiBNlNGWjWaatIx2xF0DtC+l5ikGn9/c7aDdqFIb+a+qRuEl39rX9oDy5aYy +ZhoLFOMJu8qOOeofE7e+P9mCb7FKvDE9uJdbd17CzaMOwpsc69ufiju+C4QyujYs +YyytyoFkCy1xN1o9i8QY2ncm7iWwjQlPYZMdtgiPMe4xecMQQ9xuuDFiAHujjVIS +ZyRxTxliwCCTU46JhT/lYU50wZobu1RFWD6zXIHRHNZUo3chiWsQfOPP6IFOHJna +ZbQnni3TOWBkSFi4foCIxO4h9xyK2uMwOEzE7k6DJWbRNrpbJSj8bnH2p/q/e2lG +sA5Tpr6ZjHhCA3hZItU14MPTpUGM5U3r8iO6fAh9R5Vh07qx8xk7lbikG/+ka+iy +wsCz4G4qHkUn78+KKooG7IPb2hqODLDJTZfPprF52qs6NAiccTsWOAc7iUafJ6W/ +cSEsIqA9wRCSPkHmJuFZAoIBAQDbLxGsjpHY7uUVSiVysS0G7VWppPy5of3xPCf0 +SC2fLfItjiw3PpSOf5/Kzun/YouhG8YtyTMcK9zEj0gZ4OOc0ehK+43E7hvJ6F1Z +ruAAm/+DpntVqAgLL3pNd3rsV0J1WVCmGQPRSdzP3TS+2EZGyBrFnmLykYA2LpCN +7XvmEsLeZ89LqMEcnpU206LYdiZXVR2/Oeue+ZK7jSfZ/YDAvmm0egz1nJofo+76 +0DpznJtao1nYGXdZCvsr5KgHlcPZ2XrYLVJLrwf7ltL1K8W4nrYfJKFGFdYWPbqN +Niuye7tc1o92SKxafP+UHi8UbK+b5ONUcnM/OTg/adDTygKNAoIBAQC9gVW94ZaU +hLfS7OzLwQbZ1mJZr9sHQt6LQz0xsBlQ+ssfSVrb58H3Ov+vqqJ7GHoX66XPaXYZ +7zDAOnoIrKejcD7gpdIleAnIjlkRiZgCHKet4ZjtrvW5oaWUQFG5TSMqFi0FjsZl +SdkYAOJkcckZQgRfrgvRvqLjG7+TajS2XQc/HQlBqt0SmEeCzaW79EQO70oQgak1 +zjStGpFT+RL21fKmG7Fdy9fnTYLd8za4uiQeCKIBObQhqW7wlONt+RV3szxrUCm8 +hnpBrY3KF/l+Au3fY1DGwXQZSKpRqRucllEpyQKWS2i5CoPka1f9GMn5Xb1DMnFi +kgRMbdw1YVDHAoIBAQCkkpr6rbHk6LspWRr1GwNsCBgh4LfBylgaIcj+KpPWyXDl +s7KPaHWy6TDZ3rLkBuJAfdI33rJ5nJWPIOZKSAmfXhzE8ExqaT6EQ+yTwjJ/QqJ/ +/yjsD9a6T1PNhsDNZFeKNR3RGUc8hfE+QiGwikN6MhWn/FzfNVDHXf88EezNu5iF +1fxYGsWk742qNQ446b2wJUDIrHy2ST5bhIk/rJBYKKDF2j6QzJ0M1NJDkMXSuOwN +CkoTcEukmBIvyug/ibwcfqc2WhFIsouT5JNRcyRqkwC8MYzGSu8MlBzWmq4EvdLY +ymG9tqNy7hgY5vECrPjOXPY0GhtehWAufQ1HeI6hAoIBAGQT02YNpmoUlMeNW5hW +/fk4oIPgvPjetYwyHDULXLNJXs/M+3C6udKIk9L/eAMt7/yF8/DPLxGPId3ChAPk +ujsz4eDdcVdcRz5k/mCmm2IyI9NOGmcbSKWZ2kmqlf4X4IQvZeNTfS6sizuD7AtD +lAIEvS6SSaeg14C8fLWrFt9fzZ3lBahqRYm2Zb2MQQQ7gV2pvSFueB/0IBlyuGDy +XrOAVT6nzUlNh12dr4jrEnEHzF8YDAY9pQVJ506iMmE3c8DdJQE/OmtXUWUx9HlP +o/CKN3kLh/MO4vMfTuMFPZG6SG6auwUIGevuw6xzi+oblz5b5MGB+CMoHaEHO43t +MK0CggEAUnJ//azM02OCVFMXPxd64LU66JFKnLGWImHrnuiHPa1IPfkYGtW/OHVd +vVfG8/5rhDt9rQe0xnPWShvCDpT4UltRkKr/sZkWKDYTdRGOWPQWrHFKMgGcnK54 +r6d84PR+wjZbiRZ5Bfur1FueiSED4N4iNcg5nK2l74C4VTY42CaJdDbghlU8JhMg ++ZEpbTKvfJWGjftxPoYLNnKDE/t5pMPPrLNI7ig+c/EQXPhUj0lWlrk1uVWZULtG +NKsCukrAw7cqgyvTmE7qDJiDPEVyIPrYPlUEaDpn1PrSj2iba/TtKevPvStJ7AES +xNR7q9cOGPhVmOCBFRepA9J5U3OaBA== +-----END PRIVATE KEY----- diff --git a/crates/remote/tests/__fixtures__/certs-local/client.crt b/crates/remote/tests/__fixtures__/certs-local/client.crt new file mode 100644 index 00000000000..564c6b814a5 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/client.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE5zCCAs+gAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAlsb2Nh +bGhvc3QwIBcNMjQxMTIzMDIyOTM5WhgPMzAwNTAxMjUwMjI5MzlaMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANE6 +gexQ7p3zzD4QdwWQ032xw8ixzrmUyulTjTJ4XIRSwIcfHMIiJM6TKbIDhtd3wZyp +MHOUGo8oQbsqHS+UN14OCn1fC0LlNo/XptHDnVynf2BgJPpcX8ugy7S7B2vBgkio +P75qTXMCGm1EwE9cjWzjMiZw8BTY66OFrNQ8aMMJz1+QqiEeMjGa5g/lpWT8n3l2 +8rtyjUGKr9OSLmakj7fMQt+/ZLxkZS3XK5VZz4WZkLUqP/r8V1qe2IXk7wFVdSMi +VYgJbIf/tdpQeZZXG0iF6UnsbpfTcmXp5AkiAdhRcqcRBIP45hGgSvBU8zYyNvlh ++S6WKrCetj5htmZqCOur6ytnuqJllCfIXyOL6IDtrfyIM8cWBFJ0CgQJXCzpUCWZ +4O2PacfwKFY9+qhNr1H5cD0ARm/KlcrFyLi3ZPte8eETKeor2xUEhfsUqUHLOzjh +0fRq/DGISqU0ZF/jNUe1lJxeupS0qb59/c2jCmKhExdbdvqGu97XVwjdZzk6wuK7 +hQqyewpsSZAxqYqyTxJtR2EEd9hpotpsSo087ZAkf4T6ULMU1IHQazwtaBCsKB/P +eR1b9b4BtFG1aOzvafgynHe701Ddg+GZRMStzPWdLUUgoU8YaS4bNACMue2YdqFM +3ed45wl5noGqI7TWthWpLskR/1N7fDW/ky3bQVz7AgMBAAGjQjBAMB0GA1UdDgQW +BBSlWYIl5SZ+v7XOdmQlbL9bgvlqxjAfBgNVHSMEGDAWgBR2jZmyF11is35kQLaZ +7vFtubrPXzANBgkqhkiG9w0BAQsFAAOCAgEAZNz4gfXA2l1LJh3aQhwEFRrxjz9p +xtK3u82ng9Jq1yC6Iq9Jarhw1tgwHoI5HWT1j/+waHa0mlC6AdRfiPVG5vpL8hmQ +4E6yn2pJJqsdTO8Y+4ZkcezsLI1DwnqnYeWPXrdLe0GbW0wMWmomNmtzXTFlF8QP +odoMvlHSaQI3BnYGIaYxD1+ZIrh38FL756T7d9ZQZMIAJ9eZtcKU0oMCwR96MQ1+ +Wmka0CIkmWn7BTlrF5xqf+TJVjhP0I2Qx2urYDLrHqKJA1QalHPPCRUujfTBstY6 +jcKIi7tCi1DVq3fc/VyEO1/sAo0hfIaHyymDJZTwd+DkUaI3c4TxlhbWXnhfQGjr +cOM8LYriR/uu6MHsvxAuRGp5s+KPX1e5yDX9xszbI61lbIfU/jmiShz15Da/Aykj +uI7SjkuWEQuxPX7cxDVJswOxizfX8KPT7GNDVeeZWlNWx6WJ8D8EKTGcPFdkZy3k +sGqUZ2VNxG8i+dcEgiRdBNzpVgoPi5wxWCNskIhlduiUeQ1Y7DJDX9c5iMvtJI9v +na8A+rqiq7580RVL2LddU7SxENCeRR9DirOwqw9NQFVaC0kO7IFXj2wVpaJx9hlY +GdH1/m9KcnXZk2OXLMbJPJeu4OPLRqwO+TkiNTRi1UOUpgSBn8OELwbrzFpWInRD +nthk7OIV1xYVgVc= +-----END CERTIFICATE----- diff --git a/crates/remote/tests/__fixtures__/certs-local/client.csr b/crates/remote/tests/__fixtures__/certs-local/client.csr new file mode 100644 index 00000000000..97a6d91f3b4 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/client.csr @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEWTCCAkECAQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0B +AQEFAAOCAg8AMIICCgKCAgEA0TqB7FDunfPMPhB3BZDTfbHDyLHOuZTK6VONMnhc +hFLAhx8cwiIkzpMpsgOG13fBnKkwc5QajyhBuyodL5Q3Xg4KfV8LQuU2j9em0cOd +XKd/YGAk+lxfy6DLtLsHa8GCSKg/vmpNcwIabUTAT1yNbOMyJnDwFNjro4Ws1Dxo +wwnPX5CqIR4yMZrmD+WlZPyfeXbyu3KNQYqv05IuZqSPt8xC379kvGRlLdcrlVnP +hZmQtSo/+vxXWp7YheTvAVV1IyJViAlsh/+12lB5llcbSIXpSexul9NyZenkCSIB +2FFypxEEg/jmEaBK8FTzNjI2+WH5LpYqsJ62PmG2ZmoI66vrK2e6omWUJ8hfI4vo +gO2t/IgzxxYEUnQKBAlcLOlQJZng7Y9px/AoVj36qE2vUflwPQBGb8qVysXIuLdk ++17x4RMp6ivbFQSF+xSpQcs7OOHR9Gr8MYhKpTRkX+M1R7WUnF66lLSpvn39zaMK +YqETF1t2+oa73tdXCN1nOTrC4ruFCrJ7CmxJkDGpirJPEm1HYQR32Gmi2mxKjTzt +kCR/hPpQsxTUgdBrPC1oEKwoH895HVv1vgG0UbVo7O9p+DKcd7vTUN2D4ZlExK3M +9Z0tRSChTxhpLhs0AIy57Zh2oUzd53jnCXmegaojtNa2FakuyRH/U3t8Nb+TLdtB +XPsCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQCC8hSGs50iothw+bWOGlWyu6bO +hCbDUjNPgxeARuGyzutkRXgPm4N4EILZiUv21hs+PaGA3JpzKEdnYA5+Ba0iNkvz +LV1jTont5ealb3JsP0XWSWE4suiJX01j60W2bw71FoD71kqMDeZrJ80jl7g7yMob +kPpuHfgD7JQENQxTssbT084xegihct+yN7OdnAzIWeEBLGcOnbxABi6/wPfXybfW +eZsfrHGGfqYXuMjefcKKbKbWgwUPdz39D+iMpq9WTzwO0ubj3KHsasc/g6m0mYfD +HWTX3UDlpuT1XHjX/DGIcCR+EN35wiyfrW2mQFDuSRN8VVtzyamUFGJYxmW08aYd +EKJanIGWVHNpWvhzEhwn5Y2Fy7vuMUQFpAWuRS9PMJBdaU+GJr5zZGOh/KUez+xK +qP8ffLyNdRzCgLN8TAPSfrSt9uan1N8B+Xck4wjFsHuah/1AoGgn7w8WsBuqA41Z +QWrVO+auf1u11pOTi/cvHfUgglNnMxhWEPQ7V26cpsx4u9S5veH+b2+UZcQwLq1T +48nCAQ1Qib4Slg7cMsztQ5HPdD5FTG3h9u0EnKSnTfZqaesMxwG3rmsvBVfpkPtZ +hjqawec0pcLHfj0sH3osab0aIkvdD2XdkjqqbXuipqYDRqI11Db501LBie+wWxTF +E1Kg8d5W6PXPJTeOyw== +-----END CERTIFICATE REQUEST----- diff --git a/crates/remote/tests/__fixtures__/certs-local/client.key b/crates/remote/tests/__fixtures__/certs-local/client.key new file mode 100644 index 00000000000..2041fb5f342 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/client.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDROoHsUO6d88w+ +EHcFkNN9scPIsc65lMrpU40yeFyEUsCHHxzCIiTOkymyA4bXd8GcqTBzlBqPKEG7 +Kh0vlDdeDgp9XwtC5TaP16bRw51cp39gYCT6XF/LoMu0uwdrwYJIqD++ak1zAhpt +RMBPXI1s4zImcPAU2OujhazUPGjDCc9fkKohHjIxmuYP5aVk/J95dvK7co1Biq/T +ki5mpI+3zELfv2S8ZGUt1yuVWc+FmZC1Kj/6/FdantiF5O8BVXUjIlWICWyH/7Xa +UHmWVxtIhelJ7G6X03Jl6eQJIgHYUXKnEQSD+OYRoErwVPM2Mjb5YfkuliqwnrY+ +YbZmagjrq+srZ7qiZZQnyF8ji+iA7a38iDPHFgRSdAoECVws6VAlmeDtj2nH8ChW +PfqoTa9R+XA9AEZvypXKxci4t2T7XvHhEynqK9sVBIX7FKlByzs44dH0avwxiEql +NGRf4zVHtZScXrqUtKm+ff3NowpioRMXW3b6hrve11cI3Wc5OsLiu4UKsnsKbEmQ +MamKsk8SbUdhBHfYaaLabEqNPO2QJH+E+lCzFNSB0Gs8LWgQrCgfz3kdW/W+AbRR +tWjs72n4Mpx3u9NQ3YPhmUTErcz1nS1FIKFPGGkuGzQAjLntmHahTN3neOcJeZ6B +qiO01rYVqS7JEf9Te3w1v5Mt20Fc+wIDAQABAoICABNQlWHdbsnCEd7A4lvvLLbz +zCEg2Pak176Gagh4uAG3KfLPWnZJdKBlUXoPfUky0vrFW6WvzGveyYKftqYHxry2 +Cx3bwkxgoJLS06GVr1c4VzfsaD18fZNsuqFqiprV+FMAxTGs2o19ajcgG6i34I5m +wuBtmJw0Ejy8QGIvWXR32V73Gd2IzhVqOUbKamRSNd6ernXDc2rThLvBSOtM0b+j +6aCCKKiDlBuRO4LEEbZBoOw4hEOFPAMvmslxgubanr+pkb/HLu+3GXCX8UGBtRES +26BpZSQ67tqdpuF49Dal/jGSDpqoRYTnCUP6c6FTLWQEOV6lmjdHz4ceuhnmPABq +p1XKgW2xPwb8aMruhsMdI4G/oYd4hrFqAajBYlK+o+gq8aMgBKbkWA/J6OBPl1N2 +QaFGF8zWPbZ+/az4EJKvyKzh/32J7rnt2KlyeaatH9qeLMHtqNLJx5Up1d60tmPV +lLfyY4lO8J/3lxThlz/sVivDybkye2UhkGijplI2x0bbr0sGj/02wU74CUathKo9 +THPSnTqdbXlsX1MXZtHB0TauqhHTmkDJdxwPj8irzSEn8gsXr3BWjGJzzCqgKOsy +TIS1RWX/p2ugsFwU4HEhwHI0yp7pLSLNrSyxq+bBsJRq6GacoUf0Inv24q7rSIyQ +W+ysKo4XIaRp4Tg/aZ2RAoIBAQD+suMR3x1jzHkUL3KDrN0THqW5VlvFEKhPvTNn +cF3vfduyF53sMgWNMh5XWBFBd2HtN3EfgsgQZ1ZPpOPCPEG3FoXwNLzZqdd97KK1 +mOSB1LUTio6PbZBzVdi1OOzvLMcSok44Lf4xl97ico2xb61Ap4meJt1/Fx5zbOzF +epRfbQvVH139NICw+/tC0b7Oo5kpaQdMQnbwXp/tMLKnd3r8kRf/4AHpLefdV+do +BziXKC7bQ+m+kPTgGbjtCMSqiQS1dwxSawDnhhj7XEbALvGq3S0yLaFfAG/nczmZ +Th9+ss8IzucqLFILZNpo5DMtPHQ2q05OXEO/GauTMOLNE1VnAoIBAQDSTCa+qPz7 +Jghu+Em65VWpXV74BlUkLTbZRi6GimVnYyNHbKPxFcjyBw3B4N3yQMdxpQ489qWG +c0+eVL+kcXk9FUxMUzfsivO0skF5QI2+7StIl8pLJlk1A7zzJq/cYCdwkcO8nd1p +AdcivDrAxLRg7IXJRGSd1VgsKf7PRdkGmhCpCQ+6phe0w7v5aT3vNQEt66ycJvEk ++SCbVcsQ7/uSRI5TsOduVZiF0zP91PGoQtdvddBF3aIz8AdiSbaD3H46N6gZWEOP +iAMBbmzzQ+LZR+Lm4o3T+vsvRMgD5f1s0CwpnrhUfRUjeuW8sWyXTXYUTUYfJxII +uxR51Y0fiMtNAoIBACSQCYjHPrDU6Yy8QvQkHJhiDehNPV9MxNytjHOM7e8zYhZu +zXzasXFTgIeJXPDI8oXbL6IWZCH2s607Pnbjr4tY8GFLMNEOUKEDF1h6WlNI9bWY +bl6om9PyvulFAr5S70D9i70E3TjBVH2tdCnEnlppspfBfIqCBx0KxfKTZjlMriYq +GpP+tXqysZ8l/P4s/g+zxBhuciSPcOXb18mynTUknw1cWunebSqZM977KmSIoDFZ +znRM2tpQ1vswBwt0H/js3hf2xGp4FZ3/4Sg1lPuyRIqvgjrhFS9kmyAm1t6ZTJ83 +FHg5ZI2+LJxiJfaYdxZiARu+ovsL9FVs7yIBZIECggEAVpf41EVtdlUg3+DpK0ZH +0aea9XJWGMFtdmZJvii6vKm72ytHDnyz88Unyw+3FIvMFGWsyTmeyxAsVrv69dGr +5JUih8M0ofhNhbho8W69b0LlscyfBfbSgNv7L/xcKdiGJPpAqSgwBY82cR0k7D+T +Lt318PxymfNBjV31iI/wX6GM/q22hlriJBL9EiRd1mPzCl+jAeGfJmRXIt9e8KiA +5KfKG3gas/oXBfQD0p+eqnRrlX/jtQEgS1apE7Xaq81dRMeqNlV77FxLMImx3zGy +9Vl+eygK0qDkUYB3A/PuOKI3rSOoL1IaJDP06Y/9cQf6eT2ghC9oY6P1OH3Q0I1Z +eQKCAQEA2Nl6b0b7Td4BkHL73uJeRm4lAVbBJqELLKRdKiBHK+qW5pB6OjvjRONV +lFlGGeWIOUCeJsrEJ/KGfh+mqWZoVTilGR/SceqIFUAOWwxFvuaSGKS8JJGfgsht +5hKibXTkFTTe8RtwG9qFoefHdb2YncTi8k32jYlvkbhy0ln8KajHV/b/ljqKLpLp +YYD+IuIe/cIL3ozzZGHDmfztNvjKl7iaF1uft0edupdjHUZqSZf0ttfEOdeKqL10 +CvOKKopJG8v+3C5BbNdFIFLtxJGd36lw6VO8DH7vVUWfexHDvcPCffXTYBaNW5pC +F9AqWlMi9KS36CYoMfYmLpTbzCPiIw== +-----END PRIVATE KEY----- diff --git a/crates/remote/tests/__fixtures__/certs-local/client.pem b/crates/remote/tests/__fixtures__/certs-local/client.pem new file mode 100644 index 00000000000..2041fb5f342 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/client.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDROoHsUO6d88w+ +EHcFkNN9scPIsc65lMrpU40yeFyEUsCHHxzCIiTOkymyA4bXd8GcqTBzlBqPKEG7 +Kh0vlDdeDgp9XwtC5TaP16bRw51cp39gYCT6XF/LoMu0uwdrwYJIqD++ak1zAhpt +RMBPXI1s4zImcPAU2OujhazUPGjDCc9fkKohHjIxmuYP5aVk/J95dvK7co1Biq/T +ki5mpI+3zELfv2S8ZGUt1yuVWc+FmZC1Kj/6/FdantiF5O8BVXUjIlWICWyH/7Xa +UHmWVxtIhelJ7G6X03Jl6eQJIgHYUXKnEQSD+OYRoErwVPM2Mjb5YfkuliqwnrY+ +YbZmagjrq+srZ7qiZZQnyF8ji+iA7a38iDPHFgRSdAoECVws6VAlmeDtj2nH8ChW +PfqoTa9R+XA9AEZvypXKxci4t2T7XvHhEynqK9sVBIX7FKlByzs44dH0avwxiEql +NGRf4zVHtZScXrqUtKm+ff3NowpioRMXW3b6hrve11cI3Wc5OsLiu4UKsnsKbEmQ +MamKsk8SbUdhBHfYaaLabEqNPO2QJH+E+lCzFNSB0Gs8LWgQrCgfz3kdW/W+AbRR +tWjs72n4Mpx3u9NQ3YPhmUTErcz1nS1FIKFPGGkuGzQAjLntmHahTN3neOcJeZ6B +qiO01rYVqS7JEf9Te3w1v5Mt20Fc+wIDAQABAoICABNQlWHdbsnCEd7A4lvvLLbz +zCEg2Pak176Gagh4uAG3KfLPWnZJdKBlUXoPfUky0vrFW6WvzGveyYKftqYHxry2 +Cx3bwkxgoJLS06GVr1c4VzfsaD18fZNsuqFqiprV+FMAxTGs2o19ajcgG6i34I5m +wuBtmJw0Ejy8QGIvWXR32V73Gd2IzhVqOUbKamRSNd6ernXDc2rThLvBSOtM0b+j +6aCCKKiDlBuRO4LEEbZBoOw4hEOFPAMvmslxgubanr+pkb/HLu+3GXCX8UGBtRES +26BpZSQ67tqdpuF49Dal/jGSDpqoRYTnCUP6c6FTLWQEOV6lmjdHz4ceuhnmPABq +p1XKgW2xPwb8aMruhsMdI4G/oYd4hrFqAajBYlK+o+gq8aMgBKbkWA/J6OBPl1N2 +QaFGF8zWPbZ+/az4EJKvyKzh/32J7rnt2KlyeaatH9qeLMHtqNLJx5Up1d60tmPV +lLfyY4lO8J/3lxThlz/sVivDybkye2UhkGijplI2x0bbr0sGj/02wU74CUathKo9 +THPSnTqdbXlsX1MXZtHB0TauqhHTmkDJdxwPj8irzSEn8gsXr3BWjGJzzCqgKOsy +TIS1RWX/p2ugsFwU4HEhwHI0yp7pLSLNrSyxq+bBsJRq6GacoUf0Inv24q7rSIyQ +W+ysKo4XIaRp4Tg/aZ2RAoIBAQD+suMR3x1jzHkUL3KDrN0THqW5VlvFEKhPvTNn +cF3vfduyF53sMgWNMh5XWBFBd2HtN3EfgsgQZ1ZPpOPCPEG3FoXwNLzZqdd97KK1 +mOSB1LUTio6PbZBzVdi1OOzvLMcSok44Lf4xl97ico2xb61Ap4meJt1/Fx5zbOzF +epRfbQvVH139NICw+/tC0b7Oo5kpaQdMQnbwXp/tMLKnd3r8kRf/4AHpLefdV+do +BziXKC7bQ+m+kPTgGbjtCMSqiQS1dwxSawDnhhj7XEbALvGq3S0yLaFfAG/nczmZ +Th9+ss8IzucqLFILZNpo5DMtPHQ2q05OXEO/GauTMOLNE1VnAoIBAQDSTCa+qPz7 +Jghu+Em65VWpXV74BlUkLTbZRi6GimVnYyNHbKPxFcjyBw3B4N3yQMdxpQ489qWG +c0+eVL+kcXk9FUxMUzfsivO0skF5QI2+7StIl8pLJlk1A7zzJq/cYCdwkcO8nd1p +AdcivDrAxLRg7IXJRGSd1VgsKf7PRdkGmhCpCQ+6phe0w7v5aT3vNQEt66ycJvEk ++SCbVcsQ7/uSRI5TsOduVZiF0zP91PGoQtdvddBF3aIz8AdiSbaD3H46N6gZWEOP +iAMBbmzzQ+LZR+Lm4o3T+vsvRMgD5f1s0CwpnrhUfRUjeuW8sWyXTXYUTUYfJxII +uxR51Y0fiMtNAoIBACSQCYjHPrDU6Yy8QvQkHJhiDehNPV9MxNytjHOM7e8zYhZu +zXzasXFTgIeJXPDI8oXbL6IWZCH2s607Pnbjr4tY8GFLMNEOUKEDF1h6WlNI9bWY +bl6om9PyvulFAr5S70D9i70E3TjBVH2tdCnEnlppspfBfIqCBx0KxfKTZjlMriYq +GpP+tXqysZ8l/P4s/g+zxBhuciSPcOXb18mynTUknw1cWunebSqZM977KmSIoDFZ +znRM2tpQ1vswBwt0H/js3hf2xGp4FZ3/4Sg1lPuyRIqvgjrhFS9kmyAm1t6ZTJ83 +FHg5ZI2+LJxiJfaYdxZiARu+ovsL9FVs7yIBZIECggEAVpf41EVtdlUg3+DpK0ZH +0aea9XJWGMFtdmZJvii6vKm72ytHDnyz88Unyw+3FIvMFGWsyTmeyxAsVrv69dGr +5JUih8M0ofhNhbho8W69b0LlscyfBfbSgNv7L/xcKdiGJPpAqSgwBY82cR0k7D+T +Lt318PxymfNBjV31iI/wX6GM/q22hlriJBL9EiRd1mPzCl+jAeGfJmRXIt9e8KiA +5KfKG3gas/oXBfQD0p+eqnRrlX/jtQEgS1apE7Xaq81dRMeqNlV77FxLMImx3zGy +9Vl+eygK0qDkUYB3A/PuOKI3rSOoL1IaJDP06Y/9cQf6eT2ghC9oY6P1OH3Q0I1Z +eQKCAQEA2Nl6b0b7Td4BkHL73uJeRm4lAVbBJqELLKRdKiBHK+qW5pB6OjvjRONV +lFlGGeWIOUCeJsrEJ/KGfh+mqWZoVTilGR/SceqIFUAOWwxFvuaSGKS8JJGfgsht +5hKibXTkFTTe8RtwG9qFoefHdb2YncTi8k32jYlvkbhy0ln8KajHV/b/ljqKLpLp +YYD+IuIe/cIL3ozzZGHDmfztNvjKl7iaF1uft0edupdjHUZqSZf0ttfEOdeKqL10 +CvOKKopJG8v+3C5BbNdFIFLtxJGd36lw6VO8DH7vVUWfexHDvcPCffXTYBaNW5pC +F9AqWlMi9KS36CYoMfYmLpTbzCPiIw== +-----END PRIVATE KEY----- diff --git a/crates/remote/tests/__fixtures__/certs-local/server.crt b/crates/remote/tests/__fixtures__/certs-local/server.crt new file mode 100644 index 00000000000..c7fa2e532dc --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/server.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE5zCCAs+gAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAlsb2Nh +bGhvc3QwIBcNMjQxMTIzMDIyOTM5WhgPMzAwNTAxMjUwMjI5MzlaMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAM9r +kbZUthpX53qFHf3VaAxupcSLBK3Z3hAu/QiZWef+AAT3bC4yUvlv21HBw9l2lIPs +JoFNsOMYyp2SHda6iq8Rd42H7QE2rNA/rreYHWnhsM10R522KHDWzAmRKM6tC7oT +VGUhQbq6YqTAvvT5NuPSeAmDGrqUjPMoLaxHTw+7s6RG4iC1Y1aUcAoN1ILpsfdY +j4ZORzqBHyGkL1kjocWRQtWSEDCQDLIiWVf6CNz5ZjfhJY65969upXJizX2BQfi2 +61s5rH4axbGBG/8+gi6Z55UPwY8jPlM3cE3aljG02/1/Wmt3jcVGKApbOoF0YP35 +UVm5lfW3GiKMFS7NuNBBGN0mLdSuK1Dss5+nzUhCoEG84upI7Y37gCZpopMYLnky +o6A8nSAg6s1j4O7Hqt1hF43nQrNJEWn+xXyurdZFPOQpWzOI54grV9j0sdDUmudn +NfV0ZVWKAtEnJ6VmDDqif6X/JjhRtRTlOQN9Vfjp7+xKba2kCqc2YIfYTcamp7Uu +/b55Ip+TyaCeC8E9DWqdAwfiA1Ubt2GK+vpTdkWMe8oIvuqhwwRsUQsRjKQ9Lhnj +bXHrZiIcktofFRm3GI2uj0SvrcYZ+DPAWFRWa76FdkJVO2oKHbNGRdViVHsQetZq +1Bk9SbWXbPh4xivs83q8xFqEdx8NSMuNL1GEtIAFAgMBAAGjQjBAMB0GA1UdDgQW +BBSJSsjbIULUrJcuzkteVEp6xiEbUDAfBgNVHSMEGDAWgBR2jZmyF11is35kQLaZ +7vFtubrPXzANBgkqhkiG9w0BAQsFAAOCAgEAL7axoSvm5zBNxzQa9HclejNYj1uI +pob+ZJTSFJkPqGibkSAKxFOQnF1pQiOpaCUfn53HBEbk45wgWzfW3TknTM9HeC/z +ZexzNs7WlE9S4fBr3Mft9txQSwTJZkuUiU4XLaLONp732Vk4EFnpn/QOA0v2A+8e +PFO/8YmVuHzjRslNieji4hx2M1duB34yJ7zjeWo3i6Kbxjla0EGDkDG+hSdCLwjj ++oUqG3zZHTxQwXzHVxXV07itMOJ7zprVnNRkY1PKkI/d68120P6Pw5L7Ey4VvB70 +Lzk8vf2eeeQibw4UtOl4ct5gqc6ue+vrTn7YYfmfpH5Jc8NqyebcodmwAK1tXJ91 +/gKIkYCXojfng9rNS7HpCBrBJnSorwA1xzB6gAizZk0uB1Atn2vVGJ+j+U872wZp +uisTvWa0yyxf6JoL4u7I2AjCnY+04OuHTUPmJ7dyJexl31N6KBWvyqKplU6eN30/ +INBTA+NUemifjk+SRTGo01cbe3gvdhhNspRGrkPnzVd7xUERSmL0CgpgeYi9Swck +jctnulhxU1InqgcLPV9FPGcyGUlvBc5OGsA212jv9Wg5Khvs47KYPZ5nwrfiEWpi +5vFeJxOLa6o/7jw1as3ASt/e5WUcXk3Emqg1MDe7kAcWdQx71hL4ihUtEsKrx82a +ZndDny21bi3Kbnk= +-----END CERTIFICATE----- diff --git a/crates/remote/tests/__fixtures__/certs-local/server.csr b/crates/remote/tests/__fixtures__/certs-local/server.csr new file mode 100644 index 00000000000..dbecf9854af --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/server.csr @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEWTCCAkECAQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0B +AQEFAAOCAg8AMIICCgKCAgEAz2uRtlS2GlfneoUd/dVoDG6lxIsErdneEC79CJlZ +5/4ABPdsLjJS+W/bUcHD2XaUg+wmgU2w4xjKnZId1rqKrxF3jYftATas0D+ut5gd +aeGwzXRHnbYocNbMCZEozq0LuhNUZSFBurpipMC+9Pk249J4CYMaupSM8ygtrEdP +D7uzpEbiILVjVpRwCg3Ugumx91iPhk5HOoEfIaQvWSOhxZFC1ZIQMJAMsiJZV/oI +3PlmN+Eljrn3r26lcmLNfYFB+LbrWzmsfhrFsYEb/z6CLpnnlQ/BjyM+UzdwTdqW +MbTb/X9aa3eNxUYoCls6gXRg/flRWbmV9bcaIowVLs240EEY3SYt1K4rUOyzn6fN +SEKgQbzi6kjtjfuAJmmikxgueTKjoDydICDqzWPg7seq3WEXjedCs0kRaf7FfK6t +1kU85ClbM4jniCtX2PSx0NSa52c19XRlVYoC0ScnpWYMOqJ/pf8mOFG1FOU5A31V ++Onv7EptraQKpzZgh9hNxqantS79vnkin5PJoJ4LwT0Nap0DB+IDVRu3YYr6+lN2 +RYx7ygi+6qHDBGxRCxGMpD0uGeNtcetmIhyS2h8VGbcYja6PRK+txhn4M8BYVFZr +voV2QlU7agods0ZF1WJUexB61mrUGT1JtZds+HjGK+zzerzEWoR3Hw1Iy40vUYS0 +gAUCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQCyk2Lg+F4xyNEaF8UnrdNFUmKk +Z1tfMf4d5Q8+/axUQdF/0swltp853VD85ZGoc2tK0ITuz7s167Arvqb9XY0C1aQm +EZAwNM0kiusHZ15BKXeNMxOg9Y11b5U/M1UoTijlzfXNHvmnsQOXHTj4cytewd3B +7VvL2EEVr2S2SfXqb/mZKKqPqu9n2qmOuS9YvCx9rQVyAgFMf7e0oTP0fOis1cq+ +9zqfTOix7KvMwb7bC0afa4m13CoY5BBx9S8MuJHledy9XI7nS8a4VJK4qe+UX6CX +3KttISNO2VgnWeG08q5TZZE+jGV4QW+OuwwtmcH0yOt/KIzWfLjsVcCuunoa7BsG +rxWAEI4/MGT8u1HLgkZYjtEEvW86cIERUf75zUJWyIkCRNgvO9wEpqJklFWksuDR +j6jc88kYuj/96AG1F5vfWv4srplz9KZpDN4b1uLj9/aGjWIp/LtCdC/0FkmoyJpT +iTwHeEjSSZgDeFIhVPfNf1Hcpkqbb6C3xFAxyrWMbNb8L284NVZUWQGelUhU0FD4 +ME+a6/ljgxcxlcTTSq0pA6AaqOoj4MvOI8E65DhVk6FxyqTmtIqetL04J56CYo3T +HD4vzW+CpFBkXXF70oMpjxGLfZjqA15cCW3ODZ2Cm4Hb8IDELbqw4s3RAc437ThM +2lvO0mM8uAIPIs7d7Q== +-----END CERTIFICATE REQUEST----- diff --git a/crates/remote/tests/__fixtures__/certs-local/server.key b/crates/remote/tests/__fixtures__/certs-local/server.key new file mode 100644 index 00000000000..739d261142d --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/server.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDPa5G2VLYaV+d6 +hR391WgMbqXEiwSt2d4QLv0ImVnn/gAE92wuMlL5b9tRwcPZdpSD7CaBTbDjGMqd +kh3WuoqvEXeNh+0BNqzQP663mB1p4bDNdEedtihw1swJkSjOrQu6E1RlIUG6umKk +wL70+Tbj0ngJgxq6lIzzKC2sR08Pu7OkRuIgtWNWlHAKDdSC6bH3WI+GTkc6gR8h +pC9ZI6HFkULVkhAwkAyyIllX+gjc+WY34SWOufevbqVyYs19gUH4tutbOax+GsWx +gRv/PoIumeeVD8GPIz5TN3BN2pYxtNv9f1prd43FRigKWzqBdGD9+VFZuZX1txoi +jBUuzbjQQRjdJi3UritQ7LOfp81IQqBBvOLqSO2N+4AmaaKTGC55MqOgPJ0gIOrN +Y+Dux6rdYReN50KzSRFp/sV8rq3WRTzkKVsziOeIK1fY9LHQ1JrnZzX1dGVVigLR +JyelZgw6on+l/yY4UbUU5TkDfVX46e/sSm2tpAqnNmCH2E3Gpqe1Lv2+eSKfk8mg +ngvBPQ1qnQMH4gNVG7dhivr6U3ZFjHvKCL7qocMEbFELEYykPS4Z421x62YiHJLa +HxUZtxiNro9Er63GGfgzwFhUVmu+hXZCVTtqCh2zRkXVYlR7EHrWatQZPUm1l2z4 +eMYr7PN6vMRahHcfDUjLjS9RhLSABQIDAQABAoICABmOJMlEnZ0YSH5JaV0NToD9 +nvHcuOpcheYTi/xjvHZ/TmxOOadlbuCpnelwSJuB5yFr1oCm3EzWkJwiVQfQOub/ +7W1kGljEbj1II1Qeaz1Q37IoiexN9aSValUha9gu7NtzpznAg7MoZJ/s2XogPFmM +ZFqzqvbi94y73gi0TnLfSu5KB3FFN+SCbF1ov86TUWhqomBHQ7JbF0VTT7wZTkSj +tYrPsKvzC4VlAwH6Xd4v4h6vYCu6EOCt7rdtoei9JK4qh0vZir2Mud+5SwRDJNHu +B2DPrKbgjjy9f1owPMIKCPKuHqtP8wkjsCo6fDZX6t1Puuyll2rLMO6huLhoA0V4 +WhaN/0hGemgn8MUM601uF1PvKvcRriSjHn3ZS2s5xZoJDPwkd+4DpTVz42GqDeea +TOsKf3ooWBNH22A+Az8EgoStL4aIPckQjUTyn/P9leAPFkHV6SbYRCLpZxt5BTCT +/vBqm55vZK/BhnisdZFHUYzGqEJuSdyd3eONfaWxalFx7VbXGI8X7l3srmpgoqV6 +sOi4plJ8n9AOLvW7jUYE69rkWB3vZZKN7uwvl9lCuLzzTkEh/+QZJ25GzKJvgiZZ +sp3yc5isgcIYmz4ql7bCgdU01YudOMZz2m3CxQWakw17peQZTxfVh1jPWBLLj7Ub +T3HBXPkjfty/4Sj2xTJhAoIBAQD3lN/F44laV9G+LwD2lAiLDaoIOLx5UCMioMb7 +dECZ6nP42A8s8nEdoTy59uWSIJSh0z1RG3iHzsU7Ugt9x4vqQXY/x2yQTnYDbsGL +vrCNc9D1/Kat6X3KiIo+1U6iULELAwqs/JuOITE5Opeko/xlWh7YJWuP7AdZrrAT +jcYnbJhHofUTEPnqUfeDpziKzeKOrJQcC5uZZknoj+HYd7ZGUusRt33AG4X2lsHf +Xko/kVG3YQ/zsn6N5Jdy/8B5J6x2MTqOCamqk8hAOiQt2y2r4WXZ58uYyFsyjuXX +RRGBypQT91ZCCEP4nD2Vi7MeURfzKzZbwrdKFPa0EoD7ZIOxAoIBAQDWeRolPsrq +OkuSO1m6fG+gVzmC0/1w17pq8ra0WdYcBKYTMappHpCnNa7gEjhXHQI+wEDxYhGP +bc5AVLujJeOjxm3Pkq0vySxVVTRkNobXcfxNi8YzqNLeN0uJAC4VVd+Bd52ygx0O +jjmuHGA+pmwnaQxboMp0fidkhIYBfW6W+je76f1pk1RS4gzoeokC/9lN+QFZWIfg +gIhke42U4ZA8IQu5yn7ryKCCtiUJR+TzSuYhtMTAZFuqV5kDRjbKS49Q2C1yxaqk +vNdZNJhAF+sG7qRwNMWW8E+z8DTsOe6yy5Y5RRwpHEgFFtGI25M+oZqSSaO2m6DE +xZYUtFo3afqVAoIBAQCiIBc5WvsC8icjR1x2HBJMHLKPl7e0KUoYzvf+ie4T3Hf9 +KF8nq8tu/7rofElxG2y5W773cenH5Rs68UsdHqL8z8lDkrAA/6BANzL+8+xhdMoN +i1kK4a/CBQ+ifcR0fiqxb1h6VyD2Pb4iH8BVzBznsdAo8rgxEvrtO3uKEnkEMBS4 +M9Jsnd1KGErhJ4vbOV921oRWqTNHjLgxVqcqPZpbCslsXKA4QM3E4IwsXW55mgvM +7f6oMua0vFCAugGsVNKWwz347VLxpLypcmlmrfSGbZ9oxmWIswslXHYeNdh1CddW +C82siAUatBWP7F9rJudsEYwTIrssA/ZZjRRxiyzBAoIBABXXS1RTqrJjsWJ47NBS +78MyXsb1869UKMXPZc/D/kvoJUl7G1GNLUn5648iC97DM4FUbsimJgenNQPUHS+m +IiKBiJlE5KIjKIKgJK+35c926iyzeUG1hZPfqa+P9yFxF0vF9KdDNuhOj40tkpeO +tyDkT7/dBgEK/9P6svLf5S022gom8AmaWVH/dzUNlXpat7R7ESoaG+w+6qTg+DAb +MGcPLnSDLJ2yVrY4+liRm70y+XChShAVqEgWl5x/wACl5Thka2xuuMwB9yQ8eOy+ +As836kMK/Lw8SMGYSqtr0uUeWnQGl7GQEb2C71+yjLIqHuWa3jMwchTX0krVewJ9 +pmkCggEBAM/O4qtldVl0+8ay3wT8vYSp71YHiy++/JDKTlQcEGWJbulLBHBoFR0Q +4gQLB9J6l68WQLiYudjGcasf0DoS/N8eun3X7LBCJ+MkZFkM6ARFcG+9R1qer+ND +C/Nt0ulKPtUca3eYyQ0XVDFD8qGMXNzZ7mvCBvVSYRwcAYlcosBkp4/Hnx8ghRwk +RabtDyUDCc/AAfmyu5qQh/3yVQDp9RbmizVQVN0mV9rNf9F0TbDgYsrJLl7K+FyA +4dMEbOH899UA/38Qy6fqb7FGHmi4uiPtzvHJMKtYxzgECrHHukJMsJffTvtl+GFf +ouL5jxbaAWINZunw52PnJf1KM7Soo3c= +-----END PRIVATE KEY----- diff --git a/crates/remote/tests/__fixtures__/certs-local/server.pem b/crates/remote/tests/__fixtures__/certs-local/server.pem new file mode 100644 index 00000000000..739d261142d --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs-local/server.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDPa5G2VLYaV+d6 +hR391WgMbqXEiwSt2d4QLv0ImVnn/gAE92wuMlL5b9tRwcPZdpSD7CaBTbDjGMqd +kh3WuoqvEXeNh+0BNqzQP663mB1p4bDNdEedtihw1swJkSjOrQu6E1RlIUG6umKk +wL70+Tbj0ngJgxq6lIzzKC2sR08Pu7OkRuIgtWNWlHAKDdSC6bH3WI+GTkc6gR8h +pC9ZI6HFkULVkhAwkAyyIllX+gjc+WY34SWOufevbqVyYs19gUH4tutbOax+GsWx +gRv/PoIumeeVD8GPIz5TN3BN2pYxtNv9f1prd43FRigKWzqBdGD9+VFZuZX1txoi +jBUuzbjQQRjdJi3UritQ7LOfp81IQqBBvOLqSO2N+4AmaaKTGC55MqOgPJ0gIOrN +Y+Dux6rdYReN50KzSRFp/sV8rq3WRTzkKVsziOeIK1fY9LHQ1JrnZzX1dGVVigLR +JyelZgw6on+l/yY4UbUU5TkDfVX46e/sSm2tpAqnNmCH2E3Gpqe1Lv2+eSKfk8mg +ngvBPQ1qnQMH4gNVG7dhivr6U3ZFjHvKCL7qocMEbFELEYykPS4Z421x62YiHJLa +HxUZtxiNro9Er63GGfgzwFhUVmu+hXZCVTtqCh2zRkXVYlR7EHrWatQZPUm1l2z4 +eMYr7PN6vMRahHcfDUjLjS9RhLSABQIDAQABAoICABmOJMlEnZ0YSH5JaV0NToD9 +nvHcuOpcheYTi/xjvHZ/TmxOOadlbuCpnelwSJuB5yFr1oCm3EzWkJwiVQfQOub/ +7W1kGljEbj1II1Qeaz1Q37IoiexN9aSValUha9gu7NtzpznAg7MoZJ/s2XogPFmM +ZFqzqvbi94y73gi0TnLfSu5KB3FFN+SCbF1ov86TUWhqomBHQ7JbF0VTT7wZTkSj +tYrPsKvzC4VlAwH6Xd4v4h6vYCu6EOCt7rdtoei9JK4qh0vZir2Mud+5SwRDJNHu +B2DPrKbgjjy9f1owPMIKCPKuHqtP8wkjsCo6fDZX6t1Puuyll2rLMO6huLhoA0V4 +WhaN/0hGemgn8MUM601uF1PvKvcRriSjHn3ZS2s5xZoJDPwkd+4DpTVz42GqDeea +TOsKf3ooWBNH22A+Az8EgoStL4aIPckQjUTyn/P9leAPFkHV6SbYRCLpZxt5BTCT +/vBqm55vZK/BhnisdZFHUYzGqEJuSdyd3eONfaWxalFx7VbXGI8X7l3srmpgoqV6 +sOi4plJ8n9AOLvW7jUYE69rkWB3vZZKN7uwvl9lCuLzzTkEh/+QZJ25GzKJvgiZZ +sp3yc5isgcIYmz4ql7bCgdU01YudOMZz2m3CxQWakw17peQZTxfVh1jPWBLLj7Ub +T3HBXPkjfty/4Sj2xTJhAoIBAQD3lN/F44laV9G+LwD2lAiLDaoIOLx5UCMioMb7 +dECZ6nP42A8s8nEdoTy59uWSIJSh0z1RG3iHzsU7Ugt9x4vqQXY/x2yQTnYDbsGL +vrCNc9D1/Kat6X3KiIo+1U6iULELAwqs/JuOITE5Opeko/xlWh7YJWuP7AdZrrAT +jcYnbJhHofUTEPnqUfeDpziKzeKOrJQcC5uZZknoj+HYd7ZGUusRt33AG4X2lsHf +Xko/kVG3YQ/zsn6N5Jdy/8B5J6x2MTqOCamqk8hAOiQt2y2r4WXZ58uYyFsyjuXX +RRGBypQT91ZCCEP4nD2Vi7MeURfzKzZbwrdKFPa0EoD7ZIOxAoIBAQDWeRolPsrq +OkuSO1m6fG+gVzmC0/1w17pq8ra0WdYcBKYTMappHpCnNa7gEjhXHQI+wEDxYhGP +bc5AVLujJeOjxm3Pkq0vySxVVTRkNobXcfxNi8YzqNLeN0uJAC4VVd+Bd52ygx0O +jjmuHGA+pmwnaQxboMp0fidkhIYBfW6W+je76f1pk1RS4gzoeokC/9lN+QFZWIfg +gIhke42U4ZA8IQu5yn7ryKCCtiUJR+TzSuYhtMTAZFuqV5kDRjbKS49Q2C1yxaqk +vNdZNJhAF+sG7qRwNMWW8E+z8DTsOe6yy5Y5RRwpHEgFFtGI25M+oZqSSaO2m6DE +xZYUtFo3afqVAoIBAQCiIBc5WvsC8icjR1x2HBJMHLKPl7e0KUoYzvf+ie4T3Hf9 +KF8nq8tu/7rofElxG2y5W773cenH5Rs68UsdHqL8z8lDkrAA/6BANzL+8+xhdMoN +i1kK4a/CBQ+ifcR0fiqxb1h6VyD2Pb4iH8BVzBznsdAo8rgxEvrtO3uKEnkEMBS4 +M9Jsnd1KGErhJ4vbOV921oRWqTNHjLgxVqcqPZpbCslsXKA4QM3E4IwsXW55mgvM +7f6oMua0vFCAugGsVNKWwz347VLxpLypcmlmrfSGbZ9oxmWIswslXHYeNdh1CddW +C82siAUatBWP7F9rJudsEYwTIrssA/ZZjRRxiyzBAoIBABXXS1RTqrJjsWJ47NBS +78MyXsb1869UKMXPZc/D/kvoJUl7G1GNLUn5648iC97DM4FUbsimJgenNQPUHS+m +IiKBiJlE5KIjKIKgJK+35c926iyzeUG1hZPfqa+P9yFxF0vF9KdDNuhOj40tkpeO +tyDkT7/dBgEK/9P6svLf5S022gom8AmaWVH/dzUNlXpat7R7ESoaG+w+6qTg+DAb +MGcPLnSDLJ2yVrY4+liRm70y+XChShAVqEgWl5x/wACl5Thka2xuuMwB9yQ8eOy+ +As836kMK/Lw8SMGYSqtr0uUeWnQGl7GQEb2C71+yjLIqHuWa3jMwchTX0krVewJ9 +pmkCggEBAM/O4qtldVl0+8ay3wT8vYSp71YHiy++/JDKTlQcEGWJbulLBHBoFR0Q +4gQLB9J6l68WQLiYudjGcasf0DoS/N8eun3X7LBCJ+MkZFkM6ARFcG+9R1qer+ND +C/Nt0ulKPtUca3eYyQ0XVDFD8qGMXNzZ7mvCBvVSYRwcAYlcosBkp4/Hnx8ghRwk +RabtDyUDCc/AAfmyu5qQh/3yVQDp9RbmizVQVN0mV9rNf9F0TbDgYsrJLl7K+FyA +4dMEbOH899UA/38Qy6fqb7FGHmi4uiPtzvHJMKtYxzgECrHHukJMsJffTvtl+GFf +ouL5jxbaAWINZunw52PnJf1KM7Soo3c= +-----END PRIVATE KEY----- diff --git a/crates/remote/tests/__fixtures__/certs/README.md b/crates/remote/tests/__fixtures__/certs/README.md new file mode 100644 index 00000000000..c5e0701fac0 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs/README.md @@ -0,0 +1 @@ +These certs are taken from `tonic`: https://github.com/hyperium/tonic/tree/master/examples/data/tls diff --git a/crates/remote/tests/__fixtures__/certs/ca.pem b/crates/remote/tests/__fixtures__/certs/ca.pem new file mode 100644 index 00000000000..d8195609667 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs/ca.pem @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIE3DCCA0SgAwIBAgIRAObeYbJFiVQSGR8yk44dsOYwDQYJKoZIhvcNAQELBQAw +gYUxHjAcBgNVBAoTFW1rY2VydCBkZXZlbG9wbWVudCBDQTEtMCsGA1UECwwkbHVj +aW9ATHVjaW9zLVdvcmstTUJQIChMdWNpbyBGcmFuY28pMTQwMgYDVQQDDCtta2Nl +cnQgbHVjaW9ATHVjaW9zLVdvcmstTUJQIChMdWNpbyBGcmFuY28pMB4XDTE5MDky +OTIzMzUzM1oXDTI5MDkyOTIzMzUzM1owgYUxHjAcBgNVBAoTFW1rY2VydCBkZXZl +bG9wbWVudCBDQTEtMCsGA1UECwwkbHVjaW9ATHVjaW9zLVdvcmstTUJQIChMdWNp +byBGcmFuY28pMTQwMgYDVQQDDCtta2NlcnQgbHVjaW9ATHVjaW9zLVdvcmstTUJQ +IChMdWNpbyBGcmFuY28pMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA +y/vE61ItbN/1qMYt13LMf+le1svwfkCCOPsygk7nWeRXmomgUpymqn1LnWiuB0+e +4IdVH2f5E9DknWEpPhKIDMRTCbz4jTwQfHrxCb8EGj3I8oO73pJO5S/xCedM9OrZ +qWcYWwN0GQ8cO/ogazaoZf1uTrRNHyzRyQsKyb412kDBTNEeldJZ2ljKgXXvh4HO +2ZIk9K/ZAaAf6VN8K/89rlJ9/KPgRVNsyAapE+Pb8XXKtpzeFiEcUfuXVYWtkoW+ +xyn/Zu8A1L2CXMQ1sARh7P/42BTMKr5pfraYgcBGxKXLrxoySpxCO9KqeVveKy1q +fPm5FCwFsXDr0koFLrCiR58mcIO/04Q9DKKTV4Z2a+LoqDJRY37KfBSc8sDMPhw5 +k7g3WPoa6QwXRjZTCA5fHWVgLOtcwLsnju5tBE4LDxwF6s+1wPF8NI5yUfufcEjJ +Z6JBwgoWYosVj27Lx7KBNLU/57PX9ryee691zmtswt0tP0WVBAgalhYWg99RXoa3 +AgMBAAGjRTBDMA4GA1UdDwEB/wQEAwICBDASBgNVHRMBAf8ECDAGAQH/AgEAMB0G +A1UdDgQWBBQdvlE4Bdcsjc9oaxjDCRu5FiuZkzANBgkqhkiG9w0BAQsFAAOCAYEA +BP/6o1kPINksMJZSSXgNCPZskDLyGw7auUZBnQ0ocDT3W6gXQvT/27LM1Hxoj9Eh +qU1TYdEt7ppecLQSGvzQ02MExG7H75art75oLiB+A5agDira937YbK4MCjqW481d +bDhw6ixJnY1jIvwjEZxyH6g94YyL927aSPch51fys0kSnjkFzC2RmuzDADScc4XH +5P1+/3dnIm3M5yfpeUzoaOrTXNmhn8p0RDIGrZ5kA5eISIGGD3Mm8FDssUNKndtO +g4ojHUsxb14icnAYGeye1NOhGiqN6TEFcgr6MPd0XdFNZ5c0HUaBCfN6bc+JxDV5 +MKZVJdNeJsYYwilgJNHAyZgCi30JC20xeYVtTF7CEEsMrFDGJ70Kz7o/FnRiFsA1 +ZSwVVWhhkHG2VkT4vlo0O3fYeZpenYicvy+wZNTbGK83gzHWqxxNC1z3Etg5+HRJ +F9qeMWPyfA3IHYXygiMcviyLcyNGG/SJ0EhUpYBN/Gg7wI5yFkcsxUDPPzd23O0M +-----END CERTIFICATE----- diff --git a/crates/remote/tests/__fixtures__/certs/client.key b/crates/remote/tests/__fixtures__/certs/client.key new file mode 100644 index 00000000000..f4d8da2758a --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs/client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCiiWrmzpENsI+c +Cz4aBpG+Pl8WOsrByfZx/ZnJdCZHO3MTYE6sCLhYssf0ygAEEGxvmkd4cxmfCfgf +xuT8u+D7Y5zQSoymkbWdU6/9jbNY6Ovtc+a96I1LGXOKROQw6KR3PuqLpUqEOJiB +l03qK+HMU0g56G1n31Od7HkJsDRvtePqy3I3LgpdcRps23sk46tCzZzhyfqIQ7Qf +J5qZx93tA+pfy+Xtb9XIUTIWKIp1/uyfh8Fp8HA0c9zJCSZzJOX2j3GH1TYqkVgP +egI2lhmdXhP5Q8vdhwy0UJaL28RJXA6UAg0tPZeWJe6pux9JiA81sI6My+Krrw8D +yibkGTTbAgMBAAECggEANCQhRym9HsclSsnQgkjZOE6J8nep08EWbjsMurOoE/He +WLjshAPIH6w6uSyUFLmwD51OkDVcYsiv8IG9s9YRtpOeGrPPqx/TQ0U1kAGFJ2CR +Tvt/aizQJudjSVgQXCBFontsgp/j58bAJdKEDDtHlGSjJvCJKGlcSa0ypwj/yVXt +frjROJNYzw9gMM7fN/IKF/cysdXSeLl/Q9RnHVIfC3jOFJutsILCK8+PC51dM8Fl +IOjmPmiZ080yV8RBcMRECwl53vLOE3OOpR3ZijfNCY1KU8zWi1oELJ1o6f4+cBye +7WPgFEoBew5XHXZ+ke8rh8cc0wth7ZTcC+xC/456AQKBgQDQr2EzBwXxYLF8qsN1 +R4zlzXILLdZN8a4bKfrS507/Gi1gDBHzfvbE7HfljeqrAkbKMdKNkbz3iS85SguH +jsM047xUGJg0PAcwBLHUedlSn1xDDcDHW6X8ginpA2Zz1+WAlhNz6XurA1wnjZmS +VcPxopH7QsuFCclqtt14MbBQ6QKBgQDHY3jcAVfQF+yhQ0YyM6GPLN342aTplgyJ +yz4uWVMeXacU4QzqGbf2L2hc9M2L28Xb37RWC3Q/by0vUefiC6qxRt+GJdRsOuQj +2F1uUibeWtAWp249fcfvxjLib276J+Eit18LI0s0mNR3ekK4GcjSe4NwSq5IrU8e +pBreet3dIwKBgQCxVuil4WkGd+I8jC0v5A7zVsR8hYZhlGkdgm45fgHevdMjlP5I +S3PPYxh8hj6O9o9L0k0Yq2nHfdgYujjUCNkQgBuR55iogv6kqsioRKgPE4fnH6/c +eqCy1bZh4tbUyPqqbF65mQfUCzXsEuQXvDSYiku+F0Q2mVuGCUJpmug3yQKBgEd3 +LeCdUp4xlQ0QEd74hpXM3RrO178pmwDgqj7uoU4m/zYKnBhkc3137I406F+SvE5c +1kRpApeh/64QS27IA7xazM9GS+cnDJKUgJiENY5JOoCELo03wiv8/EwQ6NQc6yMI +WrahRdlqVe0lEzjtdP+MacYb3nAKPmubIk5P96nFAoGAFAyrKpFTyXbNYBTw9Rab +TG6q7qkn+YTHN3+k4mo9NGGwZ3pXvmrKMYCIRhLMbqzsmTbFqCPPIxKsrmf8QYLh +xHYQjrCkbZ0wZdcdeV6yFSDsF218nF/12ZPE7CBOQMfZTCKFNWGL97uIVcmR6K5G +ojTkOvaUnwQtSFhNuzyr23I= +-----END PRIVATE KEY----- diff --git a/crates/remote/tests/__fixtures__/certs/client.pem b/crates/remote/tests/__fixtures__/certs/client.pem new file mode 100644 index 00000000000..bb3b82c40c5 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs/client.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCTCCAfGgAwIBAgIQYbE9d1Rft5h4ku7FSAvWdzANBgkqhkiG9w0BAQsFADAn +MSUwIwYDVQQDExxUb25pYyBFeGFtcGxlIENsaWVudCBSb290IENBMB4XDTE5MTAx +NDEyMzkzNloXDTI0MTAxMjEyMzkzNlowEjEQMA4GA1UEAxMHY2xpZW50MTCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKKJaubOkQ2wj5wLPhoGkb4+XxY6 +ysHJ9nH9mcl0Jkc7cxNgTqwIuFiyx/TKAAQQbG+aR3hzGZ8J+B/G5Py74PtjnNBK +jKaRtZ1Tr/2Ns1jo6+1z5r3ojUsZc4pE5DDopHc+6oulSoQ4mIGXTeor4cxTSDno +bWffU53seQmwNG+14+rLcjcuCl1xGmzbeyTjq0LNnOHJ+ohDtB8nmpnH3e0D6l/L +5e1v1chRMhYoinX+7J+HwWnwcDRz3MkJJnMk5faPcYfVNiqRWA96AjaWGZ1eE/lD +y92HDLRQlovbxElcDpQCDS09l5Yl7qm7H0mIDzWwjozL4quvDwPKJuQZNNsCAwEA +AaNGMEQwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAfBgNVHSME +GDAWgBQV1YOR+Jpl1fbujvWLSBEoRvsDhTANBgkqhkiG9w0BAQsFAAOCAQEAfTPu +KeHXmyVTSCUrYQ1X5Mu7VzfZlRbhoytHOw7bYGgwaFwQj+ZhlPt8nFC22/bEk4IV +AoCOli0WyPIB7Lx52dZ+v9JmYOK6ca2Aa/Dkw8Q+M3XA024FQWq3nZ6qANKC32/9 +Nk+xOcb1Qd/11stpTkRf2Oj7F7K4GnlFbY6iMyNW+RFXGKEbL5QAJDTDPIT8vw1x +oYeNPwmC042uEboCZPNXmuctiK9Wt1TAxjZT/cwdIBGGJ+xrW72abfJGs7bUcJfc +O4r9V0xVv+X0iKWTW0fwd9qjNfiEP1tFCcZb2XsNQPe/DlQZ+h98P073tZEsWI/G +KJrFspGX8vOuSdIeqw== +-----END CERTIFICATE----- diff --git a/crates/remote/tests/__fixtures__/certs/server.key b/crates/remote/tests/__fixtures__/certs/server.key new file mode 100644 index 00000000000..80984ef9000 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDyptbMyYWztgta +t1MXLMzIkaQdeeVbs1Y/qCpAdwZe/Y5ZpbzjGIjCxbB6vNRSnEbYKpytKHPzYfM7 +8d8K8bPvpnqXIiTXFT0JQlw1OHLC1fr4e598GJumAmpMYFrtqv0fbmUFTuQGbHxe +OH2vji0bvr3NKZubMfkEZP3X4sNXXoXIuW2LaS8OMGKoJaeCBvdbszEiSGj/v9Bj +pM0yLTH89NNMX1T+FtTKnuXag5g7pr6lzJj83+MzAGy4nOjseSuUimuiyG90/C5t +A5wC0Qh5RbDnkFYhC44Kxof/i6+jnfateIPNiIIwQV+2f6G/aK1hgjekT10m/eoR +YDTf+e5ZAgMBAAECggEACODt7yRYjhDVLYaTtb9f5t7dYG67Y7WWLFIc6arxQryI +XuNfm/ej2WyeXn9WTYeGWBaHERbv1zH4UnMxNBdP/C7dQXZwXqZaS2JwOUpNeK+X +tUvgtAu6dkKUXSMRcKzXAjVp4N3YHhwOGOx8PNY49FDwZPdmyDD16aFAYIvdle6/ +PSMrj38rB1sbQQdmRob2FjJBSDZ44nsr+/nilrcOFNfNnWv7tQIWYVXNcLfdK/WJ +ZCDFhA8lr/Yon6MEq6ApTj2ZYRRGXPd6UeASJkmTZEUIUbeDcje/MO8cHkREpuRH +wm3pCjR7OdO4vc+/d/QmEvu5ns6wbTauelYnL616YQKBgQD414gJtpCHauNEUlFB +v/R3DzPI5NGp9PAqovOD8nCbI49Mw61gP/ExTIPKiR5uUX/5EL04uspaNkuohXk+ +ys0G5At0NfV7W39lzhvALEaSfleybvYxppbBrc20/q8Gvi/i30NY+1LM3RdtMiEw +hKHjU0SnFhJq0InFg3AO/iCeTQKBgQD5obkbzpOidSsa55aNsUlO2qjiUY9leq9b +irAohIZ8YnuuixYvkOeSeSz1eIrA4tECeAFSgTZxYe1Iz+USru2Xg/0xNte11dJD +rBoH/yMn2gDvBK7xQ6uFMPTeYtKG0vfvpXZYSWZzGntyrHTwFk6UV+xdrt9MBdd1 +XdSn7bwOPQKBgC9VQAko8uDvUf+C8PXiv2uONrl13PPJJY3WpR9qFEVOREnDxszS +HNzVwxPZdTJiykbkCjoqPadfQJDzopZxGQLAifU29lTamKcSx3CMe3gOFDxaovXa +zD5XAxP0hfJwZsdu1G6uj5dsTrJ0oJ+L+wc0pZBqwGIU/L/XOo9/g1DZAoGAUebL +kuH98ik7EUK2VJq8EJERI9/ailLsQb6I+WIxtZGiPqwHhWencpkrNQZtj8dbB9JT +rLwUHrMgZOlAoRafgTyez4zMzS3wJJ/Mkp8U67hM4h7JPwMSvUpIrMYDiJSjIA9L +er/qSw1/Pypx22uWMHmAZWRAgvLPtAQrB0Wqk4kCgYEAr2H1PvfbwZwkSvlMt5o8 +WLnBbxcM3AKglLRbkShxxgiZYdEP71/uOtRMiL26du5XX8evItITN0DsvmXL/kcd +h29LK7LM5uLw7efz0Qxs03G6kEyIHVkacowHi5I5Ul1qI61SoV3yMB1TjIU+bXZt +0ZjC07totO0fqPOLQxonjQg= +-----END PRIVATE KEY----- diff --git a/crates/remote/tests/__fixtures__/certs/server.pem b/crates/remote/tests/__fixtures__/certs/server.pem new file mode 100644 index 00000000000..4cc97bcf4b6 --- /dev/null +++ b/crates/remote/tests/__fixtures__/certs/server.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEmDCCAwCgAwIBAgIQVEJFCgU/CZk9JEwTucWPpzANBgkqhkiG9w0BAQsFADCB +hTEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMS0wKwYDVQQLDCRsdWNp +b0BMdWNpb3MtV29yay1NQlAgKEx1Y2lvIEZyYW5jbykxNDAyBgNVBAMMK21rY2Vy +dCBsdWNpb0BMdWNpb3MtV29yay1NQlAgKEx1Y2lvIEZyYW5jbykwHhcNMTkwNjAx +MDAwMDAwWhcNMjkwOTI5MjMzNTM0WjBYMScwJQYDVQQKEx5ta2NlcnQgZGV2ZWxv +cG1lbnQgY2VydGlmaWNhdGUxLTArBgNVBAsMJGx1Y2lvQEx1Y2lvcy1Xb3JrLU1C +UCAoTHVjaW8gRnJhbmNvKTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +APKm1szJhbO2C1q3UxcszMiRpB155VuzVj+oKkB3Bl79jlmlvOMYiMLFsHq81FKc +RtgqnK0oc/Nh8zvx3wrxs++mepciJNcVPQlCXDU4csLV+vh7n3wYm6YCakxgWu2q +/R9uZQVO5AZsfF44fa+OLRu+vc0pm5sx+QRk/dfiw1dehci5bYtpLw4wYqglp4IG +91uzMSJIaP+/0GOkzTItMfz000xfVP4W1Mqe5dqDmDumvqXMmPzf4zMAbLic6Ox5 +K5SKa6LIb3T8Lm0DnALRCHlFsOeQViELjgrGh/+Lr6Od9q14g82IgjBBX7Z/ob9o +rWGCN6RPXSb96hFgNN/57lkCAwEAAaOBrzCBrDAOBgNVHQ8BAf8EBAMCBaAwEwYD +VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBQdvlE4 +Bdcsjc9oaxjDCRu5FiuZkzBWBgNVHREETzBNggtleGFtcGxlLmNvbYINKi5leGFt +cGxlLmNvbYIMZXhhbXBsZS50ZXN0gglsb2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQELBQADggGBAKb2TJ8l+e1eraNwZWizLw5fccAf +y59J1JAWdLxZyAI/bkiTlVO3DQoPZpw7XwLhefCvILkwKAL4TtIGGVC9yTb5Q5eg +rqGO3FC0yg1fn65Kf1VpVxxUVyoiM5PQ4pFJb4AicAv88rCOLD9FFuE0PKOKU/dm +Tw0WgPStoh9wsJ1RXUuTJYZs1nd1kMBlfv9NbLilnL+cR2sLktS54X5XagsBYVlf +oapRb0JtABOoQhX3U8QMq8UF8yzceRHNTN9yfLOUrW26s9nKtlWVniNhw1uPxZw9 +RHM7w9/4+a9LXtEDYg4IP/1mm0ywBoUqy1O6hA73uId+Yi/kFBks/GyYaGjKgYcO +23B75tkPGYEdGuGZYLzZNHbXg4V0UxFQG3KA1pUiSnD3bN2Rxs+CMpzORnOeK3xi +EooKgAPYsehItoQOMPpccI2xHdSAMWtwUgOKrefUQujkx2Op+KFlspF0+WJ6AZEe +2D4hyWaEZsvvILXapwqHDCuN3/jSUlTIqUoE1w== +-----END CERTIFICATE----- diff --git a/crates/task-runner/Cargo.toml b/crates/task-runner/Cargo.toml index 2dcea2b2d87..5b6edfb731a 100644 --- a/crates/task-runner/Cargo.toml +++ b/crates/task-runner/Cargo.toml @@ -22,13 +22,14 @@ moon_console = { path = "../console" } moon_platform = { path = "../../legacy/core/platform" } moon_process = { path = "../process" } moon_project = { path = "../project" } +moon_remote = { path = "../remote" } moon_task = { path = "../task" } moon_task_hasher = { path = "../task-hasher" } moon_time = { path = "../time" } miette = { workspace = true } serde = { workspace = true } starbase_archive = { workspace = true } -starbase_utils = { workspace = true, features = ["glob"] } +starbase_utils = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } diff --git a/crates/task-runner/src/output_archiver.rs b/crates/task-runner/src/output_archiver.rs index c75f5ab01fe..b65330f05a1 100644 --- a/crates/task-runner/src/output_archiver.rs +++ b/crates/task-runner/src/output_archiver.rs @@ -1,12 +1,13 @@ use crate::task_runner_error::TaskRunnerError; +use moon_action::Operation; use moon_api::Moonbase; use moon_app_context::AppContext; use moon_common::color; -use moon_config::ProjectConfig; +use moon_project::Project; +use moon_remote::{compute_digests_for_outputs, Digest, RemoteService}; use moon_task::{TargetError, TargetScope, Task}; use starbase_archive::tar::TarPacker; use starbase_archive::Archiver; -use starbase_utils::glob; use std::path::{Path, PathBuf}; use tracing::{debug, instrument, warn}; @@ -15,13 +16,17 @@ use tracing::{debug, instrument, warn}; /// can be hydrated easily. pub struct OutputArchiver<'task> { pub app: &'task AppContext, - pub project_config: &'task ProjectConfig, + pub project: &'task Project, pub task: &'task Task, } impl<'task> OutputArchiver<'task> { - #[instrument(skip(self))] - pub async fn archive(&self, hash: &str) -> miette::Result> { + #[instrument(skip(self, operation))] + pub async fn archive( + &self, + digest: &Digest, + operation: Option<&Operation>, + ) -> miette::Result> { if !self.is_archivable()? { return Ok(None); } @@ -35,31 +40,41 @@ impl<'task> OutputArchiver<'task> { } // If so, create and pack the archive! - let archive_file = self.app.cache_engine.hash.get_archive_path(hash); + let archive_file = self.app.cache_engine.hash.get_archive_path(&digest.hash); if !archive_file.exists() { - if !self.app.cache_engine.is_writable() { + if self.app.cache_engine.is_writable() { debug!( task_target = self.task.target.as_str(), - hash, "Cache is not writable, skipping output archiving" + hash = &digest.hash, + "Archiving task outputs from project" ); - return Ok(None); - } - - debug!( - task_target = self.task.target.as_str(), - hash, "Archiving task outputs from project" - ); + self.create_local_archive(&digest.hash, &archive_file)?; - self.create_local_archive(hash, &archive_file)?; - - if archive_file.exists() { - self.upload_to_remote_storage(hash, &archive_file).await?; + if archive_file.exists() { + self.upload_to_remote_storage(&digest.hash, &archive_file) + .await?; + } + } else { + debug!( + task_target = self.task.target.as_str(), + hash = &digest.hash, + "Cache is not writable, skipping output archiving" + ); } } - Ok(Some(archive_file)) + // Then cache the result in the remote service + if let Some(operation) = operation { + self.upload_to_remote_service(digest, operation).await?; + } + + Ok(if archive_file.exists() { + Some(archive_file) + } else { + None + }) } pub fn is_archivable(&self) -> miette::Result { @@ -86,7 +101,7 @@ impl<'task> OutputArchiver<'task> { } } TargetScope::Tag(tag_id) => { - if self.project_config.tags.contains(tag_id) && is_matching_task { + if self.project.config.tags.contains(tag_id) && is_matching_task { return Ok(true); } } @@ -124,7 +139,9 @@ impl<'task> OutputArchiver<'task> { // If all globs are negated, then the empty check will always // fail, resulting in archives not being created if has_globs && !all_negated_globs { - let outputs = glob::walk_files(&self.app.workspace_root, &self.task.output_globs)?; + let outputs = self + .task + .get_output_files(&self.app.workspace_root, false)?; return Ok(!outputs.is_empty()); } @@ -133,7 +150,7 @@ impl<'task> OutputArchiver<'task> { } #[instrument(skip(self))] - pub fn create_local_archive(&self, hash: &str, archive_file: &Path) -> miette::Result<()> { + fn create_local_archive(&self, hash: &str, archive_file: &Path) -> miette::Result<()> { debug!( task_target = self.task.target.as_str(), hash, @@ -177,7 +194,7 @@ impl<'task> OutputArchiver<'task> { } #[instrument(skip(self))] - pub async fn upload_to_remote_storage( + async fn upload_to_remote_storage( &self, hash: &str, archive_file: &Path, @@ -190,4 +207,24 @@ impl<'task> OutputArchiver<'task> { Ok(()) } + + #[instrument(skip(self, operation))] + async fn upload_to_remote_service( + &self, + digest: &Digest, + operation: &Operation, + ) -> miette::Result<()> { + if let Some(remote) = RemoteService::session() { + let output_digests = compute_digests_for_outputs( + self.task.get_output_files(&self.app.workspace_root, true)?, + &self.app.workspace_root, + )?; + + remote + .save_operation_with_outputs(digest, operation, output_digests) + .await?; + } + + Ok(()) + } } diff --git a/crates/task-runner/src/output_hydrater.rs b/crates/task-runner/src/output_hydrater.rs index 4eed21eface..7e500783781 100644 --- a/crates/task-runner/src/output_hydrater.rs +++ b/crates/task-runner/src/output_hydrater.rs @@ -1,6 +1,9 @@ +use crate::run_state::read_stdlog_state_files; +use moon_action::Operation; use moon_api::Moonbase; use moon_app_context::AppContext; use moon_common::color; +use moon_remote::{Digest, RemoteService}; use moon_task::Task; use starbase_archive::tar::TarUnpacker; use starbase_archive::Archiver; @@ -11,6 +14,7 @@ use tracing::{debug, instrument, warn}; #[derive(Clone, Copy, Debug, PartialEq)] pub enum HydrateFrom { LocalCache, + Moonbase, PreviousOutput, RemoteCache, } @@ -21,46 +25,67 @@ pub struct OutputHydrater<'task> { } impl<'task> OutputHydrater<'task> { - #[instrument(skip(self))] - pub async fn hydrate(&self, hash: &str, from: HydrateFrom) -> miette::Result { - // Only hydrate when the hash is different from the previous build, - // as we can assume the outputs from the previous build still exist? - if matches!(from, HydrateFrom::PreviousOutput) { - return Ok(true); - } - - let archive_file = self.app.cache_engine.hash.get_archive_path(hash); - - if self.app.cache_engine.is_readable() { - debug!( - task_target = self.task.target.as_str(), - hash, "Hydrating cached outputs into project" - ); - - // Attempt to download from remote cache to `.moon/outputs/` - if !archive_file.exists() && matches!(from, HydrateFrom::RemoteCache) { - self.download_from_remote_storage(hash, &archive_file) - .await?; - } - - // Otherwise hydrate the cached archive into the task's outputs - if archive_file.exists() { - self.unpack_local_archive(hash, &archive_file)?; - - return Ok(true); + #[instrument(skip(self, operation))] + pub async fn hydrate( + &self, + from: HydrateFrom, + digest: &Digest, + operation: &mut Operation, + ) -> miette::Result { + match from { + // Only hydrate when the hash is different from the previous build, + // as we can assume the outputs from the previous build still exist? + HydrateFrom::PreviousOutput => Ok(true), + + // Based on the remote execution APIs + HydrateFrom::RemoteCache => self.download_from_remote_service(digest, operation).await, + + // Otherwise write to local cache, then download archive from moonbase + HydrateFrom::LocalCache | HydrateFrom::Moonbase => { + let archive_file = self.app.cache_engine.hash.get_archive_path(&digest.hash); + + if self.app.cache_engine.is_readable() { + debug!( + task_target = self.task.target.as_str(), + hash = &digest.hash, + "Hydrating cached outputs into project" + ); + + // Attempt to download from remote cache to `.moon/outputs/` + if !archive_file.exists() && matches!(from, HydrateFrom::Moonbase) { + self.download_from_remote_storage(&digest.hash, &archive_file) + .await?; + } + + // Otherwise hydrate the cached archive into the task's outputs + if archive_file.exists() { + self.unpack_local_archive(&digest.hash, &archive_file)?; + + read_stdlog_state_files( + self.app + .cache_engine + .state + .get_target_dir(&self.task.target), + operation, + )?; + + return Ok(true); + } + } else { + debug!( + task_target = self.task.target.as_str(), + hash = &digest.hash, + "Cache is not readable, skipping output hydration" + ); + } + + Ok(false) } - } else { - debug!( - task_target = self.task.target.as_str(), - hash, "Cache is not readable, skipping output hydration" - ); } - - Ok(false) } #[instrument(skip(self))] - pub fn unpack_local_archive(&self, hash: &str, archive_file: &Path) -> miette::Result { + fn unpack_local_archive(&self, hash: &str, archive_file: &Path) -> miette::Result { debug!( task_target = self.task.target.as_str(), hash, @@ -98,7 +123,7 @@ impl<'task> OutputHydrater<'task> { } #[instrument(skip(self))] - pub async fn download_from_remote_storage( + async fn download_from_remote_storage( &self, hash: &str, archive_file: &Path, @@ -111,4 +136,19 @@ impl<'task> OutputHydrater<'task> { Ok(()) } + + #[instrument(skip(self, operation))] + async fn download_from_remote_service( + &self, + digest: &Digest, + operation: &mut Operation, + ) -> miette::Result { + if let Some(remote) = RemoteService::session() { + remote.restore_operation(digest, operation).await?; + + return Ok(true); + } + + Ok(false) + } } diff --git a/crates/task-runner/src/run_state.rs b/crates/task-runner/src/run_state.rs index 6cb7d9d83e2..07d2abbc0a8 100644 --- a/crates/task-runner/src/run_state.rs +++ b/crates/task-runner/src/run_state.rs @@ -1,4 +1,7 @@ +use moon_action::Operation; use moon_cache_item::cache_item; +use starbase_utils::fs; +use std::path::PathBuf; cache_item!( pub struct TaskRunCacheState { @@ -8,3 +11,54 @@ cache_item!( pub target: String, } ); + +pub fn read_stdlog_state_files( + state_dir: PathBuf, + operation: &mut Operation, +) -> miette::Result<()> { + if let Some(output) = operation.get_output_mut() { + let err_path = state_dir.join("stderr.log"); + let out_path = state_dir.join("stdout.log"); + + if err_path.exists() { + output.set_stderr(fs::read_file(err_path)?); + } + + if out_path.exists() { + output.set_stdout(fs::read_file(out_path)?); + } + } + + Ok(()) +} + +pub fn write_stdlog_state_files(state_dir: PathBuf, operation: &Operation) -> miette::Result<()> { + let err_path = state_dir.join("stderr.log"); + let out_path = state_dir.join("stdout.log"); + + if let Some(output) = operation.get_output() { + fs::write_file( + err_path, + output + .stderr + .as_ref() + .map(|log| log.as_bytes()) + .unwrap_or_default(), + )?; + + fs::write_file( + out_path, + output + .stdout + .as_ref() + .map(|log| log.as_bytes()) + .unwrap_or_default(), + )?; + } else { + // Ensure logs from a previous run are removed + fs::remove_file(err_path)?; + fs::remove_file(out_path)?; + } + + Ok(()) +} diff --git a/crates/task-runner/src/task_runner.rs b/crates/task-runner/src/task_runner.rs index 831715f2354..8faeb81be2e 100644 --- a/crates/task-runner/src/task_runner.rs +++ b/crates/task-runner/src/task_runner.rs @@ -2,7 +2,7 @@ use crate::command_builder::CommandBuilder; use crate::command_executor::CommandExecutor; use crate::output_archiver::OutputArchiver; use crate::output_hydrater::{HydrateFrom, OutputHydrater}; -use crate::run_state::TaskRunCacheState; +use crate::run_state::*; use crate::task_runner_error::TaskRunnerError; use moon_action::{ActionNode, ActionStatus, Operation, OperationList, OperationMeta}; use moon_action_context::{ActionContext, TargetState}; @@ -13,6 +13,7 @@ use moon_console::TaskReportItem; use moon_platform::PlatformManager; use moon_process::ProcessError; use moon_project::Project; +use moon_remote::{Digest, RemoteService}; use moon_task::Task; use moon_task_hasher::TaskHasher; use moon_time::{is_stale, now_millis}; @@ -38,6 +39,7 @@ pub struct TaskRunner<'task> { hydrater: OutputHydrater<'task>, // Public for testing + pub action_digest: Digest, pub cache: CacheItem, pub operations: OperationList, pub report_item: TaskReportItem, @@ -65,10 +67,10 @@ impl<'task> TaskRunner<'task> { Ok(Self { cache, - archiver: OutputArchiver { - app, - project_config: &project.config, - task, + archiver: OutputArchiver { app, project, task }, + action_digest: Digest { + hash: String::new(), + size_bytes: 0, }, hydrater: OutputHydrater { app, task }, platform_manager: PlatformManager::read(), @@ -303,6 +305,18 @@ impl<'task> TaskRunner<'task> { return Ok(Some(HydrateFrom::LocalCache)); } + // Check if the outputs have been cached in the remote service + if let Some(remote) = RemoteService::session() { + if remote.is_operation_cached(&self.action_digest).await? { + debug!( + task_target = self.task.target.as_str(), + hash, "Cache hit in remote service, will attempt to download output blobs" + ); + + return Ok(Some(HydrateFrom::RemoteCache)); + } + } + // Check if archive exists in moonbase (remote storage) by querying the artifacts // endpoint. This only checks that the database record exists! if let Some(moonbase) = Moonbase::session() { @@ -314,7 +328,7 @@ impl<'task> TaskRunner<'task> { "Cache hit in remote cache, will attempt to download the archive" ); - return Ok(Some(HydrateFrom::RemoteCache)); + return Ok(Some(HydrateFrom::Moonbase)); } } @@ -374,8 +388,9 @@ impl<'task> TaskRunner<'task> { "Generating a unique hash for this task" ); + let hash_engine = &self.app.cache_engine.hash; + let mut hasher = hash_engine.create_hasher(node.label()); let mut operation = Operation::hash_generation(); - let mut hasher = self.app.cache_engine.hash.create_hasher(node.label()); // Hash common fields trace!( @@ -436,13 +451,18 @@ impl<'task> TaskRunner<'task> { ) .await?; - let hash = self.app.cache_engine.hash.save_manifest(hasher)?; + let (hash, size_bytes) = hash_engine.save_manifest(hasher)?; operation.meta.set_hash(&hash); operation.finish(ActionStatus::Passed); self.operations.push(operation); + self.action_digest = Digest { + hash: hash.clone(), + size_bytes: size_bytes as i64, + }; + debug!( task_target = self.task.target.as_str(), hash = &hash, @@ -509,7 +529,7 @@ impl<'task> TaskRunner<'task> { }; if let Some(last_attempt) = result.attempts.get_last_execution() { - self.save_logs(last_attempt)?; + self.persist_state(last_attempt)?; } // Extract the attempts from the result @@ -585,7 +605,11 @@ impl<'task> TaskRunner<'task> { "Running cache archiving operation" ); - let archived = match self.archiver.archive(hash).await? { + let archived = match self + .archiver + .archive(&self.action_digest, self.operations.get_last_execution()) + .await? + { Some(archive_file) => { debug!( task_target = self.task.target.as_str(), @@ -618,11 +642,6 @@ impl<'task> TaskRunner<'task> { pub async fn hydrate(&mut self, context: &ActionContext, hash: &str) -> miette::Result { let mut operation = Operation::output_hydration(); - debug!( - task_target = self.task.target.as_str(), - "Running cache hydration operation" - ); - // Not cached let Some(from) = self.is_cached(hash).await? else { debug!( @@ -638,7 +657,17 @@ impl<'task> TaskRunner<'task> { }; // Did not hydrate - if !self.hydrater.hydrate(hash, from).await? { + debug!( + task_target = self.task.target.as_str(), + hydrate_from = ?from, + "Running cache hydration operation" + ); + + if !self + .hydrater + .hydrate(from, &self.action_digest, &mut operation) + .await? + { debug!(task_target = self.task.target.as_str(), "Did not hydrate"); operation.finish(ActionStatus::Invalid); @@ -657,13 +686,19 @@ impl<'task> TaskRunner<'task> { // Fill in these values since the command executor does not run! self.report_item.output_prefix = Some(context.get_target_prefix(&self.task.target)); - self.load_logs(&mut operation)?; + if let Some(output) = operation.get_output_mut() { + output.command = Some(self.task.get_command_line()); + output.exit_code = Some(self.cache.data.exit_code); + } + // Then finalize the operation and target state operation.finish(match from { - HydrateFrom::RemoteCache => ActionStatus::CachedFromRemote, + HydrateFrom::Moonbase | HydrateFrom::RemoteCache => ActionStatus::CachedFromRemote, _ => ActionStatus::Cached, }); + self.persist_state(&operation)?; + context.set_target_state(&self.task.target, TargetState::Passed(hash.to_owned())); self.operations.push(operation); @@ -709,63 +744,17 @@ impl<'task> TaskRunner<'task> { Ok(()) } - fn load_logs(&self, operation: &mut Operation) -> miette::Result<()> { - if let Some(output) = operation.get_output_mut() { - let state_dir = self - .app + fn persist_state(&mut self, operation: &Operation) -> miette::Result<()> { + write_stdlog_state_files( + self.app .cache_engine .state - .get_target_dir(&self.task.target); - let err_path = state_dir.join("stderr.log"); - let out_path = state_dir.join("stdout.log"); - - output.exit_code = Some(self.cache.data.exit_code); - - if err_path.exists() { - output.set_stderr(fs::read_file(err_path)?); - } - - if out_path.exists() { - output.set_stdout(fs::read_file(out_path)?); - } - } - - Ok(()) - } - - fn save_logs(&mut self, operation: &Operation) -> miette::Result<()> { - let state_dir = self - .app - .cache_engine - .state - .get_target_dir(&self.task.target); - let err_path = state_dir.join("stderr.log"); - let out_path = state_dir.join("stdout.log"); + .get_target_dir(&self.task.target), + operation, + )?; if let Some(output) = operation.get_output() { self.cache.data.exit_code = output.get_exit_code(); - - fs::write_file( - err_path, - output - .stderr - .as_ref() - .map(|log| log.as_bytes()) - .unwrap_or_default(), - )?; - - fs::write_file( - out_path, - output - .stdout - .as_ref() - .map(|log| log.as_bytes()) - .unwrap_or_default(), - )?; - } else { - // Ensure logs from a previous run are removed - fs::remove_file(err_path)?; - fs::remove_file(out_path)?; } Ok(()) diff --git a/crates/task-runner/tests/output_archiver_test.rs b/crates/task-runner/tests/output_archiver_test.rs index 547dc7c099a..318d253e134 100644 --- a/crates/task-runner/tests/output_archiver_test.rs +++ b/crates/task-runner/tests/output_archiver_test.rs @@ -1,6 +1,7 @@ mod utils; use moon_cache::CacheMode; +use moon_remote::Digest; use moon_task::Target; use starbase_archive::Archiver; use std::env; @@ -8,6 +9,13 @@ use std::fs; use std::sync::Arc; use utils::*; +fn stub_digest() -> Digest { + Digest { + hash: "hash123".into(), + size_bytes: 0, + } +} + mod output_archiver { use super::*; @@ -18,8 +26,9 @@ mod output_archiver { async fn does_nothing_if_no_outputs_in_task() { let container = TaskRunnerContainer::new("archive", "no-outputs").await; let archiver = container.create_archiver(); + let digest = stub_digest(); - assert!(archiver.archive("hash123").await.unwrap().is_none()); + assert!(archiver.archive(&digest, None).await.unwrap().is_none()); } #[tokio::test] @@ -27,8 +36,9 @@ mod output_archiver { async fn errors_if_outputs_not_created() { let container = TaskRunnerContainer::new("archive", "file-outputs").await; let archiver = container.create_archiver(); + let digest = stub_digest(); - archiver.archive("hash123").await.unwrap(); + archiver.archive(&digest, None).await.unwrap(); } #[tokio::test] @@ -37,8 +47,9 @@ mod output_archiver { container.sandbox.create_file("project/file.txt", ""); let archiver = container.create_archiver(); + let digest = stub_digest(); - assert!(archiver.archive("hash123").await.unwrap().is_some()); + assert!(archiver.archive(&digest, None).await.unwrap().is_some()); assert!(container .sandbox .path() @@ -55,7 +66,9 @@ mod output_archiver { .create_file(".moon/cache/outputs/hash123.tar.gz", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); assert_eq!(fs::metadata(file).unwrap().len(), 0); } @@ -66,13 +79,14 @@ mod output_archiver { container.sandbox.create_file("project/file.txt", ""); let archiver = container.create_archiver(); + let digest = stub_digest(); container .app_context .cache_engine .force_mode(CacheMode::Off); - assert!(archiver.archive("hash123").await.unwrap().is_none()); + assert!(archiver.archive(&digest, None).await.unwrap().is_none()); env::remove_var("MOON_CACHE"); } @@ -83,13 +97,14 @@ mod output_archiver { container.sandbox.create_file("project/file.txt", ""); let archiver = container.create_archiver(); + let digest = stub_digest(); container .app_context .cache_engine .force_mode(CacheMode::Read); - assert!(archiver.archive("hash123").await.unwrap().is_none()); + assert!(archiver.archive(&digest, None).await.unwrap().is_none()); env::remove_var("MOON_CACHE"); } @@ -100,7 +115,9 @@ mod output_archiver { container.sandbox.create_file("project/file.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); @@ -116,7 +133,9 @@ mod output_archiver { container.sandbox.create_file("project/three.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); @@ -138,7 +157,9 @@ mod output_archiver { container.sandbox.create_file("project/file.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); @@ -160,7 +181,9 @@ mod output_archiver { container.sandbox.create_file("project/c.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); @@ -178,7 +201,9 @@ mod output_archiver { container.sandbox.create_file("project/c.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); @@ -194,7 +219,9 @@ mod output_archiver { container.sandbox.create_file("project/file.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); @@ -210,7 +237,9 @@ mod output_archiver { container.sandbox.create_file("project/c.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); @@ -226,7 +255,9 @@ mod output_archiver { container.sandbox.create_file("project/dir/file.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); @@ -242,7 +273,9 @@ mod output_archiver { container.sandbox.create_file("project/c/file.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); @@ -259,7 +292,9 @@ mod output_archiver { container.sandbox.create_file("project/dir/file.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); @@ -276,7 +311,9 @@ mod output_archiver { container.sandbox.create_file("shared/z.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); @@ -294,7 +331,9 @@ mod output_archiver { container.sandbox.create_file("project/file.txt", ""); let archiver = container.create_archiver(); - let file = archiver.archive("hash123").await.unwrap().unwrap(); + let digest = stub_digest(); + + let file = archiver.archive(&digest, None).await.unwrap().unwrap(); let dir = container.sandbox.path().join("out"); Archiver::new(&dir, &file).unpack_from_ext().unwrap(); diff --git a/crates/task-runner/tests/output_hydrater_test.rs b/crates/task-runner/tests/output_hydrater_test.rs index ad18c3d1625..82b8fe2b1fa 100644 --- a/crates/task-runner/tests/output_hydrater_test.rs +++ b/crates/task-runner/tests/output_hydrater_test.rs @@ -1,31 +1,46 @@ mod utils; +use moon_action::Operation; use moon_cache::CacheMode; +use moon_remote::Digest; use moon_task_runner::output_hydrater::HydrateFrom; use std::env; use utils::*; +fn stub_digest() -> Digest { + Digest { + hash: "hash123".into(), + size_bytes: 0, + } +} + +fn stub_operation() -> Operation { + Operation::output_hydration() +} + mod output_hydrater { use super::*; mod unpack { use super::*; - #[tokio::test] - async fn does_nothing_if_no_hash() { - let container = TaskRunnerContainer::new("archive", "file-outputs").await; - let hydrater = container.create_hydrator(); + // #[tokio::test] + // async fn does_nothing_if_no_hash() { + // let container = TaskRunnerContainer::new("archive", "file-outputs").await; + // let hydrater = container.create_hydrator(); - assert!(!hydrater.hydrate("", HydrateFrom::LocalCache).await.unwrap()); - } + // assert!(!hydrater.hydrate("", HydrateFrom::LocalCache).await.unwrap()); + // } #[tokio::test] async fn does_nothing_if_from_prev_outputs() { let container = TaskRunnerContainer::new("archive", "file-outputs").await; let hydrater = container.create_hydrator(); + let digest = stub_digest(); + let mut operation = stub_operation(); assert!(hydrater - .hydrate("hash123", HydrateFrom::PreviousOutput) + .hydrate(HydrateFrom::PreviousOutput, &digest, &mut operation) .await .unwrap()); } @@ -38,6 +53,8 @@ mod output_hydrater { .create_file(".moon/cache/outputs/hash123.tar.gz", ""); let hydrater = container.create_hydrator(); + let digest = stub_digest(); + let mut operation = stub_operation(); container .app_context @@ -45,7 +62,7 @@ mod output_hydrater { .force_mode(CacheMode::Off); assert!(!hydrater - .hydrate("hash123", HydrateFrom::LocalCache) + .hydrate(HydrateFrom::LocalCache, &digest, &mut operation) .await .unwrap()); @@ -60,6 +77,8 @@ mod output_hydrater { .create_file(".moon/cache/outputs/hash123.tar.gz", ""); let hydrater = container.create_hydrator(); + let digest = stub_digest(); + let mut operation = stub_operation(); container .app_context @@ -67,7 +86,7 @@ mod output_hydrater { .force_mode(CacheMode::Write); assert!(!hydrater - .hydrate("hash123", HydrateFrom::LocalCache) + .hydrate(HydrateFrom::LocalCache, &digest, &mut operation) .await .unwrap()); @@ -82,8 +101,11 @@ mod output_hydrater { assert!(!container.sandbox.path().join("project/file.txt").exists()); let hydrater = container.create_hydrator(); + let digest = stub_digest(); + let mut operation = stub_operation(); + hydrater - .hydrate("hash123", HydrateFrom::LocalCache) + .hydrate(HydrateFrom::LocalCache, &digest, &mut operation) .await .unwrap(); @@ -102,8 +124,11 @@ mod output_hydrater { .exists()); let hydrater = container.create_hydrator(); + let digest = stub_digest(); + let mut operation = stub_operation(); + hydrater - .hydrate("hash123", HydrateFrom::LocalCache) + .hydrate(HydrateFrom::LocalCache, &digest, &mut operation) .await .unwrap(); diff --git a/crates/task-runner/tests/task_runner_test.rs b/crates/task-runner/tests/task_runner_test.rs index 925ac315d49..b4aa77a9dcd 100644 --- a/crates/task-runner/tests/task_runner_test.rs +++ b/crates/task-runner/tests/task_runner_test.rs @@ -3,6 +3,7 @@ mod utils; use moon_action::ActionStatus; use moon_action_context::*; use moon_cache::CacheMode; +use moon_remote::Digest; use moon_task::Target; use moon_task_runner::output_hydrater::HydrateFrom; use moon_task_runner::TaskRunner; @@ -10,6 +11,13 @@ use moon_time::now_millis; use std::env; use utils::*; +fn stub_digest() -> Digest { + Digest { + hash: "hash123".into(), + size_bytes: 0, + } +} + mod task_runner { use super::*; @@ -699,6 +707,11 @@ mod task_runner { mod execute { use super::*; + fn setup_exec_state(runner: &mut TaskRunner) { + runner.report_item.hash = Some("hash123".into()); + runner.action_digest = stub_digest(); + } + #[tokio::test] async fn executes_and_sets_success_state() { let container = TaskRunnerContainer::new_os("runner", "success").await; @@ -708,7 +721,8 @@ mod task_runner { let node = container.create_action_node(); let context = ActionContext::default(); - runner.report_item.hash = Some("hash123".into()); + setup_exec_state(&mut runner); + runner.execute(&context, &node).await.unwrap(); assert_eq!( @@ -751,8 +765,9 @@ mod task_runner { let node = container.create_action_node(); let context = ActionContext::default(); + setup_exec_state(&mut runner); + // Swallow panic so we can check operations - runner.report_item.hash = Some("hash123".into()); let _ = runner.execute(&context, &node).await; assert_eq!( @@ -774,7 +789,8 @@ mod task_runner { let node = container.create_action_node(); let context = ActionContext::default(); - runner.report_item.hash = Some("hash123".into()); + setup_exec_state(&mut runner); + runner.execute(&context, &node).await.unwrap(); let operation = runner.operations.last().unwrap(); @@ -797,8 +813,9 @@ mod task_runner { let node = container.create_action_node(); let context = ActionContext::default(); + setup_exec_state(&mut runner); + // Swallow panic so we can check operations - runner.report_item.hash = Some("hash123".into()); let _ = runner.execute(&context, &node).await; let operation = runner.operations.last().unwrap(); @@ -820,7 +837,8 @@ mod task_runner { let node = container.create_action_node(); let context = ActionContext::default(); - runner.report_item.hash = Some("hash123".into()); + setup_exec_state(&mut runner); + runner.execute(&context, &node).await.unwrap(); assert!(container @@ -841,8 +859,9 @@ mod task_runner { let node = container.create_action_node(); let context = ActionContext::default(); + setup_exec_state(&mut runner); + // Swallow panic so we can check operations - runner.report_item.hash = Some("hash123".into()); let _ = runner.execute(&context, &node).await; let operation = runner @@ -1048,6 +1067,7 @@ mod task_runner { runner.cache.data.exit_code = 0; runner.cache.data.hash = "hash123".into(); + runner.action_digest = stub_digest(); } #[tokio::test] @@ -1093,9 +1113,11 @@ mod task_runner { use super::*; use std::fs; - fn setup_local_state(container: &TaskRunnerContainer, _runner: &mut TaskRunner) { + fn setup_local_state(container: &TaskRunnerContainer, runner: &mut TaskRunner) { container.sandbox.enable_git(); container.pack_archive(); + + runner.action_digest = stub_digest(); } #[tokio::test] diff --git a/crates/task-runner/tests/utils.rs b/crates/task-runner/tests/utils.rs index cba8d73da4d..df1361727b0 100644 --- a/crates/task-runner/tests/utils.rs +++ b/crates/task-runner/tests/utils.rs @@ -79,7 +79,7 @@ impl TaskRunnerContainer { pub fn create_archiver(&self) -> OutputArchiver { OutputArchiver { app: &self.app_context, - project_config: &self.project.config, + project: &self.project, task: &self.task, } } diff --git a/crates/task/src/task.rs b/crates/task/src/task.rs index 32a7500298d..96ea0f62546 100644 --- a/crates/task/src/task.rs +++ b/crates/task/src/task.rs @@ -1,7 +1,7 @@ use crate::task_options::TaskOptions; use moon_common::{ cacheable, - path::{ProjectRelativePathBuf, WorkspaceRelativePathBuf}, + path::{PathExt, ProjectRelativePathBuf, WorkspaceRelativePathBuf}, Id, }; use moon_config::{ @@ -85,7 +85,10 @@ cacheable!( pub type_of: TaskType, #[serde(skip)] - pub walk_cache: OnceCell>, + pub inputs_cache: OnceCell>, + + #[serde(skip)] + pub outputs_cache: OnceCell>, } ); @@ -134,31 +137,55 @@ impl Task { &self, workspace_root: &Path, ) -> miette::Result> { - let mut list = vec![]; + let mut list = FxHashSet::default(); for path in &self.input_files { // Detect if file actually exists if path.to_path(workspace_root).is_file() { - list.push(path.to_owned()); + list.insert(path.to_owned()); } } if !self.input_globs.is_empty() { let globs = &self.input_globs; let walk_paths = self - .walk_cache + .inputs_cache + .get_or_try_init(|| glob::walk_files(workspace_root, globs))?; + + // Glob results are absolute paths! + for file in walk_paths { + list.insert(file.relative_to(workspace_root).unwrap()); + } + } + + Ok(list.into_iter().collect()) + } + + /// Return a list of all workspace-relative output files. + pub fn get_output_files( + &self, + workspace_root: &Path, + include_non_globs: bool, + ) -> miette::Result> { + let mut list = FxHashSet::default(); + + if include_non_globs { + list.extend(self.output_files.clone()); + } + + if !self.output_globs.is_empty() { + let globs = &self.output_globs; + let walk_paths = self + .outputs_cache .get_or_try_init(|| glob::walk_files(workspace_root, globs))?; // Glob results are absolute paths! for file in walk_paths { - list.push( - WorkspaceRelativePathBuf::from_path(file.strip_prefix(workspace_root).unwrap()) - .unwrap(), - ); + list.insert(file.relative_to(workspace_root).unwrap()); } } - Ok(list) + Ok(list.into_iter().collect()) } /// Return true if the task is a "build" type. diff --git a/crates/workspace/src/workspace_builder.rs b/crates/workspace/src/workspace_builder.rs index b08bb320447..0ed4240f8eb 100644 --- a/crates/workspace/src/workspace_builder.rs +++ b/crates/workspace/src/workspace_builder.rs @@ -135,7 +135,7 @@ impl<'app> WorkspaceBuilder<'app> { graph_contents.add_configs(graph.hash_required_configs().await?); graph_contents.gather_env(); - let hash = cache_engine + let (hash, _) = cache_engine .hash .save_manifest_without_hasher("Workspace graph", &graph_contents)?; diff --git a/justfile b/justfile index c6d5c9f66e3..9d362abd2a1 100644 --- a/justfile +++ b/justfile @@ -72,3 +72,15 @@ moon-check: schemas: cargo run -p moon_config_schema --features typescript + +clean-bazel-remote: + rm -f ~/.moon/bazel-cache/cas.v2/.DS_Store && rm -f ~/.moon/bazel-cache/ac.v2/.DS_Store + +bazel-remote: + just clean-bazel-remote && bazel-remote --dir ~/.moon/bazel-cache --max_size 10 --storage_mode uncompressed --grpc_address 0.0.0.0:9092 + +bazel-remote-tls: + just clean-bazel-remote && bazel-remote --dir ~/.moon/bazel-cache --max_size 10 --storage_mode uncompressed --grpc_address 0.0.0.0:9092 --tls_cert_file=./crates/remote/tests/__fixtures__/certs-local/server.crt --tls_key_file=./crates/remote/tests/__fixtures__/certs-local/server.key + +bazel-remote-mtls: + just clean-bazel-remote && bazel-remote --dir ~/.moon/bazel-cache --max_size 10 --storage_mode uncompressed --tls_cert_file=./crates/remote/tests/__fixtures__/certs-local/server.crt --tls_key_file=./crates/remote/tests/__fixtures__/certs-local/server.key --tls_ca_file=./crates/remote/tests/__fixtures__/certs-local/ca.crt diff --git a/packages/types/src/project-config.ts b/packages/types/src/project-config.ts index 548d96ff69c..6b2b7363a91 100644 --- a/packages/types/src/project-config.ts +++ b/packages/types/src/project-config.ts @@ -2,8 +2,8 @@ /* eslint-disable */ -import type { PartialTaskConfig, PlatformType, TaskConfig } from './tasks-config'; import type { UnresolvedVersionSpec } from './toolchain-config'; +import type { PartialTaskConfig, PlatformType, TaskConfig } from './tasks-config'; /** The task-to-task relationship of the dependency. */ export type DependencyType = 'cleanup' | 'required' | 'optional'; diff --git a/packages/types/src/workspace-config.ts b/packages/types/src/workspace-config.ts index d03b98f62d3..84852a9b62c 100644 --- a/packages/types/src/workspace-config.ts +++ b/packages/types/src/workspace-config.ts @@ -239,6 +239,69 @@ export interface RunnerConfig { logRunningCommand: boolean; } +/** Configures the action cache (AC) and content addressable cache (CAS). */ +export interface RemoteCacheConfig { + /** @default 'moon-outputs' */ + instanceName?: string; +} + +/** Configures for both server and client authentication with mTLS. */ +export interface RemoteMtlsConfig { + /** + * If true, assume that the server supports HTTP/2, + * even if it doesn't provide protocol negotiation via ALPN. + */ + assumeHttp2: boolean; + /** + * A file path, relative from the workspace root, to the + * client's PEM encoded X509 certificate. + */ + clientCert: string; + /** + * A file path, relative from the workspace root, to the + * client's private key. + */ + clientKey: string; + /** The domain name in which to verify the TLS certificate. */ + domain: string | null; + /** + * A file path, relative from the workspace root, to the + * servers's PEM encoded X509 certificate. + */ + serverCert: string; +} + +/** Configures for server-only authentication with TLS. */ +export interface RemoteTlsConfig { + /** + * If true, assume that the server supports HTTP/2, + * even if it doesn't provide protocol negotiation via ALPN. + */ + assumeHttp2: boolean; + /** A file path, relative from the workspace root, to a PEM encoded X509 certificate. */ + cert: string; + /** The domain name in which to verify the TLS certificate. */ + domain: string | null; +} + +/** Configures the remote service, powered by the Bazel Remote Execution API. */ +export interface RemoteConfig { + /** Configures the action cache (AC) and content addressable cache (CAS). */ + cache: RemoteCacheConfig; + /** + * The remote host to connect and send requests to. + * Supports gRPC protocols. + */ + host: string; + /** + * Connect to the host using server and client authentication with mTLS. + * This takes precedence over normal TLS. + */ + mtls: RemoteMtlsConfig | null; + /** Connect to the host using server-only authentication with TLS. */ + tls: RemoteTlsConfig | null; +} + /** The format to use for generated VCS hook files. */ export type VcsHookFormat = 'bash' | 'native'; @@ -331,6 +394,8 @@ export interface WorkspaceConfig { * @default true */ telemetry?: boolean; + /** Configures aspects of the remote service. */ + unstable_remote: RemoteConfig | null; /** Configures the version control system (VCS). */ vcs: VcsConfig; /** Requires a specific version of the `moon` binary. */ @@ -563,6 +628,69 @@ export interface PartialRunnerConfig { logRunningCommand?: boolean | null; } +/** Configures the action cache (AC) and content addressable cache (CAS). */ +export interface PartialRemoteCacheConfig { + /** @default 'moon-outputs' */ + instanceName?: string | null; +} + +/** Configures for both server and client authentication with mTLS. */ +export interface PartialRemoteMtlsConfig { + /** + * If true, assume that the server supports HTTP/2, + * even if it doesn't provide protocol negotiation via ALPN. + */ + assumeHttp2?: boolean | null; + /** + * A file path, relative from the workspace root, to the + * client's PEM encoded X509 certificate. + */ + clientCert?: string | null; + /** + * A file path, relative from the workspace root, to the + * client's private key. + */ + clientKey?: string | null; + /** The domain name in which to verify the TLS certificate. */ + domain?: string | null; + /** + * A file path, relative from the workspace root, to the + * servers's PEM encoded X509 certificate. + */ + serverCert?: string | null; +} + +/** Configures for server-only authentication with TLS. */ +export interface PartialRemoteTlsConfig { + /** + * If true, assume that the server supports HTTP/2, + * even if it doesn't provide protocol negotiation via ALPN. + */ + assumeHttp2?: boolean | null; + /** A file path, relative from the workspace root, to a PEM encoded X509 certificate. */ + cert?: string | null; + /** The domain name in which to verify the TLS certificate. */ + domain?: string | null; +} + +/** Configures the remote service, powered by the Bazel Remote Execution API. */ +export interface PartialRemoteConfig { + /** Configures the action cache (AC) and content addressable cache (CAS). */ + cache?: PartialRemoteCacheConfig | null; + /** + * The remote host to connect and send requests to. + * Supports gRPC protocols. + */ + host?: string | null; + /** + * Connect to the host using server and client authentication with mTLS. + * This takes precedence over normal TLS. + */ + mtls?: PartialRemoteMtlsConfig | null; + /** Connect to the host using server-only authentication with TLS. */ + tls?: PartialRemoteTlsConfig | null; +} + /** Configures the version control system (VCS). */ export interface PartialVcsConfig { /** @@ -640,6 +768,8 @@ export interface PartialWorkspaceConfig { * @default true */ telemetry?: boolean | null; + /** Configures aspects of the remote service. */ + unstable_remote?: PartialRemoteConfig | null; /** Configures the version control system (VCS). */ vcs?: PartialVcsConfig | null; /** Requires a specific version of the `moon` binary. */ diff --git a/scripts/data/generateCerts.sh b/scripts/data/generateCerts.sh new file mode 100644 index 00000000000..677c9860eba --- /dev/null +++ b/scripts/data/generateCerts.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Copied from: https://github.com/bazelbuild/bazel/blob/master/src/test/testdata/test_tls_certificate/README.md + +SERVER_CN=localhost +CLIENT_CN=localhost # Used when doing mutual TLS + +cd ./crates/remote/tests/__fixtures__/certs-local + +echo Generate CA key: +openssl genrsa -passout pass:1111 -des3 -out ca.key 4096 + +echo Generate CA certificate: +# Generates ca.crt which is the trustCertCollectionFile +openssl req -passin pass:1111 -new -x509 -days 358000 -key ca.key -out ca.crt -subj "/CN=${SERVER_CN}" + +echo Generate server key: +openssl genrsa -passout pass:1111 -des3 -out server.key 4096 + +echo Generate server signing request: +openssl req -passin pass:1111 -new -key server.key -out server.csr -subj "/CN=${SERVER_CN}" + +echo Self-signed server certificate: +# Generates server.crt which is the certChainFile for the server +openssl x509 -req -passin pass:1111 -days 358000 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out server.crt + +echo Remove passphrase from server key: +openssl rsa -passin pass:1111 -in server.key -out server.key + +echo Generate client key +openssl genrsa -passout pass:1111 -des3 -out client.key 4096 + +echo Generate client signing request: +openssl req -passin pass:1111 -new -key client.key -out client.csr -subj "/CN=${CLIENT_CN}" + +echo Self-signed client certificate: +# Generates client.crt which is the clientCertChainFile for the client (need for mutual TLS only) +openssl x509 -passin pass:1111 -req -days 358000 -in client.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out client.crt + +echo Remove passphrase from client key: +openssl rsa -passin pass:1111 -in client.key -out client.key + +echo Converting the private keys to X.509: +openssl pkcs8 -topk8 -nocrypt -in ca.key -out ca.pem +# Generates client.pem which is the clientPrivateKeyFile for the Client (needed for mutual TLS only) +openssl pkcs8 -topk8 -nocrypt -in client.key -out client.pem +# Generates server.pem which is the privateKeyFile for the Server +openssl pkcs8 -topk8 -nocrypt -in server.key -out server.pem diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index e5557edaef8..37a0aba7a6b 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -1,5 +1,6 @@ FROM node:latest WORKDIR /app +ENV CI=true # Install moon binary # RUN npm install -g @moonrepo/cli diff --git a/tests/docker/Dockerfile.staged b/tests/docker/Dockerfile.staged index 978ffea18cb..0e863f9ef67 100644 --- a/tests/docker/Dockerfile.staged +++ b/tests/docker/Dockerfile.staged @@ -1,6 +1,7 @@ #### BASE FROM node:latest AS base WORKDIR /app +ENV CI=true # Install moon binary # RUN npm install -g @moonrepo/cli diff --git a/website/static/schemas/workspace.json b/website/static/schemas/workspace.json index 760504f75fd..861a565b643 100644 --- a/website/static/schemas/workspace.json +++ b/website/static/schemas/workspace.json @@ -120,6 +120,18 @@ "default": true, "type": "boolean" }, + "unstable_remote": { + "title": "unstable_remote", + "description": "Configures aspects of the remote service.", + "anyOf": [ + { + "$ref": "#/definitions/RemoteConfig" + }, + { + "type": "null" + } + ] + }, "vcs": { "title": "vcs", "description": "Configures the version control system (VCS).", @@ -488,6 +500,133 @@ "description": "Strategies and protocols for locating plugins.", "type": "string" }, + "RemoteCacheConfig": { + "description": "Configures the action cache (AC) and content addressable cache (CAS).", + "type": "object", + "properties": { + "instanceName": { + "title": "instanceName", + "default": "moon-outputs", + "type": "string" + } + }, + "additionalProperties": false + }, + "RemoteConfig": { + "description": "Configures the remote service, powered by the Bazel Remote Execution API.", + "type": "object", + "properties": { + "cache": { + "title": "cache", + "description": "Configures the action cache (AC) and content addressable cache (CAS).", + "allOf": [ + { + "$ref": "#/definitions/RemoteCacheConfig" + } + ] + }, + "host": { + "title": "host", + "description": "The remote host to connect and send requests to. Supports gRPC protocols.", + "type": "string" + }, + "mtls": { + "title": "mtls", + "description": "Connect to the host using server and client authentication with mTLS. This takes precedence over normal TLS.", + "anyOf": [ + { + "$ref": "#/definitions/RemoteMtlsConfig" + }, + { + "type": "null" + } + ] + }, + "tls": { + "title": "tls", + "description": "Connect to the host using server-only authentication with TLS.", + "anyOf": [ + { + "$ref": "#/definitions/RemoteTlsConfig" + }, + { + "type": "null" + } + ], + "markdownDescription": "Connect to the host using server-only authentication with TLS." + } + }, + "additionalProperties": false + }, + "RemoteMtlsConfig": { + "description": "Configures for both server and client authentication with mTLS.", + "type": "object", + "properties": { + "assumeHttp2": { + "title": "assumeHttp2", + "description": "If true, assume that the server supports HTTP/2, even if it doesn't provide protocol negotiation via ALPN.", + "type": "boolean" + }, + "clientCert": { + "title": "clientCert", + "description": "A file path, relative from the workspace root, to the client's PEM encoded X509 certificate.", + "type": "string" + }, + "clientKey": { + "title": "clientKey", + "description": "A file path, relative from the workspace root, to the client's private key.", + "type": "string" + }, + "domain": { + "title": "domain", + "description": "The domain name in which to verify the TLS certificate.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "serverCert": { + "title": "serverCert", + "description": "A file path, relative from the workspace root, to the servers's PEM encoded X509 certificate.", + "type": "string" + } + }, + "additionalProperties": false + }, + "RemoteTlsConfig": { + "description": "Configures for server-only authentication with TLS.", + "type": "object", + "properties": { + "assumeHttp2": { + "title": "assumeHttp2", + "description": "If true, assume that the server supports HTTP/2, even if it doesn't provide protocol negotiation via ALPN.", + "type": "boolean" + }, + "cert": { + "title": "cert", + "description": "A file path, relative from the workspace root, to a PEM encoded X509 certificate.", + "type": "string" + }, + "domain": { + "title": "domain", + "description": "The domain name in which to verify the TLS certificate.", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "markdownDescription": "Configures for server-only authentication with TLS." + }, "RunnerConfig": { "description": "Configures aspects of the task runner (also known as the action pipeline).", "type": "object",