diff --git a/crates/app/src/commands/docker/file.rs b/crates/app/src/commands/docker/file.rs index b3ec7fab9e0..97aabdeffc7 100644 --- a/crates/app/src/commands/docker/file.rs +++ b/crates/app/src/commands/docker/file.rs @@ -180,7 +180,8 @@ fn get_base_image_from_platform(platform: &PlatformType) -> &str { PlatformType::Bun => "oven/bun:latest", PlatformType::Deno => "denoland/deno:latest", PlatformType::Node => "node:latest", + PlatformType::Python => "python:latest", PlatformType::Rust => "rust:latest", - _ => "scratch", + PlatformType::System | PlatformType::Unknown => "scratch", } } diff --git a/crates/config/src/toolchain/python_config.rs b/crates/config/src/toolchain/python_config.rs index 2cdb1b3ca17..9c7e2e4cab4 100644 --- a/crates/config/src/toolchain/python_config.rs +++ b/crates/config/src/toolchain/python_config.rs @@ -19,16 +19,16 @@ pub struct PythonConfig { #[setting(nested)] pub pip: Option, - /// Defines the virtual environment name which will be created on workspace root. - /// Project dependencies will be installed into this. Defaults to `.venv` - #[setting(default = ".venv")] - pub venv_name: String, - /// Assumes only the root `requirements.txt` is used for dependencies. /// Can be used to support the "one version policy" pattern. #[setting(default = true)] pub root_requirements_only: bool, + /// Defines the virtual environment name, which will be created in the workspace root. + /// Project dependencies will be installed into this. + #[setting(default = ".venv")] + pub venv_name: String, + /// The version of Python to download, install, and run `python` tasks with. #[setting(env = "MOON_PYTHON_VERSION")] pub version: Option, diff --git a/crates/task-runner/src/output_hydrater.rs b/crates/task-runner/src/output_hydrater.rs index 7e500783781..48bdfeacceb 100644 --- a/crates/task-runner/src/output_hydrater.rs +++ b/crates/task-runner/src/output_hydrater.rs @@ -32,7 +32,7 @@ impl<'task> OutputHydrater<'task> { digest: &Digest, operation: &mut Operation, ) -> miette::Result { - match from { + let result = match from { // Only hydrate when the hash is different from the previous build, // as we can assume the outputs from the previous build still exist? HydrateFrom::PreviousOutput => Ok(true), @@ -43,6 +43,7 @@ impl<'task> OutputHydrater<'task> { // Otherwise write to local cache, then download archive from moonbase HydrateFrom::LocalCache | HydrateFrom::Moonbase => { let archive_file = self.app.cache_engine.hash.get_archive_path(&digest.hash); + let mut hydrated = false; if self.app.cache_engine.is_readable() { debug!( @@ -61,15 +62,7 @@ impl<'task> OutputHydrater<'task> { if archive_file.exists() { self.unpack_local_archive(&digest.hash, &archive_file)?; - read_stdlog_state_files( - self.app - .cache_engine - .state - .get_target_dir(&self.task.target), - operation, - )?; - - return Ok(true); + hydrated = true } } else { debug!( @@ -79,8 +72,28 @@ impl<'task> OutputHydrater<'task> { ); } - Ok(false) + Ok(hydrated) + } + }; + + match result { + Ok(hydrated) => { + // If not from the remote cache, we need to read the + // locally cached stdout/stderr into the operation + // so that it can be replayed in the console + if !matches!(from, HydrateFrom::RemoteCache) { + read_stdlog_state_files( + self.app + .cache_engine + .state + .get_target_dir(&self.task.target), + operation, + )?; + } + + Ok(hydrated) } + Err(error) => Err(error), } } diff --git a/crates/task-runner/src/task_runner.rs b/crates/task-runner/src/task_runner.rs index 8faeb81be2e..dc0c0429af7 100644 --- a/crates/task-runner/src/task_runner.rs +++ b/crates/task-runner/src/task_runner.rs @@ -725,9 +725,12 @@ impl<'task> TaskRunner<'task> { let mut operation = Operation::task_execution(&self.task.command); - if let (Some(output), Some(error)) = (operation.get_output_mut(), report) { + if let Some(output) = operation.get_output_mut() { output.exit_code = Some(-1); - output.set_stderr(error.to_string()); + + if let Some(error) = report { + output.set_stderr(error.to_string()); + } } operation.finish(ActionStatus::Aborted); diff --git a/website/blog/2024-11-25_moon-v1.30.mdx b/website/blog/2024-11-25_moon-v1.30.mdx new file mode 100644 index 00000000000..a8683524cb6 --- /dev/null +++ b/website/blog/2024-11-25_moon-v1.30.mdx @@ -0,0 +1,163 @@ +--- +slug: moon-v1.30 +title: moon v1.30 - Python support, self-hosted remote caching, task graph, and more! +authors: [milesj] +tags: [affected, detection, tracker, task, graph, self-hosted, remote, cache, python, toolchain] +image: ./img/moon/v1.30.png +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +It's been almost 2 months since our last release, and we're excited to announce some major features +requested by the community! + + + +## Experimental Python tier 2 and 3 support + +Thanks to a contribution from [@harlequin](https://github.com/harlequin), we now provide Python tier +[2](/docs/how-it-works/languages#tier-2--platform) and +[3](/docs/how-it-works/languages#tier-3--toolchain) support. Python is a very popular language, so +it was about time that we officially supported it in some capacity. When enabling Python in moon, +the following functionality will be enabled: + +- Will download and install Python into the toolchain (if `python.version` is defined). +- Will parse `requirements.txt` to extract dependency and version information for hashing. +- Will automatically activate virtual environments and setup `PATH` for tasks. +- Will automatically install dependencies for `requirements.txt` via pip. +- And a handful of internal interoperability features. + +However, we're still marking this implementation as experimental, as it hasn't been thoroughly +tested, and we're not 100% positive the workflows are what users expect. So please provide any +feedback, good or bad! + +Furthermore, as mentioned above, we install dependencies with pip (the version of pip that comes +pre-installed with the current Python version). At this time, _we do not support_ other package +managers like Poetry, Hatch, PDM, Rye, or uv, but we will in the future! + +### New `python` configurations + +Languages in +[moon are enabled through configuration](/docs/how-it-works/languages#enabling-a-language) blocks in +[`.moon/toolchain.yml`](/docs/config/toolchain), and Python is no different. We now support a +[`python`](/docs/config/toolchain#python) toolchain setting +([view all available settings](/docs/config/toolchain)). + +```yaml title=".moon/toolchain.yml" +python: + version: '3.14.0' +``` + +When the `python` setting is defined, it will enable the language and +[deep platform integration](/docs/how-it-works/languages#tier-2--platform), and when the +`python.version` field is defined, it will further enable +[toolchain support](/docs/how-it-works/languages#tier-3--toolchain). Both of these features provide +heavy automation, improving the overall developer experience. + +This is fantastic, but what if another Python project in the monorepo requires a different toolchain +channel/version? If so, they can use the new [`toolchain.python`](/docs/config/project#python) +setting in [`moon.yml`](/docs/config/project) to define project-level overrides. + +```yaml title="/moon.yml" +toolchain: + python: + version: '3.12.0' +``` + +### Built-in virtual environments + +Of course we also have support for Python virtual environments. When running a task, moon will +automatically enable the virtual environment in the workspace root or a project root (depending on +config)! The name of the venv can be customized with the +[`python.venvName`](/docs/config/toolchain#venvname) setting, otherwise it defaults to `.venv`. + +```yaml title=".moon/toolchain.yml" +python: + venvName: '.venvcustom' +``` + +## Unstable self-hosted remote caching + +This has been a request from the community for a very long time, and we get it, not every user wants +to store their build artifacts (not source code) in a third-party cloud provider. While we're proud +of our [moonbase service](/moonbase), it wasn't a viable option for many companies because of their +proprietary requirements. We spent a few months reworking moonbase to work as a self-hosted service, +so users can host it as on-prem solution, but it has been a very costly initiative. During this +process, we came to the conclusion that spending our time and resources on moonbase simply isn't +worth it, so we made the hard decision to sunset moonbase in the future. + +So what does that mean for remote caching? Simply put, you can now host your own remote caching +service! Instead of building a custom API for consumers to implement, we opted to implement the +[Bazel Remote Execution API](https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto), +which supports a content addressable storage (CAS) API, and is used by other popular build tools, +like Bazel, Buck, Pants, and more! + +Because we opted for a community solution, we can now focus all our efforts on [moon](/moon) and +[proto](/proto)! Additionally, adopting RE API allows you, the user, to use an off-the-shelf +solution, like [`bazel-remote`](https://github.com/buchgr/bazel-remote), instead of building your +own custom caching server! For example, to make use of remote caching, simply serve `bazel-remote`: + +```shell +bazel-remote --dir /path/to/moon-cache --max_size 10 --storage_mode uncompressed --grpc_address 0.0.0.0:9092 +``` + +And then configure the new [`unstable_remote`](/docs/config/workspace#unstable_remote) setting in +[`.moon/workspace.yml`](/docs/config/workspace). + +```yaml title=".moon/workspace.yml" +unstable_remote: + host: 'grpc://your-host.com:9092' +``` + +Pretty awesome right? Jump to the +[official remote caching](/docs/guides/remote-cache#self-hosted-unstable) documentation for more +information on this implementation. + +### Unsupported features + +Since this is a new feature, we're marking it as unstable, as it hasn't been thoroughly tested, and +_does not_ support the entire Bazel RE API. The following features _have not_ been implemented, but +will be in the future. + +- HTTP(S) host (we only support gRPC(S)) +- Digest hashing functions besides SHA256 +- Compression formats (we only support identity/uncompressed right now) +- Write/read bytestream for large blobs (4mb is the current limit) +- Better TLS/mTLS support (it has some issues) +- Directory blob types + +## New task graph and improved affected tracker + +In our [last release](./moon-v1.29#new-affected-projects-tracker), we announced a new affected +tracker for projects, but _not_ for tasks. The reason behind this was simple, we couldn't! Up until +now, we had no concept of a task graph, we had a project graph (that had tasks) and an action graph +(that ran tasks), but the relationships between tasks were split across both of these graphs. + +This made it complicated to support tasks for the new affected tracker, as the action graph +_consumes_ the tracker, not the other way around. To remedy this issue, we now support an official +task graph, which is derived from the project graph, and then feeds into the action graph. Since the +task graph sits outside of the action graph, we're now able to support tasks in the affected +tracker! + +Because of the new task graph, the following improvements have been introduced: + +- Tasks are now supported in the affected tracker (as mentioned). +- We can now properly query dependencies and dependents of specific tasks. +- We can now associate types to relationships (graph edges). For example, in the future we can add + optional, cleanup, and other kinds of dependencies. +- We've added a new command, [`moon task-graph`](/docs/commands/task-graph), that can visualize + tasks in isolation. +- We've updated the [`moon query tasks`](/docs/commands/query/tasks) to derive information from the + task graph. + +## Other changes + +View the [official release](https://github.com/moonrepo/moon/releases/tag/v1.30.0) for a full list +of changes. + +- Added basic support for Git submodules, and will now extract touched files from all submodules. +- Added 7 new token variables: `$arch`, `$os`, `$osFamily`, `$vcsBranch`, `$vcsRepository`, + `$vcsRevision`, `$workingDir` +- Added a `rust.binstallVersion` setting to `.moon/toolchain.yml`. +- Updated Pkl configurations to support `read()` for environment variables. diff --git a/website/blog/img/moon/v1.30.png b/website/blog/img/moon/v1.30.png new file mode 100644 index 00000000000..111e204f04b Binary files /dev/null and b/website/blog/img/moon/v1.30.png differ diff --git a/website/docs/commands/project-graph.mdx b/website/docs/commands/project-graph.mdx index c204870ce35..07625f8acff 100644 --- a/website/docs/commands/project-graph.mdx +++ b/website/docs/commands/project-graph.mdx @@ -2,7 +2,7 @@ title: project-graph --- -The `moon project-graph [name]` (or `moon pg`) command will generate and serve a visual graph of all +The `moon project-graph [id]` (or `moon pg`) command will generate and serve a visual graph of all configured projects as nodes, with dependencies between as edges, and can also output the graph in [Graphviz DOT format](https://graphviz.org/doc/info/lang.html). @@ -14,7 +14,7 @@ $ moon project-graph $ moon project-graph --dot > graph.dot ``` -> A project name can be passed to focus the graph to only that project and it's dependencies. For +> A project name can be passed to focus the graph to only that project and its dependencies. For > example, `moon project-graph app`. ### Arguments diff --git a/website/docs/commands/query/tasks.mdx b/website/docs/commands/query/tasks.mdx index 2370a73dadf..d1af9b5346b 100644 --- a/website/docs/commands/query/tasks.mdx +++ b/website/docs/commands/query/tasks.mdx @@ -52,18 +52,24 @@ passing the `--json` flag. The output has the following structure: #### Affected -- `--affected` - Filter projects that have been affected by touched files. +- `--affected` - Filter tasks that have been affected by touched files. +- `--downstream` - Include downstream dependents of queried tasks. Supports "none" (default), + "direct", "deep". + +- `--upstream` - Include upstream dependencies of queried tasks. Supports "none", "direct", "deep" + (default). + -#### Filters +#### Filters All option values are case-insensitive regex patterns. -- `--alias ` - Filter projects that match this alias. -- `--id ` - Filter projects that match this ID/name. -- `--language ` - Filter projects of this programming language. -- `--source ` - Filter projects that match this source path. -- `--tasks ` - Filter projects that have the following tasks. -- `--type ` - Filter project of this type. +- `--command ` - Filter tasks that match this command. +- `--id ` - Filter tasks that match this ID. +- `--platform ` - Filter tasks of this platform. +- `--project ` - Filter tasks that belong to this project. +- `--script ` - Filter tasks that match this script. +- `--type ` - Filter tasks of this type. ### Configuration diff --git a/website/docs/commands/task-graph.mdx b/website/docs/commands/task-graph.mdx new file mode 100644 index 00000000000..d76925c2e8e --- /dev/null +++ b/website/docs/commands/task-graph.mdx @@ -0,0 +1,47 @@ +--- +title: task-graph +--- + +import VersionLabel from '@site/src/components/Docs/VersionLabel'; + + + +The `moon task-graph [target]` (or `moon tg`) command will generate and serve a visual graph of all +configured tasks as nodes, with dependencies between as edges, and can also output the graph in +[Graphviz DOT format](https://graphviz.org/doc/info/lang.html). + +```shell +# Run the visualizer locally +$ moon task-graph + +# Export to DOT format +$ moon task-graph --dot > graph.dot +``` + +> A task target can be passed to focus the graph to only that task and its dependencies. For +> example, `moon task-graph app:build`. + +### Arguments + +- `[target]` - Optional target of task to focus. + +### Options + +- `--dependents` - Include direct dependents of the focused task. +- `--dot` - Print the graph in DOT format. +- `--json` - Print the graph in JSON format. + +## Example output + +The following output is an example of the graph in DOT format. + +```dot +digraph { + 0 [ label="types:build" style=filled, shape=oval, fillcolor=gray, fontcolor=black] + 1 [ label="runtime:build" style=filled, shape=oval, fillcolor=gray, fontcolor=black] + 2 [ label="website:build" style=filled, shape=oval, fillcolor=gray, fontcolor=black] + 1 -> 0 [ label="required" arrowhead=box, arrowtail=box] + 2 -> 1 [ label="required" arrowhead=box, arrowtail=box] + 2 -> 0 [ label="required" arrowhead=box, arrowtail=box] +} +``` diff --git a/website/docs/concepts/token.mdx b/website/docs/concepts/token.mdx index 6ef30a6e20e..4840da63a67 100644 --- a/website/docs/concepts/token.mdx +++ b/website/docs/concepts/token.mdx @@ -362,8 +362,32 @@ A token variable is a value that starts with `$` and is substituted to a value d current workspace, project, and task. And unlike token functions, token variables can be placed _within_ content when necessary, and supports multiple variables within the same content. +### Environment + +- `$arch` - The current host architecture, derived from the Rust + [`ARCH` constant](https://doc.rust-lang.org/std/env/consts/constant.ARCH.html). +- `$os` - The current operating system, derived from the Rust + [`OS` constant](https://doc.rust-lang.org/std/env/consts/constant.OS.html). +- `$osFamily` - The current operating system family, either `unix` or `windows`. + +```yaml +# Configured as +tasks: + build: + command: 'example --arch $arch' + +# Resolves to +tasks: + build: + command: + - 'example' + - '--arch' + - 'aarch64' +``` + ### Workspace +- `$workingDir` - The current working directory. - `$workspaceRoot` - Absolute file path to the workspace root. ```yaml @@ -469,3 +493,24 @@ tasks: - '--date' - '2023-03-17' ``` + +### VCS + +- `$vcsBranch` - The current branch. +- `$vcsRepository` - The repository slug, in the format of `owner/repo`. +- `$vcsRevision` - The current revision (commit, etc). + +```yaml +# Configured as +tasks: + build: + command: 'example --branch $vcsBranch' + +# Resolves to +tasks: + build: + command: + - 'example' + - '--branch' + - 'master' +``` diff --git a/website/docs/config/project.mdx b/website/docs/config/project.mdx index 72c00009bb1..35c50007d31 100644 --- a/website/docs/config/project.mdx +++ b/website/docs/config/project.mdx @@ -369,6 +369,7 @@ The Docker image to use in the base stage. Defaults to an image based on the pro - `oven/bun:latest` for Bun - `denoland/deno:latest` for Deno - `node:latest` for Node.js +- `python:latest` for Python - `rust:latest` for Rust - `scratch` for everything else @@ -1459,6 +1460,25 @@ toolchain: version: '12.12.0' ``` +### `python` + + + +Configures Python for this project and overrides the top-level [`python`](./toolchain#python) +setting. + +#### `version` + +Defines the explicit Python +[version/channel specification](../concepts/toolchain#version-specification) to use when _running +tasks_ for this project. + +```yaml title="moon.yml" {2,3} +toolchain: + python: + version: '3.12.0' +``` + ### `rust` diff --git a/website/docs/config/toolchain.mdx b/website/docs/config/toolchain.mdx index 0edd47e571a..f8d0a0e410e 100644 --- a/website/docs/config/toolchain.mdx +++ b/website/docs/config/toolchain.mdx @@ -38,6 +38,37 @@ taking precedence over those defined in the extended configuration. ::: +## `moon` + + + +Configures how moon will receive information about latest releases and download locations. + +### `manifestUrl` + + + +Defines an HTTPS URL in which to fetch the current version information from. + +```yaml title=".moon/toolchain.yml" {2} +moon: + manifestUrl: 'https://proxy.corp.net/moon/version' +``` + +### `downloadUrl` + + + +Defines an HTTPS URL in which the moon binary can be downloaded from. The download file name is +hard-coded and will be appended to the provided URL. + +Defaults to downloading from GitHub: https://github.com/moonrepo/moon/releases + +```yaml title=".moon/toolchain.yml" {2} +moon: + downloadUrl: 'https://github.com/moonrepo/moon/releases/latest/download' +``` + ## JavaScript ## `bun` @@ -157,37 +188,6 @@ deno: lockfile: true ``` -## `moon` - - - -Configures how moon will receive information about latest releases and download locations. - -### `manifestUrl` - - - -Defines an HTTPS URL in which to fetch the current version information from. - -```yaml title=".moon/toolchain.yml" {2} -moon: - manifestUrl: 'https://proxy.corp.net/moon/version' -``` - -### `downloadUrl` - - - -Defines an HTTPS URL in which the moon binary can be downloaded from. The download file name is -hard-coded and will be appended to the provided URL. - -Defaults to downloading from GitHub: https://github.com/moonrepo/moon/releases - -```yaml title=".moon/toolchain.yml" {2} -moon: - downloadUrl: 'https://github.com/moonrepo/moon/releases/latest/download' -``` - ## `node` @@ -721,31 +721,39 @@ Both imports can optionally be nested within a `src` directory. > This setting runs _after_ [`syncProjectReferences`](#syncprojectreferences) and will inherit any > synced references from that setting. -## Python +## Python -## `python` +## `python` -Enables and configures Python. +Enables and configures [Python](https://www.python.org/). ### `version` -Defines the explicit Python toolchain If this field is _not defined_, the global `python` binary -will be used. +Defines the explicit Python toolchain +[version specification](../concepts/toolchain#version-specification) to use. If this field is _not +defined_, the global `python` binary will be used. ```yaml title=".moon/toolchain.yml" {2} python: version: '3.11.10' ``` +:::info + +Python installation's are based on pre-built binaries provided by +[indygreg/python-build-standalone](https://github.com/indygreg/python-build-standalone). + +::: + > Version can also be defined with [`.prototools`](../proto/config). -### `rootPackageOnly` +### `rootRequirementsOnly` - + Supports the "single version policy" or "one version rule" patterns by only allowing dependencies in the root `requirements.txt`, and only installing dependencies in the workspace root, and not within @@ -754,28 +762,30 @@ Defaults to `true`. ```yaml title=".moon/toolchain.yml" {2} python: - rootPackageOnly: false + rootRequirementsOnly: false ``` -### `venv_name` +### `venvName` - + -Defines the virtual environment name which will be created on workspace root, project dependencies -will be installed into this. Defaults to `.venv` +Defines the virtual environment file name, which will be created in the workspace or project root, +and where dependencies will be installed into. Defaults to `.venv` ```yaml title=".moon/toolchain.yml" {2} python: - venv_name: '.my-custom-venv' + venvName: '.my-custom-venv' ``` ### `pip` -#### `install_args` +Optional fields for defining pip specific configuration. + +#### `installArgs` - + Customize the arguments that will be passed to the pip install command, when the `InstallDeps` action is triggered in the pipeline. These arguments are used both locally and in CI. diff --git a/website/docs/config/workspace.mdx b/website/docs/config/workspace.mdx index 6582eeb1a65..2a065a5a48a 100644 --- a/website/docs/config/workspace.mdx +++ b/website/docs/config/workspace.mdx @@ -301,21 +301,6 @@ docker: Enable or disable experiments that alter core functionality. -### `strictProjectIds` - - - -When building the project graph, disallows referencing the original ID of a project (inferred from -the folder name) when the project has been renamed to a new ID using the [`id`](./project#id) -setting. - -Defaults to `false` but will be enabled to `true` in the future. - -```yaml title=".moon/workspace.yml" {1,2} -experiments: - strictProjectIds: true -``` - ## `extensions` @@ -497,6 +482,134 @@ notifier: webhookUrl: 'https://api.company.com/some/endpoint' ``` +## `unstable_remote` + + + +Configures a remote service, primarily for cloud-based caching of artifacts. Learn more about this +in the [remote caching](../guides/remote-cache) guide. + +### `cache` + + + +Configures aspects of the caching layer, primarily the action cache (AC) and content addressable +cache (CAS). + +#### `instanceName` + + + +A +[unique identifier](https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto#L223) +used to distinguish between the various instances on the host. This allows the same remote service +to serve and partition multiple moon repositories. Defaults to `moon-outputs`. + +```yaml title=".moon/workspace.yml" {2} +unstable_remote: + cache: + instanceName: 'custom-dir-name' +``` + +> We suggest changing the instance name to the name of your repository! + +### `host` + + + +The host URL to communicate with when uploading and download artifacts. Supports gRPC through the +`grpc://` and `grpcs://` protocols. This field is required! + +```yaml title=".moon/workspace.yml" {2} +unstable_remote: + host: 'grpc://your-host.com:9092' +``` + +### `mtls` + + + +Connect to the host using server and client authentication with mTLS. This takes precedence over +normal TLS. + +```yaml title=".moon/workspace.yml" {3-7} +unstable_remote: + # ... + mtls: + caCert: 'certs/ca.pem' + clientCert: 'certs/client.pem' + clientKey: 'certs/client.key' + domain: 'your-host.com' +``` + +#### `assumeHttp2` + + + +If true, assume that the host supports HTTP/2, even if it doesn't provide protocol negotiation via +ALPN. + +#### `caCert` + + + +A file path, relative from the workspace root, to the certificate authority PEM encoded X509 +certificate (typically `ca.pem`). + +#### `clientCert` + + + +A file path, relative from the workspace root, to the client's PEM encoded X509 certificate +(typically `client.pem`). + +#### `clientKey` + + + +A file path, relative from the workspace root, to the client's PEM encoded X509 private key +(typically `client.key`). + +#### `domain` + + + +The domain name in which to verify the TLS certificate. + +### `tls` + + + +Connect to the host using server-only authentication with TLS. + +```yaml title=".moon/workspace.yml" {3-5} +unstable_remote: + # ... + tls: + cert: 'certs/ca.pem' + domain: 'your-host.com' +``` + +#### `assumeHttp2` + + + +If true, assume that the host supports HTTP/2, even if it doesn't provide protocol negotiation via +ALPN. + +#### `cert` + + + +A file path, relative from the workspace root, to the certificate authority PEM encoded X509 +certificate (typically `ca.pem`). + +#### `domain` + + + +The domain name in which to verify the TLS certificate. + ## `runner` diff --git a/website/docs/guides/remote-cache.mdx b/website/docs/guides/remote-cache.mdx index b86b2fd8df9..47d4b55edae 100644 --- a/website/docs/guides/remote-cache.mdx +++ b/website/docs/guides/remote-cache.mdx @@ -3,7 +3,7 @@ title: Remote caching toc_max_heading_level: 6 --- -import RemoteCacheBeta from '@site/src/components/Forms/RemoteCacheBeta'; +import VersionLabel from '@site/src/components/Docs/VersionLabel'; Is your CI pipeline running slower than usual? Are you tired of running the same build over and over although nothing has changed? Do you wish to reuse the same local cache across other machines and @@ -14,17 +14,93 @@ computation time, and alleviate resources. It achieves this by uploading hashed storage provider, like AWS S3 or Google Cloud, and downloading them on demand when a build matches a derived hash. -## Requirements +To make use of remote caching, we provide 2 solutions. -- moon >= 0.23.3 +## Self-hosted (unstable) -## Sign up for an account +This solution allows you to host any remote caching service that is compatible with the +[Bazel Remote Execution v2 API](https://github.com/bazelbuild/remote-apis/tree/main/build/bazel/remote/execution/v2), +such as [`bazel-remote`](https://github.com/buchgr/bazel-remote). When using this solution, the +following RE API features must be enabled: -Remote caching requires a [moonrepo.app](https://moonrepo.app) account, powered by our -[moonbase](/moonbase) service. Creating an account requires a GitHub account. In the future we'll -support additional OAuth providers. +- Action result caching +- Content addressable storage caching +- SHA256 digest hashing +- gRPC requests -### Create an organization +:::warning + +This feature and its implementation is currently unstable, and its documentation is incomplete. +Please report any issues on GitHub or through Discord! + +::: + +### Host your service + +When you have chosen (or built) a compatible service, host it and make it available through gRPC (we +do not support HTTP at this time). For example, if you plan to use `bazel-remote`, you can do +something like the following: + +```bash +bazel-remote --dir /path/to/moon-cache --max_size 10 --storage_mode uncompressed --grpc_address 0.0.0.0:9092 +``` + +:::info + +View the official [`bazel-remote`](https://github.com/buchgr/bazel-remote#usage) documentation for +all the available options, like storing artifacts in S3, configuring authentication (TLS/mTLS), +proxies, and more. + +::: + +### Configure remote caching + +Once your service is running, you can enable remote caching by configuring the +[`unstable_remote`](../config/workspace#unstable_remote) settings in +[`.moon/workspace.yml`](../config/workspace). At minimum, the only setting that is required is +`host`. + +```yaml title=".moon/workspace.yml" +unstable_remote: + host: 'grpc://your-host.com:9092' +``` + +#### TLS and mTLS + +We have rudimentary support for TLS and mTLS, but it's very unstable, and has not been thoroughly +tested. There's also [many](https://github.com/hyperium/tonic/issues/1652) +[many](https://github.com/hyperium/tonic/issues/1989) +[issues](https://github.com/hyperium/tonic/issues/1033) around authentication in Tonic. + +```yaml title=".moon/workspace.yml" +# TLS +unstable_remote: + host: 'grpcs://your-host.com:9092' + tls: + cert: 'certs/ca.pem' + domain: 'your-host.com' + +# mTLS +unstable_remote: + host: 'grpcs://your-host.com:9092' + mtls: + caCert: 'certs/ca.pem' + clientCert: 'certs/client.pem' + clientKey: 'certs/client.key' + domain: 'your-host.com' +``` + +## Cloud-hosted: moonbase + +This solution utilizes our [moonbase](/moonbase) service, which is a SaaS cloud offering hosted and +managed by us. This is also known as third-party hosting. + +### Sign up for an account + +Remote caching requires a [moonrepo.app](https://moonrepo.app) account. Creating an account requires +a GitHub account. In the future we'll support additional OAuth providers. + +#### Create an organization A moonbase organization can represent a company, team, group, or personal workspace, and is required for housing repositories, projects, artifacts, and more. Begin by creating an organization from the @@ -33,7 +109,7 @@ dashboard. > The "Region" field is very important for remote caching, as it configures which S3 region to store > artifacts in. This _cannot_ be changed after creation, so choose wisely! -### Create a repository +#### Create a repository A moonbase repository is a direct relationship to a codebase repository, and is necessary for linking artifacts to the appropriate tasks. From an organization's page, you can create a @@ -42,7 +118,7 @@ repository. > The "Slug" field must match your repository's "remote/repo" naming exactly, as we compare these > values for authentication purposes. -## Enabling remote caching +### Enabling remote caching Once you have a [moonrepo.app](https://moonrepo.app) account, an organization, and repository, you can enable remote caching in your CI pipeline. From a moonbase organization page, you can view your @@ -62,10 +138,10 @@ repository slug (`/`) must match. #### What is an artifact? -In the context of moon and remote caching, an artifact is a tar archive (`.tar.gz`) that contains -all [outputs of a task](../config/project#outputs), as well as the stdout and stderr of the task -that generated the outputs. Artifacts are uniquely identified by the -[moon generated hash](../concepts/cache#hashing). +In the context of moon and remote caching, an artifact is the +[outputs of a task](../config/project#outputs), a tar archive (`.tar.gz`) that contains all these +outputs, as well as the stdout and stderr of the task that generated the outputs. Artifacts are +uniquely identified by the [moon generated hash](../concepts/cache#hashing). #### Do I have to use remote caching? @@ -84,9 +160,10 @@ verify this, you can inspect the tar archives in `.moon/cache/outputs`. No, moon does not collect any PII as part of the remote caching process. -However, to use remote caching, you must create a moonrepo account in which we require an email -address, and information about your organization and repository. +However, to use [moonbase](/moonbase) remote caching, you must create a moonrepo account in which we +require an email address, and information about your organization and repository. #### Are artifacts encrypted? -Yes! We use AWS's built-in SSE-S3 encryption for all S3 buckets. +For [moonbase](/moonbase), yes! We use AWS's built-in SSE-S3 encryption for all S3 buckets. For +self-hosted services, encryption is up to you. diff --git a/website/docs/how-it-works/action-graph.mdx b/website/docs/how-it-works/action-graph.mdx index f7a68806875..2d423334185 100644 --- a/website/docs/how-it-works/action-graph.mdx +++ b/website/docs/how-it-works/action-graph.mdx @@ -8,8 +8,8 @@ When you run a [task](../config/project#tasks-1) on the command line, we generat ensure [dependencies](../config/project#deps) of tasks have ran before running run the primary task. The action graph is a representation of all [tasks](../concepts/task), derived from the -[project graph](./project-graph), and is also represented internally as a directed acyclic graph -(DAG). +[project graph](./project-graph) and [task graph](./task-graph), and is also represented internally +as a directed acyclic graph (DAG). diff --git a/website/docs/how-it-works/project-graph.mdx b/website/docs/how-it-works/project-graph.mdx index 1db3ddef951..f5dcbe0c4d0 100644 --- a/website/docs/how-it-works/project-graph.mdx +++ b/website/docs/how-it-works/project-graph.mdx @@ -21,7 +21,7 @@ your workspace. ## Relationships -A relationship is between a dependent (the parent project) and a dependency/requirement (a child +A relationship is between a dependent (downstream project) and a dependency/requirement (upstream project). Relationships are derived from source code and configuration files within the repository, and fall into 1 of 2 categories: @@ -82,21 +82,8 @@ integration. Great question, the project graph is used throughout the codebase to accomplish a variety of functions, but mainly: -- Is fed into the [action graph](./action-graph) to determine relationships of tasks between other +- Is fed into the [task graph](./task-graph) to determine relationships of tasks between other tasks, and across projects. - Powers our [Docker](../guides/docker) layer caching and scaffolding implementations. - Utilized for [project syncing](../commands/sync) to ensure a healthy repository state. - Determines affected projects in [continuous integration](../guides/ci) workflows. - - diff --git a/website/docs/how-it-works/task-graph.mdx b/website/docs/how-it-works/task-graph.mdx new file mode 100644 index 00000000000..5bd317ca7dc --- /dev/null +++ b/website/docs/how-it-works/task-graph.mdx @@ -0,0 +1,43 @@ +--- +title: Task graph +--- + +import TaskGraph from '@site/src/components/Docs/TaskGraph'; + +The task graph is a representation of all configured +[tasks in the workspace](../config/workspace#projects) and their relationships between each other, +and is represented internally as a directed acyclic graph (DAG). This graph is derived from +information in the [project graph](./project-graph). Below is a visual representation of a task +graph. + + + +:::info + +The [`moon task-graph`](../commands/task-graph) command can be used to view the structure of your +workspace. + +::: + +## Relationships + +A relationship is between a dependent (downstream task) and a dependency/requirement (upstream +task). Relationships are derived explicitly with the task [`deps`](../config/project#deps) setting, +and fall into 1 of 2 categories: + +### Required + +These are dependencies that are required to run and complete with a success, before the owning task +can run. If a required dependency fails, then the owning task will abort. + +### Optional + +The opposite of [required](#required), these are dependencies that can either a) not exist during +task inheritance, or b) run and fail without aborting the owning task. + +## What is the graph used for? + +Great question, the task graph is extremely important for running tasks (duh), and it also: + +- Is fed into the [action graph](./action-graph) that can be executed in topological order. +- Determines affected tasks in [continuous integration](../guides/ci) workflows. diff --git a/website/docs/intro.mdx b/website/docs/intro.mdx index 241ad785fd5..7fa7043f461 100644 --- a/website/docs/intro.mdx +++ b/website/docs/intro.mdx @@ -83,8 +83,8 @@ incrementally integrate and improve them over time. The 4 tiers are as follows: | └─ npm, pnpm, yarn | 🟢 | ⚪️ | 🟢 | 🟢 | | PHP | 🟢 | 🟢 | | | | └─ Composer | 🟢 | ⚪️ | | | -| Python | 🟢 | 🟢 | | | -| └─ Pip | 🟢 | ⚪️ | | | +| Python | 🟢 | 🟢 | 🟣 | 🟣 | +| └─ Pip | 🟢 | ⚪️ | 🟣 | | | Ruby | 🟢 | 🟢 | | | | └─ Gems, Bundler | 🟢 | ⚪️ | | | | Rust | 🟢 | 🟢 | 🟣 | 🟣 | diff --git a/website/sidebars.ts b/website/sidebars.ts index e6ee9229159..6c28374d6f9 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -10,7 +10,12 @@ const sidebars: SidebarsConfig = { label: 'How it works', collapsed: true, collapsible: true, - items: ['how-it-works/languages', 'how-it-works/project-graph', 'how-it-works/action-graph'], + items: [ + 'how-it-works/languages', + 'how-it-works/project-graph', + 'how-it-works/task-graph', + 'how-it-works/action-graph', + ], link: { type: 'generated-index', title: 'How it works', @@ -172,6 +177,7 @@ const sidebars: SidebarsConfig = { }, }, 'commands/task', + 'commands/task-graph', 'commands/teardown', 'commands/templates', 'commands/upgrade', diff --git a/website/src/components/Docs/TaskGraph.tsx b/website/src/components/Docs/TaskGraph.tsx new file mode 100644 index 00000000000..32250f22327 --- /dev/null +++ b/website/src/components/Docs/TaskGraph.tsx @@ -0,0 +1,78 @@ +import React, { useEffect, useRef } from 'react'; +import { renderGraph } from '../../utils/renderGraph'; + +export default function TaskGraph() { + const graphRef = useRef(null); + + useEffect(() => { + if (graphRef.current) { + renderGraph(graphRef.current, { + edges: [ + { + data: { + source: 'target-app-build', + target: 'target-app-clean', + }, + }, + { + data: { + source: 'target-app-build', + target: 'target-components-build', + }, + }, + { + data: { + source: 'target-app-build', + target: 'target-types-codegen', + }, + }, + { + data: { + source: 'target-components-build', + target: 'target-types-codegen', + }, + }, + ], + nodes: [ + { + data: { + id: 'target-app-build', + label: 'app:build', + type: 'sm', + }, + }, + { + data: { + id: 'target-app-clean', + label: 'app:clean', + type: 'sm', + }, + }, + { + data: { + id: 'target-components-build', + label: 'components:build', + type: 'sm', + }, + }, + { + data: { + id: 'target-types-codegen', + label: 'types:codegen', + type: 'sm', + }, + }, + ], + }); + } + }, []); + + return ( +
+ ); +}