External path dependencies work

.bazelci/presubmit.yml

@@ -533,6 +533,22 @@ tasks:
     test_targets:
       - "//..."
     build_flags: *aspects_flags
+  crate_universe_local_path_external:
+    name: Crate Universe Local Path External
+    platform: ubuntu2004
+    working_directory: examples/crate_universe_local_path
+    run_targets:
+      - "//:vendor_lazy_static_out_of_tree"
+    test_targets:
+      - "//..."
+  crate_universe_local_path_in_tree:
+    name: Crate Universe Local Path In Tree
+    platform: ubuntu2004
+    working_directory: examples/crate_universe_local_path
+    run_targets:
+      - "//:vendor_lazy_static_in_tree"
+    test_targets:
+      - "//..."
 # See https://github.com/bazelbuild/rules_rust/issues/2186 about re-enabling these.
 #  crate_universe_examples_windows:
 #    name: Crate Universe Examples

crate_universe/extensions.bzl

@@ -6,6 +6,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "generate_config_file", "generate_splicing_manifest")
 load("//crate_universe/private:generate_utils.bzl", "CARGO_BAZEL_GENERATOR_SHA256", "CARGO_BAZEL_GENERATOR_URL", "GENERATOR_ENV_VARS", "render_config")
+load("//crate_universe/private:local_crate_mirror.bzl", "local_crate_mirror")
 load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_SHA256S", "CARGO_BAZEL_URLS")
 load("//rust/platform:triple.bzl", "get_host_triple")
 load("//rust/platform:triple_mappings.bzl", "system_to_binary_ext")
@@ -133,6 +134,8 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo
     lockfile_path = tag_path.get_child("lockfile.json")
     module_ctx.file(lockfile_path, "")
 
+    paths_to_track_file = module_ctx.path("paths-to-track")
+
     cargo_bazel([
         "generate",
         "--cargo-lockfile",
@@ -148,8 +151,16 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo
         "--repin",
         "--lockfile",
         lockfile_path,
+        "--nonhermetic-root-bazel-workspace-dir",
+        module_ctx.path(Label("@@//:MODULE.bazel")).dirname,
+        "--paths-to-track",
+        paths_to_track_file,
     ])
 
+    paths_to_track = json.decode(module_ctx.read(paths_to_track_file))
+    for path in paths_to_track:
+        module_ctx.read(path)
+
     crates_dir = tag_path.get_child(cfg.name)
     _generate_repo(
         name = cfg.name,
@@ -212,6 +223,22 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo
                 strip_prefix = repo.get("strip_prefix", None),
                 **kwargs
             )
+        elif "Path" in repo:
+            options = {
+                "config": rendering_config,
+                "crate_context": crate,
+                "platform_conditions": contents["conditions"],
+                "supported_platform_triples": cfg.supported_platform_triples,
+            }
+            kwargs = {}
+            if len(CARGO_BAZEL_URLS) == 0:
+                kwargs["generator"] = "@cargo_bazel_bootstrap//:cargo-bazel"
+            local_crate_mirror(
+                name = crate_repo_name,
+                options_json = json.encode(options),
+                path = repo["Path"]["path"],
+                **kwargs
+            )
         else:
             fail("Invalid repo: expected Http or Git to exist for crate %s-%s, got %s" % (name, version, repo))
 

crate_universe/private/crates_repository.bzl

@@ -71,6 +71,8 @@ def _crates_repository_impl(repository_ctx):
             "metadata": metadata_path,
         })
 
+    paths_to_track_file = repository_ctx.path("paths-to-track")
+
     # Run the generator
     execute_generator(
         repository_ctx = repository_ctx,
@@ -82,10 +84,15 @@ def _crates_repository_impl(repository_ctx):
         repository_dir = repository_ctx.path("."),
         cargo = cargo_path,
         rustc = rustc_path,
+        paths_to_track_file = paths_to_track_file,
         # sysroot = tools.sysroot,
         **kwargs
     )
 
+    paths_to_track = json.decode(repository_ctx.read(paths_to_track_file))
+    for path in paths_to_track:
+        repository_ctx.read(path)
+
     # Determine the set of reproducible values
     attrs = {attr: getattr(repository_ctx.attr, attr) for attr in dir(repository_ctx.attr)}
     exclude = ["to_json", "to_proto"]

crate_universe/private/crates_vendor.bzl

@@ -34,6 +34,7 @@ ${{_ENVIRON[@]}} \\
 {env} \\
     "{bin}" \\
     {args} \\
+    --nonhermetic-root-bazel-workspace-dir="${{BUILD_WORKSPACE_DIRECTORY}}" \\
     "$@"
 """
 
@@ -42,7 +43,7 @@ _WINDOWS_WRAPPER = """\
 set RUNTIME_PWD=%CD%
 {env}
 
-{bin} {args} %*
+{bin} {args} --nonhermetic-root-bazel-workspace-dir=%BUILD_WORKSPACE_DIRECTORY% %*
 exit %ERRORLEVEL%
 """
 

crate_universe/private/generate_utils.bzl

@@ -433,6 +433,7 @@ def execute_generator(
         repository_dir,
         cargo,
         rustc,
+        paths_to_track_file,
         metadata = None):
     """Execute the `cargo-bazel` binary to produce `BUILD` and `.bzl` files.
 
@@ -446,6 +447,7 @@ def execute_generator(
         repository_dir (path): The output path for the Bazel module and BUILD files.
         cargo (path): The path of a Cargo binary.
         rustc (path): The path of a Rustc binary.
+        paths_to_track_file (path): Path to file where generator should write which files should trigger re-generating as a JSON list.
         metadata (path, optional): The path to a Cargo metadata json file. If this is set, it indicates to
             the generator that repinning is required. This file must be adjacent to a `Cargo.toml` and
             `Cargo.lock` file.
@@ -470,8 +472,18 @@ def execute_generator(
         cargo,
         "--rustc",
         rustc,
+        "--nonhermetic-root-bazel-workspace-dir",
+        repository_ctx.workspace_root,
+        "--paths-to-track",
+        paths_to_track_file,
     ]
 
+    if repository_ctx.attr.generator:
+        args.extend([
+            "--generator",
+            repository_ctx.attr.generator,
+        ])
+
     if lockfile_path:
         args.extend([
             "--lockfile",

crate_universe/private/local_crate_mirror.bzl

@@ -0,0 +1,63 @@
+"""`local_crate_mirror` rule implementation.
+
+This is a private implementation detail of crate_universe, and should not be relied on in manually written code.
+
+This is effectively a `local_repository` rule impementation, but where the BUILD.bazel file is generated using the `cargo-bazel render` command.
+"""
+
+load("//crate_universe/private:common_utils.bzl", "execute")
+load("//crate_universe/private:generate_utils.bzl", "get_generator")
+load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_SHA256S", "CARGO_BAZEL_URLS")
+load("//rust/platform:triple.bzl", "get_host_triple")
+
+def _local_crate_mirror_impl(repository_ctx):
+    path = repository_ctx.path(repository_ctx.attr.path)
+
+    host_triple = get_host_triple(repository_ctx)
+
+    generator, _generator_sha256 = get_generator(repository_ctx, host_triple.str)
+
+    execute(repository_ctx, ["bash", "-c", "cp -r {}/* {}/".format(path, repository_ctx.path("."))])
+
+    paths_to_track = execute(repository_ctx, ["bash", "-c", "find {} -type f".format(path)]).stdout.strip().split("\n")
+    for path_to_track in paths_to_track:
+        if path_to_track:
+            repository_ctx.read(path_to_track)
+
+    execute(repository_ctx, [generator, "render", "--options-json", repository_ctx.attr.options_json, "--output-path", repository_ctx.path("BUILD.bazel")])
+
+    repository_ctx.file("WORKSPACE", "")
+
+local_crate_mirror = repository_rule(
+    implementation = _local_crate_mirror_impl,
+    attrs = {
+        "generator": attr.string(
+            doc = (
+                "The absolute label of a generator. Eg. `@cargo_bazel_bootstrap//:cargo-bazel`. " +
+                "This is typically used when bootstrapping"
+            ),
+        ),
+        "generator_sha256s": attr.string_dict(
+            doc = "Dictionary of `host_triple` -> `sha256` for a `cargo-bazel` binary.",
+            default = CARGO_BAZEL_SHA256S,
+        ),
+        "generator_urls": attr.string_dict(
+            doc = (
+                "URL template from which to download the `cargo-bazel` binary. `{host_triple}` and will be " +
+                "filled in according to the host platform."
+            ),
+            default = CARGO_BAZEL_URLS,
+        ),
+        "options_json": attr.string(
+            doc = "JSON serialized instance of a crate_universe::context::SingleBuildFileRenderContext",
+        ),
+        "path": attr.string(
+            # TODO: Verify what happens if this is not an absolute path.
+            doc = "Absolute path to the BUILD.bazel file to generate.",
+        ),
+        "quiet": attr.bool(
+            doc = "If stdout and stderr should not be printed to the terminal.",
+            default = True,
+        ),
+    },
+)

crate_universe/private/srcs.bzl

@@ -11,6 +11,7 @@ CARGO_BAZEL_SRCS = [
     Label("//crate_universe:src/cli.rs"),
     Label("//crate_universe:src/cli/generate.rs"),
     Label("//crate_universe:src/cli/query.rs"),
+    Label("//crate_universe:src/cli/render.rs"),
     Label("//crate_universe:src/cli/splice.rs"),
     Label("//crate_universe:src/cli/vendor.rs"),
     Label("//crate_universe:src/config.rs"),

crate_universe/src/cli.rs

@@ -2,6 +2,7 @@
 
 mod generate;
 mod query;
+mod render;
 mod splice;
 mod vendor;
 
@@ -15,12 +16,14 @@ use tracing_subscriber::FmtSubscriber;
 
 pub use self::generate::GenerateOptions;
 pub use self::query::QueryOptions;
+pub use self::render::RenderOptions;
 pub use self::splice::SpliceOptions;
 pub use self::vendor::VendorOptions;
 
 // Entrypoints
 pub use generate::generate;
 pub use query::query;
+pub use render::render;
 pub use splice::splice;
 pub use vendor::vendor;
 
@@ -42,6 +45,9 @@ pub enum Options {
 
     /// Vendor BUILD files to the workspace with either repository definitions or `cargo vendor` generated sources.
     Vendor(VendorOptions),
+
+    /// Render a BUILD file for a single crate.
+    Render(RenderOptions),
 }
 
 // Convenience wrappers to avoid dependencies in the binary
@@ -51,7 +57,7 @@ pub fn parse_args() -> Options {
     Options::parse()
 }
 
-const EXPECTED_LOGGER_NAMES: [&str; 4] = ["Generate", "Splice", "Query", "Vendor"];
+const EXPECTED_LOGGER_NAMES: [&str; 5] = ["Generate", "Splice", "Query", "Vendor", "Render"];
 
 /// A wrapper for the tracing-subscriber default [FormatEvent]
 /// that prepends the name of the active CLI option.

crate_universe/src/cli/generate.rs

@@ -2,18 +2,21 @@
 
 use std::fs;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
 use anyhow::{bail, Context as AnyhowContext, Result};
+use camino::Utf8PathBuf;
 use cargo_lock::Lockfile;
 use clap::Parser;
 
 use crate::config::Config;
 use crate::context::Context;
 use crate::lockfile::{lock_context, write_lockfile};
-use crate::metadata::{load_metadata, Annotations, Cargo};
+use crate::metadata::{load_metadata, Annotations, Cargo, SourceAnnotation};
 use crate::rendering::{write_outputs, Renderer};
 use crate::splicing::SplicingManifest;
 use crate::utils::normalize_cargo_file_paths;
+use crate::utils::starlark::Label;
 
 /// Command line options for the `generate` subcommand
 #[derive(Parser, Debug)]
@@ -63,6 +66,23 @@ pub struct GenerateOptions {
     /// If true, outputs will be printed instead of written to disk.
     #[clap(long)]
     pub dry_run: bool,
+
+    /// The path to the Bazel root workspace (i.e. the directory containing the WORKSPACE.bazel file or similar).
+    /// BE CAREFUL with this value. We never want to include it in a lockfile hash (to keep lockfiles portable),
+    /// which means you also should not use it anywhere that _should_ be guarded by a lockfile hash.
+    /// You basically never want to use this value.
+    #[clap(long)]
+    pub nonhermetic_root_bazel_workspace_dir: Utf8PathBuf,
+
+    #[clap(long)]
+    pub paths_to_track: PathBuf,
+
+    /// The label of this binary, if it was built in bootstrap mode.
+    /// BE CAREFUL with this value. We never want to include it in a lockfile hash (to keep lockfiles portable),
+    /// which means you also should not use it anywhere that _should_ be guarded by a lockfile hash.
+    /// You basically never want to use this value.
+    #[clap(long)]
+    pub(crate) generator: Option<Label>,
 }
 
 pub fn generate(opt: GenerateOptions) -> Result<()> {
@@ -75,15 +95,26 @@ pub fn generate(opt: GenerateOptions) -> Result<()> {
             let context = Context::try_from_path(lockfile)?;
 
             // Render build files
-            let outputs = Renderer::new(config.rendering, config.supported_platform_triples)
-                .render(&context)?;
+            let outputs = Renderer::new(
+                Arc::new(config.rendering),
+                Arc::new(config.supported_platform_triples),
+            )
+            .render(&context, opt.generator)?;
 
             // make file paths compatible with bazel labels
             let normalized_outputs = normalize_cargo_file_paths(outputs, &opt.repository_dir);
 
             // Write the outputs to disk
             write_outputs(normalized_outputs, opt.dry_run)?;
 
+            write_paths_to_track(
+                &opt.paths_to_track,
+                context
+                    .crates
+                    .values()
+                    .filter_map(|crate_context| crate_context.repository.as_ref()),
+            )?;
+
             return Ok(());
         }
     }
@@ -112,17 +143,24 @@ pub fn generate(opt: GenerateOptions) -> Result<()> {
     let (cargo_metadata, cargo_lockfile) = load_metadata(metadata_path)?;
 
     // Annotate metadata
-    let annotations = Annotations::new(cargo_metadata, cargo_lockfile.clone(), config.clone())?;
+    let annotations = Annotations::new(
+        cargo_metadata,
+        cargo_lockfile.clone(),
+        config.clone(),
+        &opt.nonhermetic_root_bazel_workspace_dir,
+    )?;
+
+    write_paths_to_track(&opt.paths_to_track, annotations.lockfile.crates.values())?;
 
     // Generate renderable contexts for each package
     let context = Context::new(annotations, config.rendering.are_sources_present())?;
 
     // Render build files
     let outputs = Renderer::new(
-        config.rendering.clone(),
-        config.supported_platform_triples.clone(),
+        Arc::new(config.rendering.clone()),
+        Arc::new(config.supported_platform_triples.clone()),
     )
-    .render(&context)?;
+    .render(&context, opt.generator)?;
 
     // make file paths compatible with bazel labels
     let normalized_outputs = normalize_cargo_file_paths(outputs, &opt.repository_dir);
@@ -159,3 +197,23 @@ fn update_cargo_lockfile(path: &Path, cargo_lockfile: Lockfile) -> Result<()> {
 
     Ok(())
 }
+
+fn write_paths_to_track<'a, SourceAnnotations: Iterator<Item = &'a SourceAnnotation>>(
+    output_file: &Path,
+    source_annotations: SourceAnnotations,
+) -> Result<()> {
+    let paths_to_track: std::collections::BTreeSet<_> = source_annotations
+        .filter_map(|v| {
+            if let SourceAnnotation::Path { path } = v {
+                Some(path.join("Cargo.toml"))
+            } else {
+                None
+            }
+        })
+        .collect();
+    std::fs::write(
+        output_file,
+        serde_json::to_string(&paths_to_track).context("Failed to serialize paths to track")?,
+    )
+    .context("Failed to write paths to track")
+}