From 1214f3acc9cdab7c1188195b94593d89f62f7694 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Sun, 15 Dec 2024 20:57:34 +0000 Subject: [PATCH] Support repository_set from bzlmod --- examples/musl_cross_compiling/MODULE.bazel | 136 +++++++++++++++++- examples/musl_cross_compiling/WORKSPACE.bazel | 127 ---------------- rust/extensions.bzl | 62 +++++++- rust/repositories.bzl | 42 +++++- 4 files changed, 226 insertions(+), 141 deletions(-) diff --git a/examples/musl_cross_compiling/MODULE.bazel b/examples/musl_cross_compiling/MODULE.bazel index 00bb18361f..27a0e0e5a0 100644 --- a/examples/musl_cross_compiling/MODULE.bazel +++ b/examples/musl_cross_compiling/MODULE.bazel @@ -1,6 +1,130 @@ -############################################################################### -# Bazel now uses Bzlmod by default to manage external dependencies. -# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel. -# -# For more details, please check https://github.com/bazelbuild/bazel/issues/18958 -############################################################################### +module( + name = "musl_cross_compiling_example", + version = "0.0.0", +) + +bazel_dep( + name = "rules_rust", + version = "0.0.0", +) +local_path_override( + module_name = "rules_rust", + path = "../..", +) + +bazel_dep( + name = "platforms", + version = "0.0.10", +) +bazel_dep( + name = "rules_shell", + version = "0.3.0", +) + +RUST_EDITION = "2021" + +RUST_VERSION = "1.80.0" + +rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") +rust.toolchain( + edition = RUST_EDITION, + versions = [RUST_VERSION], +) + +# This overrides a default rust_repository_set created by rust_register_toolchain. +# It must be named exactly this. +# Each exec triple needs one of these calls per target triple it supports. +# The first call needs all of the attrs, the subsequent calls should only set name, target_triple, and target_compatible_with. +rust.repository_set( + name = "rust_linux_x86_64", + edition = RUST_EDITION, + exec_triple = "x86_64-unknown-linux-gnu", + target_compatible_with = [ + "@//linker_config:unknown", + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + target_triple = "x86_64-unknown-linux-gnu", + versions = [RUST_VERSION], +) +rust.repository_set( + name = "rust_linux_x86_64", + target_compatible_with = [ + "@//linker_config:musl", + "@platforms//cpu:arm64", + "@platforms//os:linux", + ], + target_triple = "aarch64-unknown-linux-musl", +) +rust.repository_set( + name = "rust_linux_x86_64", + target_compatible_with = [ + "@//linker_config:musl", + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + target_triple = "x86_64-unknown-linux-musl", +) + +# We don't need to register a repository_set for exec_triple == target_triple if we're not customising it in any way: +# one will get registered by default. +# But we do for the Linux case above, because we want to add the "@//linker_config:unknown" constraint in that case. +rust.repository_set( + name = "rust_darwin_x86_64", + edition = RUST_EDITION, + exec_triple = "x86_64-apple-darwin", + target_compatible_with = [ + "@//linker_config:musl", + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + target_triple = "x86_64-unknown-linux-musl", + versions = [RUST_VERSION], +) +rust.repository_set( + name = "rust_darwin_x86_64", + target_compatible_with = [ + "@//linker_config:musl", + "@platforms//cpu:arm64", + "@platforms//os:linux", + ], + target_triple = "aarch64-unknown-linux-musl", +) +rust.repository_set( + name = "rust_darwin_aarch64", + edition = RUST_EDITION, + exec_triple = "aarch64-apple-darwin", + target_compatible_with = [ + "@//linker_config:musl", + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + target_triple = "x86_64-unknown-linux-musl", + versions = [RUST_VERSION], +) +rust.repository_set( + name = "rust_darwin_aarch64", + target_compatible_with = [ + "@//linker_config:musl", + "@platforms//cpu:arm64", + "@platforms//os:linux", + ], + target_triple = "aarch64-unknown-linux-musl", +) +use_repo(rust, "rust_toolchains") + +register_toolchains("@rust_toolchains//:all") + +crate = use_extension( + "@rules_rust//crate_universe:extensions.bzl", + "crate", +) +crate.from_cargo( + name = "cu", + cargo_lockfile = "//:Cargo.Bazel.lock", + manifests = [ + "//:Cargo.toml", + "//:local_proc_macro/Cargo.toml", + ], +) +use_repo(crate, "cu") diff --git a/examples/musl_cross_compiling/WORKSPACE.bazel b/examples/musl_cross_compiling/WORKSPACE.bazel index c12fc4c4f9..1993ad6f48 100644 --- a/examples/musl_cross_compiling/WORKSPACE.bazel +++ b/examples/musl_cross_compiling/WORKSPACE.bazel @@ -1,107 +1,3 @@ -local_repository( - name = "rules_rust", - path = "../..", -) - -load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains", "rust_repository_set") - -rules_rust_dependencies() - -EDITION = "2021" - -# Before 1.80.0, proc macros couldn't be used when exec!=target where exec and target platforms use different shared library extension (i.e. so vs dylib) because of an error in rustc's handling of extensions. -RUST_VERSION = "1.80.0" - -rust_repository_set( - name = "darwin_x86_64_to_x86_64_musl_tuple", - edition = EDITION, - exec_triple = "x86_64-apple-darwin", - # Setting this extra_target_triples allows differentiating the musl case from the non-musl case, in case multiple linux-targeting toolchains are registered. - extra_target_triples = { - "x86_64-unknown-linux-musl": [ - "@//linker_config:musl", - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - }, - versions = [RUST_VERSION], -) - -rust_repository_set( - name = "darwin_arm64_to_x86_64_musl_tuple", - edition = EDITION, - exec_triple = "aarch64-apple-darwin", - extra_target_triples = { - "x86_64-unknown-linux-musl": [ - "@//linker_config:musl", - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - }, - versions = [RUST_VERSION], -) - -rust_repository_set( - name = "darwin_x86_64_to_arm64_musl_tuple", - edition = EDITION, - exec_triple = "x86_64-apple-darwin", - # Setting this extra_target_triples allows differentiating the musl case from the non-musl case, in case multiple linux-targeting toolchains are registered. - extra_target_triples = { - "aarch64-unknown-linux-musl": [ - "@//linker_config:musl", - "@platforms//cpu:arm64", - "@platforms//os:linux", - ], - }, - versions = [RUST_VERSION], -) - -rust_repository_set( - name = "darwin_arm64_to_arm64_musl_tuple", - edition = EDITION, - exec_triple = "aarch64-apple-darwin", - extra_target_triples = { - "aarch64-unknown-linux-musl": [ - "@//linker_config:musl", - "@platforms//cpu:arm64", - "@platforms//os:linux", - ], - }, - versions = [RUST_VERSION], -) - -# This overrides a default rust_repository_set created by rust_register_toolchain. -# It must be named exactly this, and must be called before rust_register_toolchain is. -rust_repository_set( - name = "rust_linux_x86_64", - edition = EDITION, - exec_triple = "x86_64-unknown-linux-gnu", - # Setting this extra_target_triples allows differentiating the musl case from the non-musl case, in case multiple linux-targeting toolchains are registered. - extra_target_triples = { - "aarch64-unknown-linux-musl": [ - "@//linker_config:musl", - "@platforms//cpu:arm64", - "@platforms//os:linux", - ], - "x86_64-unknown-linux-gnu": [ - "@//linker_config:unknown", - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - "x86_64-unknown-linux-musl": [ - "@//linker_config:musl", - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - }, - versions = [RUST_VERSION], -) - -rust_register_toolchains( - edition = EDITION, - versions = [RUST_VERSION], -) - load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( @@ -118,26 +14,3 @@ load_musl_toolchains(extra_target_compatible_with = ["@//linker_config:musl"]) load("@musl_toolchains//:toolchains.bzl", "register_musl_toolchains") register_musl_toolchains() - -load("@rules_rust//crate_universe:repositories.bzl", "crate_universe_dependencies") - -crate_universe_dependencies(bootstrap = True) - -load("@rules_rust//crate_universe:defs.bzl", "crates_repository") - -crates_repository( - name = "cu", - cargo_lockfile = "//:Cargo.Bazel.lock", - # `generator` is not necessary in official releases. - # See load statement for `cargo_bazel_bootstrap`. - generator = "@cargo_bazel_bootstrap//:cargo-bazel", - lockfile = "//:Cargo.Bazel.lock.json", - manifests = [ - "//:Cargo.toml", - "//:local_proc_macro/Cargo.toml", - ], -) - -load("@cu//:defs.bzl", "crate_repositories") - -crate_repositories() diff --git a/rust/extensions.bzl b/rust/extensions.bzl index 61dd9fd096..63bf7acdc9 100644 --- a/rust/extensions.bzl +++ b/rust/extensions.bzl @@ -2,7 +2,7 @@ load("@bazel_features//:features.bzl", "bazel_features") load("//rust:defs.bzl", "rust_common") -load("//rust:repositories.bzl", "rust_register_toolchains", "rust_toolchain_tools_repository") +load("//rust:repositories.bzl", "DEFAULT_TOOLCHAIN_TRIPLES", "rust_register_toolchains", "rust_repository_set", "rust_toolchain_tools_repository") load("//rust/platform:triple.bzl", "get_host_triple") load( "//rust/private:repository_utils.bzl", @@ -48,6 +48,45 @@ def _rust_impl(module_ctx): # See https://github.com/bazelbuild/bazel/discussions/22024 for discussion. root, rules_rust = _find_modules(module_ctx) + toolchain_triples = dict(DEFAULT_TOOLCHAIN_TRIPLES) + + repository_sets = root.tags.repository_set + + grouped_repository_sets = {} + for repository_set in repository_sets: + if repository_set.name not in grouped_repository_sets: + grouped_repository_sets[repository_set.name] = { + "allocator_library": repository_set.allocator_library, + "dev_components": repository_set.dev_components, + "edition": repository_set.edition, + "exec_triple": repository_set.exec_triple, + "extra_target_triples": {repository_set.target_triple: [str(v) for v in repository_set.target_compatible_with]}, + "name": repository_set.name, + "rustfmt_version": repository_set.rustfmt_version, + "sha256s": repository_set.sha256s, + "urls": repository_set.urls, + "versions": repository_set.versions, + } + else: + for attr_name in _RUST_REPOSITORY_SET_TAG_ATTRS.keys(): + if attr_name in ["extra_target_triples", "name", "target_compatible_with", "target_triple"]: + continue + attr_value = getattr(repository_set, attr_name, None) + if attr_value and attr_value != grouped_repository_sets[repository_set.name][attr_name]: + fail("You must only set {} on the first call to repository_set for a particular name but it was set multiple times for {}".format(attr_name, repository_set.name)) + grouped_repository_sets[repository_set.name]["extra_target_triples"][repository_set.target_triple] = [str(v) for v in repository_set.target_compatible_with] + + extra_toolchain_infos = {} + + for repository_set in grouped_repository_sets.values(): + toolchain_infos = rust_repository_set( + register_toolchain = False, + **repository_set + ) + extra_toolchain_infos.update(**toolchain_infos) + if toolchain_triples.get(repository_set["exec_triple"]) == repository_set["name"]: + toolchain_triples.pop(repository_set["exec_triple"], None) + toolchains = root.tags.toolchain or rules_rust.tags.toolchain for toolchain in toolchains: @@ -77,6 +116,8 @@ def _rust_impl(module_ctx): versions = toolchain.versions, register_toolchains = False, aliases = toolchain.aliases, + toolchain_triples = toolchain_triples, + extra_toolchain_infos = extra_toolchain_infos, ) metadata_kwargs = {} if bazel_features.external_deps.extension_metadata_has_reproducible: @@ -111,6 +152,24 @@ _COMMON_TAG_KWARGS = { ), } +_RUST_REPOSITORY_SET_TAG_ATTRS = { + "exec_triple": attr.string(doc = "Exec triple for this repository_set."), + "name": attr.string(doc = "Name of the repository_set - if you're looking to replace default toolchains you must use the exact name you're replacing."), + "target_compatible_with": attr.label_list(doc = "List of platform constraints this toolchain produces, for the particular target_triple this call is for."), + "target_triple": attr.string(doc = "target_triple to configure."), + "versions": attr.string_list( + doc = ( + "A list of toolchain versions to download. This parameter only accepts one version " + + "per channel. E.g. `[\"1.65.0\", \"nightly/2022-11-02\", \"beta/2020-12-30\"]`. " + + "May be set to an empty list (`[]`) to inhibit `rules_rust` from registering toolchains." + ), + ), +} | _COMMON_TAG_KWARGS + +_RUST_REPOSITORY_SET_TAG = tag_class( + attrs = _RUST_REPOSITORY_SET_TAG_ATTRS, +) + _RUST_TOOLCHAIN_TAG = tag_class( attrs = { "aliases": attr.string_dict( @@ -161,6 +220,7 @@ rust = module_extension( doc = "Rust toolchain extension.", implementation = _rust_impl, tag_classes = { + "repository_set": _RUST_REPOSITORY_SET_TAG, "toolchain": _RUST_TOOLCHAIN_TAG, }, ) diff --git a/rust/repositories.bzl b/rust/repositories.bzl index f158f4bb61..4762a798a8 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -152,7 +152,9 @@ def rust_register_toolchains( versions = _RUST_TOOLCHAIN_VERSIONS, aliases = {}, hub_name = None, - compact_windows_names = _COMPACT_WINDOWS_NAMES): + compact_windows_names = _COMPACT_WINDOWS_NAMES, + toolchain_triples = DEFAULT_TOOLCHAIN_TRIPLES, + extra_toolchain_infos = None): """Emits a default set of toolchains for Linux, MacOS, and Freebsd Skip this macro and call the `rust_repository_set` macros directly if you need a compiler for \ @@ -243,7 +245,7 @@ def rust_register_toolchains( rust_analyzer_repo_name, )) - for exec_triple, name in DEFAULT_TOOLCHAIN_TRIPLES.items(): + for exec_triple, name in toolchain_triples.items(): maybe( rust_repository_set, name = name, @@ -307,6 +309,15 @@ def rust_register_toolchains( fail("No repositories were created matching the requested names to alias:\n{}".format("\n".join(sorted(aliases)))) if hub_name: + if extra_toolchain_infos: + for name, info in extra_toolchain_infos.items(): + toolchain_names.append(name) + toolchain_labels[name] = info["label"] + exec_compatible_with_by_toolchain[name] = info["exec_compatible_with"] + target_compatible_with_by_toolchain[name] = info["target_compatible_with"] + toolchain_target_settings[name] = info["target_settings"] + toolchain_types[name] = info["toolchain_type"] + toolchain_repository_hub( name = hub_name, toolchain_names = toolchain_names, @@ -1133,7 +1144,8 @@ def rust_repository_set( toolchains. This is to avoid MAX_PATH issues. """ - all_toolchain_names = [] + all_toolchain_details = {} + toolchain_labels = [] for toolchain in _get_toolchain_repositories( name = name, exec_triple = exec_triple, @@ -1153,7 +1165,7 @@ def rust_repository_set( else: fail("extra_rustc_flags should be a list or a dict") - all_toolchain_names.append(rust_toolchain_repository( + toolchain_label = rust_toolchain_repository( name = toolchain.name, allocator_library = allocator_library, global_allocator_library = global_allocator_library, @@ -1175,16 +1187,32 @@ def rust_repository_set( version = toolchain.channel.version, exec_compatible_with = exec_compatible_with, target_compatible_with = toolchain.target_constraints, - )) + ) + toolchain_labels.append(toolchain_label) + all_toolchain_details[toolchain.name] = { + # TODO: Don't duplicate the defaulting here + "exec_compatible_with": exec_compatible_with or triple_to_constraint_set(exec_triple), + # "label": toolchain_label, + # TODO: Don't duplicate + "label": "@{}_tools//:rust_toolchain".format(toolchain.name), + "name": toolchain.name, + # TODO: Don't duplicate the defaulting here + "target_compatible_with": toolchain.target_constraints or triple_to_constraint_set(toolchain.target_triple), + "target_settings": target_settings, + # TODO: Don't duplicate + "toolchain_type": "@rules_rust//rust:toolchain", + } # This repository exists to allow `rust_repository_set` to work with the `maybe` wrapper. rust_toolchain_set_repository( name = name, - toolchains = all_toolchain_names, + toolchains = toolchain_labels, ) # Register toolchains if register_toolchain: - native.register_toolchains(*all_toolchain_names) + native.register_toolchains(*toolchain_labels) native.register_toolchains(str(Label("//rust/private/dummy_cc_toolchain:dummy_cc_wasm32_toolchain"))) native.register_toolchains(str(Label("//rust/private/dummy_cc_toolchain:dummy_cc_wasm64_toolchain"))) + + return all_toolchain_details