From 60079a74ec6690ff9d35c55e8315f3f23bc9b0c4 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Wed, 27 Nov 2024 08:38:53 -0800 Subject: [PATCH] Added support for compact windows repository names, avoiding MAX_PATH --- .bazelci/presubmit.yml | 23 +++-- cargo/private/cargo_bootstrap.bzl | 6 ++ cargo/private/cargo_utils.bzl | 36 +++++++- crate_universe/deps_bootstrap.bzl | 5 +- crate_universe/private/common_utils.bzl | 1 + crate_universe/private/crates_repository.bzl | 4 + .../private/internal_extensions.bzl | 2 + .../rules_nodejs/test/BUILD.bazel | 5 ++ rust/private/repository_utils.bzl | 4 + rust/repositories.bzl | 84 +++++++++++++++++-- 10 files changed, 145 insertions(+), 25 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 8958eb37d5..b241e1af85 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -983,19 +983,16 @@ tasks: - "//..." test_targets: - "//..." - # # TODO: https://github.com/bazelbuild/rules_rust/issues/3021 - # # The length of the workspace name results in paths that are too long - # # on windows for linking. - # extensions_wasm_bindgen_windows: - # platform: windows - # name: Extensions wasm-bindgen - # working_directory: extensions/wasm_bindgen - # build_flags: *aspects_flags - # test_flags: *aspects_flags - # build_targets: - # - "//..." - # test_targets: - # - "//..." + extensions_wasm_bindgen_windows: + platform: windows + name: Extensions wasm-bindgen + working_directory: extensions/wasm_bindgen + build_flags: *aspects_flags + test_flags: *aspects_flags + build_targets: + - "//..." + test_targets: + - "//..." buildifier: version: latest diff --git a/cargo/private/cargo_bootstrap.bzl b/cargo/private/cargo_bootstrap.bzl index b634aa3e0c..f63d7c23ae 100644 --- a/cargo/private/cargo_bootstrap.bzl +++ b/cargo/private/cargo_bootstrap.bzl @@ -196,6 +196,7 @@ def _cargo_bootstrap_repository_impl(repository_ctx): host_triple = get_host_triple(repository_ctx) cargo_template = repository_ctx.attr.rust_toolchain_cargo_template rustc_template = repository_ctx.attr.rust_toolchain_rustc_template + compress_windows_names = repository_ctx.attr.compressed_windows_toolchain_names tools = get_rust_tools( cargo_template = cargo_template, @@ -203,6 +204,7 @@ def _cargo_bootstrap_repository_impl(repository_ctx): host_triple = host_triple, channel = channel, version = version, + compress_windows_names = compress_windows_names, ) binary_name = repository_ctx.attr.binary or repository_ctx.name @@ -264,6 +266,10 @@ cargo_bootstrap_repository = repository_rule( allow_single_file = ["Cargo.toml"], mandatory = True, ), + "compressed_windows_toolchain_names": attr.bool( + doc = "Wether or not the toolchain names of windows toolchains are expected to be in a `compressed` format.", + default = True, + ), "env": attr.string_dict( doc = ( "A mapping of platform triple to a set of environment variables. See " + diff --git a/cargo/private/cargo_utils.bzl b/cargo/private/cargo_utils.bzl index b70d8e2393..c4e943d612 100644 --- a/cargo/private/cargo_utils.bzl +++ b/cargo/private/cargo_utils.bzl @@ -3,6 +3,7 @@ load("//rust/platform:triple_mappings.bzl", "system_to_binary_ext") def _resolve_repository_template( + *, template, abi = None, arch = None, @@ -11,7 +12,8 @@ def _resolve_repository_template( tool = None, triple = None, vendor = None, - version = None): + version = None, + compress_windows_names = False): """Render values into a repository template string Args: @@ -25,6 +27,8 @@ def _resolve_repository_template( triple (str, optional): The host triple vendor (str, optional): The host vendor name version (str, optional): The Rust version used in the toolchain. + compress_windows_names (bool): Whether or not the windows repositories are + to be in the compressed form. Returns: string: The resolved template string based on the given parameters """ @@ -52,9 +56,33 @@ def _resolve_repository_template( if channel: template = template.replace("{channel}", channel) + if compress_windows_names and system == "windows": + repo, _, target = template.partition("//") + prefix = "" + if repo.startswith("@"): + repo = repo[1:] + prefix = "@" + elif repo.startswith("@@"): + repo = repo[2:] + prefix = "@@" + + suffix = "" + if repo.endswith("_tools"): + repo = repo[:-len("_tools")] + suffix = "_tools" + + return "{}rw-{}{}//{}".format(prefix, abs(hash(repo)), suffix, target) + return template -def get_rust_tools(cargo_template, rustc_template, host_triple, channel, version): +def get_rust_tools( + *, + cargo_template, + rustc_template, + host_triple, + channel, + version, + compress_windows_names): """Retrieve `cargo` and `rustc` labels based on the host triple. Args: @@ -63,6 +91,8 @@ def get_rust_tools(cargo_template, rustc_template, host_triple, channel, version host_triple (struct): The host's triple. See `@rules_rust//rust/platform:triple.bzl`. channel (str): The Rust toolchain channel. version (str): The version (or iso date in case of beta or nightly channels) of Cargo+Rustc to use. + compress_windows_names (bool): Whether or not the windows repositories are + to be in the compressed form. Returns: struct: A struct containing the labels of expected tools @@ -79,6 +109,7 @@ def get_rust_tools(cargo_template, rustc_template, host_triple, channel, version system = host_triple.system, abi = host_triple.abi, tool = "cargo" + extension, + compress_windows_names = compress_windows_names, )) rustc_label = Label(_resolve_repository_template( @@ -91,6 +122,7 @@ def get_rust_tools(cargo_template, rustc_template, host_triple, channel, version system = host_triple.system, abi = host_triple.abi, tool = "rustc" + extension, + compress_windows_names = compress_windows_names, )) return struct( diff --git a/crate_universe/deps_bootstrap.bzl b/crate_universe/deps_bootstrap.bzl index fa4e6a7e72..37591b9949 100644 --- a/crate_universe/deps_bootstrap.bzl +++ b/crate_universe/deps_bootstrap.bzl @@ -7,7 +7,10 @@ load("//crate_universe/private:srcs.bzl", "CARGO_BAZEL_SRCS") # buildifier: disable=bzl-visibility load("//rust/private:common.bzl", "rust_common") -def cargo_bazel_bootstrap(name = "cargo_bazel_bootstrap", rust_version = rust_common.default_version, **kwargs): +def cargo_bazel_bootstrap( + name = "cargo_bazel_bootstrap", + rust_version = rust_common.default_version, + **kwargs): """An optional repository which bootstraps `cargo-bazel` for use with `crates_repository` Args: diff --git a/crate_universe/private/common_utils.bzl b/crate_universe/private/common_utils.bzl index 923c0e0e07..e9dfd787b3 100644 --- a/crate_universe/private/common_utils.bzl +++ b/crate_universe/private/common_utils.bzl @@ -94,6 +94,7 @@ def get_rust_tools(repository_ctx, host_triple): host_triple = host_triple, channel = channel, version = version, + compress_windows_names = repository_ctx.attr.compressed_windows_toolchain_names, ) def _cargo_home_path(repository_ctx): diff --git a/crate_universe/private/crates_repository.bzl b/crate_universe/private/crates_repository.bzl index 7924711372..4a1952c634 100644 --- a/crate_universe/private/crates_repository.bzl +++ b/crate_universe/private/crates_repository.bzl @@ -215,6 +215,10 @@ CARGO_BAZEL_REPIN=1 CARGO_BAZEL_REPIN_ONLY=crate_index bazel sync --only=crate_i ), mandatory = True, ), + "compressed_windows_toolchain_names": attr.bool( + doc = "Wether or not the toolchain names of windows toolchains are expected to be in a `compressed` format.", + default = True, + ), "generate_binaries": attr.bool( doc = ( "Whether to generate `rust_binary` targets for all the binary crates in every package. " + diff --git a/crate_universe/private/internal_extensions.bzl b/crate_universe/private/internal_extensions.bzl index 666ed2ed1e..1b8aa53e08 100644 --- a/crate_universe/private/internal_extensions.bzl +++ b/crate_universe/private/internal_extensions.bzl @@ -11,6 +11,7 @@ def _internal_deps_impl(module_ctx): direct_deps.extend(cargo_bazel_bootstrap( rust_toolchain_cargo_template = "@rust_host_tools//:bin/{tool}", rust_toolchain_rustc_template = "@rust_host_tools//:bin/{tool}", + compressed_windows_toolchain_names = False, )) # is_dev_dep is ignored here. It's not relevant for internal_deps, as dev @@ -35,6 +36,7 @@ def _internal_dev_deps_impl(module_ctx): direct_deps.extend(cross_installer_deps( rust_toolchain_cargo_template = "@rust_host_tools//:bin/{tool}", rust_toolchain_rustc_template = "@rust_host_tools//:bin/{tool}", + compressed_windows_toolchain_names = False, )) # is_dev_dep is ignored here. It's not relevant for internal_deps, as dev diff --git a/extensions/wasm_bindgen/rules_nodejs/test/BUILD.bazel b/extensions/wasm_bindgen/rules_nodejs/test/BUILD.bazel index d24b352bd3..610033fd58 100644 --- a/extensions/wasm_bindgen/rules_nodejs/test/BUILD.bazel +++ b/extensions/wasm_bindgen/rules_nodejs/test/BUILD.bazel @@ -60,4 +60,9 @@ nodejs_test( ":hello_world_web_wasm_bindgen", ], entry_point = "//test:hello_world_wasm_test.js", + # TODO: Fix test on Windows + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), ) diff --git a/rust/private/repository_utils.bzl b/rust/private/repository_utils.bzl index 708329c831..7eb6d5bd32 100644 --- a/rust/private/repository_utils.bzl +++ b/rust/private/repository_utils.bzl @@ -266,6 +266,7 @@ rust_toolchain( extra_rustc_flags = {extra_rustc_flags}, extra_exec_rustc_flags = {extra_exec_rustc_flags}, opt_level = {opt_level}, + tags = ["rust_version={version}"], ) """ @@ -273,6 +274,7 @@ def BUILD_for_rust_toolchain( name, exec_triple, target_triple, + version, allocator_library, global_allocator_library, default_edition, @@ -288,6 +290,7 @@ def BUILD_for_rust_toolchain( name (str): The name of the toolchain declaration exec_triple (triple): The rust-style target that this compiler runs on target_triple (triple): The rust-style target triple of the tool + version (str): The Rust version for the toolchain. allocator_library (str, optional): Target that provides allocator functions when rust_library targets are embedded in a cc_binary. global_allocator_library (str, optional): Target that provides allocator functions when a global allocator is used with cc_common_link. This target is only used in the target configuration; exec builds still use the symbols provided @@ -340,6 +343,7 @@ def BUILD_for_rust_toolchain( extra_rustc_flags = extra_rustc_flags, extra_exec_rustc_flags = extra_exec_rustc_flags, opt_level = opt_level, + version = version, ) _build_file_for_toolchain_template = """\ diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 3930a6161e..3122d326e2 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -46,6 +46,8 @@ DEFAULT_TOOLCHAIN_TRIPLES = { "x86_64-unknown-linux-gnu": "rust_linux_x86_64", } +_COMPACT_WINDOWS_NAMES = True + def rules_rust_dependencies(): """Dependencies used in the implementation of `rules_rust`.""" @@ -138,7 +140,8 @@ def rust_register_toolchains( urls = DEFAULT_STATIC_RUST_URL_TEMPLATES, versions = _RUST_TOOLCHAIN_VERSIONS, aliases = {}, - hub_name = None): + hub_name = None, + compact_windows_names = _COMPACT_WINDOWS_NAMES): """Emits a default set of toolchains for Linux, MacOS, and Freebsd Skip this macro and call the `rust_repository_set` macros directly if you need a compiler for \ @@ -174,6 +177,8 @@ def rust_register_toolchains( per channel. E.g. `["1.65.0", "nightly/2022-11-02", "beta/2020-12-30"]`. aliases (dict, optional): A mapping of "full" repository name to another name to use instead. hub_name (str, optional): The name of the bzlmod hub repository for toolchains. + compact_windows_names (bool): Whether or not to produce compact repository names for windows + toolchains. This is to avoid MAX_PATH issues. """ if not rustfmt_version: if len(versions) == 1: @@ -265,7 +270,15 @@ def rust_register_toolchains( rustfmt_repo_name, )) - for toolchain in _get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, fallback_target_compatible_with = None, aliases = aliases): + for toolchain in _get_toolchain_repositories( + name = name, + exec_triple = exec_triple, + extra_target_triples = extra_target_triples, + versions = versions, + fallback_target_compatible_with = None, + aliases = aliases, + compact_windows_names = compact_windows_names, + ): toolchain_names.append(toolchain.name) toolchain_labels[toolchain.name] = "@{}//:{}".format(toolchain.name + "_tools", "rust_toolchain") exec_compatible_with_by_toolchain[toolchain.name] = triple_to_constraint_set(exec_triple) @@ -469,6 +482,7 @@ def _rust_toolchain_tools_repository_impl(ctx): extra_rustc_flags = ctx.attr.extra_rustc_flags, extra_exec_rustc_flags = ctx.attr.extra_exec_rustc_flags, opt_level = ctx.attr.opt_level if ctx.attr.opt_level else None, + version = ctx.attr.version, )) # Not all target triples are expected to have dev components @@ -959,10 +973,38 @@ rust_toolchain_set_repository = repository_rule( implementation = _rust_toolchain_set_repository_impl, ) -def _get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, fallback_target_compatible_with, aliases = {}): +def _get_toolchain_repositories( + *, + name, + exec_triple, + extra_target_triples, + versions, + fallback_target_compatible_with, + compact_windows_names, + aliases): + """Collect structs represent toolchain repositories matching the given parameters. + + Args: + name (str): The base name to use for toolchains. + exec_triple (triple): The execution triple associated with the toolchain. + extra_target_triples (list[triple]): Additional target triples to get toolchains for. + the `exec_triple` is a default `target_triple`. + versions (str): The version of rustc to use. + fallback_target_compatible_with (list): _description_ + compact_windows_names (bool): Whether or not to produce compact repository names for windows + toolchains. This is to avoid MAX_PATH issues. + aliases (dict): Replacement names to use for toolchains created by this macro. + + Returns: + list[struct]: A list of toolchain structs + - name: The name of the toolchain repository. + - target_triple: The target triple of the toolchain. + - channel: The toolchain channel (nightly/stable). + - target_constraints: Bazel constrants assicated with the toolchain. + """ extra_target_triples_list = extra_target_triples.keys() if type(extra_target_triples) == "dict" else extra_target_triples - toolchain_repos = [] + toolchain_repos = {} for target_triple in depset([exec_triple] + extra_target_triples_list).to_list(): # Parse all provided versions while checking for duplicates @@ -994,16 +1036,29 @@ def _get_toolchain_repositories(name, exec_triple, extra_target_triples, version full_name = "{}__{}__{}".format(name, target_triple, channel.name) if full_name in aliases: full_name = aliases.pop(full_name) - toolchain_repos.append(struct( + elif compact_windows_names and "windows" in exec_triple: + full_name = "rw-{}".format(abs(hash(full_name))) + + toolchain_repo = struct( name = full_name, target_triple = target_triple, channel = channel, target_constraints = target_constraints, - )) + ) + + if full_name in toolchain_repos: + fail("Duplicate toolchain name of {} found in Rust toolchain repositories:\n{}\n{}".format( + full_name, + toolchain_repos[full_name], + toolchain_repo, + )) + + toolchain_repos[full_name] = toolchain_repo - return toolchain_repos + return toolchain_repos.values() def rust_repository_set( + *, name, versions, exec_triple, @@ -1025,7 +1080,8 @@ def rust_repository_set( register_toolchain = True, exec_compatible_with = None, default_target_compatible_with = None, - aliases = {}): + aliases = {}, + compact_windows_names = _COMPACT_WINDOWS_NAMES): """Assembles a remote repository for the given toolchain params, produces a proxy repository \ to contain the toolchain declaration, and registers the toolchains. @@ -1062,10 +1118,20 @@ def rust_repository_set( exec_compatible_with (list, optional): A list of constraints for the execution platform for this toolchain. default_target_compatible_with (list, optional): A list of constraints for the target platform for this toolchain when the exec platform is the same as the target platform. aliases (dict): Replacement names to use for toolchains created by this macro. + compact_windows_names (bool): Whether or not to produce compact repository names for windows + toolchains. This is to avoid MAX_PATH issues. """ all_toolchain_names = [] - for toolchain in _get_toolchain_repositories(name, exec_triple, extra_target_triples, versions, default_target_compatible_with, aliases): + for toolchain in _get_toolchain_repositories( + name = name, + exec_triple = exec_triple, + extra_target_triples = extra_target_triples, + versions = versions, + fallback_target_compatible_with = default_target_compatible_with, + aliases = aliases, + compact_windows_names = compact_windows_names, + ): # Infer toolchain-specific rustc flags depending on the type (list, dict, optional) of extra_rustc_flags if extra_rustc_flags == None: toolchain_extra_rustc_flags = []