diff --git a/MODULE.bazel b/MODULE.bazel index 488b480..02d6d6b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -4,9 +4,12 @@ module( compatibility_level = 1, ) -bazel_dep(name = "bazel_skylib", version = "1.7.0") +bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "platforms", version = "0.0.10") -bazel_dep(name = "rules_python", version = "0.32.2") +bazel_dep(name = "rules_python", version = "0.33.2") -internal_deps = use_extension("@rules_ophiuchus//python:extensions.bzl", "internal_deps") -use_repo(internal_deps, "rules_ophiuchus_pip", "rules_ophiuchus_poetry_deps") +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +use_repo(python, "python_3_11_host") + +internal_deps = use_extension("@rules_ophiuchus//python:internal_deps.bzl", "internal_deps") +use_repo(internal_deps, "rules_ophiuchus_defs", "rules_ophiuchus_pip", "rules_ophiuchus_poetry_deps") diff --git a/README.md b/README.md index 864136a..65ddbd7 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This allows to use platform information of resolved Python toolchains and build Minimum requirements: -* Bazel 6.x +* Bazel 6.x and rules_python with registered Python >= 3.11 toolchain. ## Getting started @@ -17,7 +17,7 @@ Minimum requirements: To import `rules_ophiuchus` in your project, you first need to add it to your `MODULE.bazel` file ```python -bazel_dep(name = "rules_python", version = "0.32.2") +bazel_dep(name = "rules_python", version = "0.33.2") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") diff --git a/examples/aspect_rules_py/MODULE.bazel b/examples/aspect_rules_py/MODULE.bazel index 749c3df..1a8f243 100644 --- a/examples/aspect_rules_py/MODULE.bazel +++ b/examples/aspect_rules_py/MODULE.bazel @@ -4,7 +4,7 @@ module( ) bazel_dep(name = "aspect_rules_py", version = "0.7.3") -bazel_dep(name = "rules_python", version = "0.32.2") +bazel_dep(name = "rules_python", version = "0.33.2") # Register a hermetic Python toolchain rather than rely on a locally-installed # interpreter. diff --git a/examples/cc_toolchain/MODULE.bazel b/examples/cc_toolchain/MODULE.bazel index 030f740..28c28a4 100644 --- a/examples/cc_toolchain/MODULE.bazel +++ b/examples/cc_toolchain/MODULE.bazel @@ -1,4 +1,4 @@ -bazel_dep(name = "rules_python", version = "0.32.2") +bazel_dep(name = "rules_python", version = "0.33.2") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") diff --git a/examples/markers/.bazelversion b/examples/markers/.bazelversion index 960068c..e02168e 100644 --- a/examples/markers/.bazelversion +++ b/examples/markers/.bazelversion @@ -1 +1 @@ -last_downstream_green \ No newline at end of file +last_green \ No newline at end of file diff --git a/examples/markers/MODULE.bazel b/examples/markers/MODULE.bazel index 1dbe1d2..aac38f2 100644 --- a/examples/markers/MODULE.bazel +++ b/examples/markers/MODULE.bazel @@ -1,4 +1,4 @@ -bazel_dep(name = "rules_python", version = "0.32.2") +bazel_dep(name = "rules_python", version = "0.33.2") python_version = "3_12" diff --git a/examples/simple/MODULE.bazel b/examples/simple/MODULE.bazel index 030f740..28c28a4 100644 --- a/examples/simple/MODULE.bazel +++ b/examples/simple/MODULE.bazel @@ -1,4 +1,4 @@ -bazel_dep(name = "rules_python", version = "0.32.2") +bazel_dep(name = "rules_python", version = "0.33.2") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") diff --git a/examples/simple/poetry.lock b/examples/simple/poetry.lock index 6b04dce..a97ae37 100644 --- a/examples/simple/poetry.lock +++ b/examples/simple/poetry.lock @@ -2,17 +2,17 @@ [[package]] name = "boto3" -version = "1.34.131" +version = "1.34.139" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.131-py3-none-any.whl", hash = "sha256:05e388cb937e82be70bfd7eb0c84cf8011ff35cf582a593873ac21675268683b"}, - {file = "boto3-1.34.131.tar.gz", hash = "sha256:dab8f72a6c4e62b4fd70da09e08a6b2a65ea2115b27dd63737142005776ef216"}, + {file = "boto3-1.34.139-py3-none-any.whl", hash = "sha256:98b2a12bcb30e679fa9f60fc74145a39db5ec2ca7b7c763f42896e3bd9b3a38d"}, + {file = "boto3-1.34.139.tar.gz", hash = "sha256:32b99f0d76ec81fdca287ace2c9744a2eb8b92cb62bf4d26d52a4f516b63a6bf"}, ] [package.dependencies] -botocore = ">=1.34.131,<1.35.0" +botocore = ">=1.34.139,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -21,13 +21,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.131" +version = "1.34.139" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.131-py3-none-any.whl", hash = "sha256:13b011d7b206ce00727dcee26548fa3b550db9046d5a0e90ac25a6e6c8fde6ef"}, - {file = "botocore-1.34.131.tar.gz", hash = "sha256:502ddafe1d627fcf1e4c007c86454e5dd011dba7c58bd8e8a5368a79f3e387dc"}, + {file = "botocore-1.34.139-py3-none-any.whl", hash = "sha256:dd1e085d4caa2a4c1b7d83e3bc51416111c8238a35d498e9d3b04f3b63b086ba"}, + {file = "botocore-1.34.139.tar.gz", hash = "sha256:df023d8cf8999d574214dad4645cb90f9d2ccd1494f6ee2b57b1ab7522f6be77"}, ] [package.dependencies] @@ -133,13 +133,13 @@ six = ">=1.5" [[package]] name = "s3transfer" -version = "0.10.1" +version = "0.10.2" description = "An Amazon S3 Transfer Manager" optional = false -python-versions = ">= 3.8" +python-versions = ">=3.8" files = [ - {file = "s3transfer-0.10.1-py3-none-any.whl", hash = "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d"}, - {file = "s3transfer-0.10.1.tar.gz", hash = "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19"}, + {file = "s3transfer-0.10.2-py3-none-any.whl", hash = "sha256:eca1c20de70a39daee580aef4986996620f365c4e0fda6a86100231d62f1bf69"}, + {file = "s3transfer-0.10.2.tar.gz", hash = "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6"}, ] [package.dependencies] diff --git a/examples/torch/MODULE.bazel b/examples/torch/MODULE.bazel index 030f740..28c28a4 100644 --- a/examples/torch/MODULE.bazel +++ b/examples/torch/MODULE.bazel @@ -1,4 +1,4 @@ -bazel_dep(name = "rules_python", version = "0.32.2") +bazel_dep(name = "rules_python", version = "0.33.2") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") diff --git a/examples/torch/poetry.lock b/examples/torch/poetry.lock index 0d5b6b0..fb06f16 100644 --- a/examples/torch/poetry.lock +++ b/examples/torch/poetry.lock @@ -13,13 +13,13 @@ files = [ [[package]] name = "filelock" -version = "3.15.3" +version = "3.15.4" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.15.3-py3-none-any.whl", hash = "sha256:0151273e5b5d6cf753a61ec83b3a9b7d8821c39ae9af9d7ecf2f9e2f17404103"}, - {file = "filelock-3.15.3.tar.gz", hash = "sha256:e1199bf5194a2277273dacd50269f0d87d0682088a3c561c15674ea9005d8635"}, + {file = "filelock-3.15.4-py3-none-any.whl", hash = "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"}, + {file = "filelock-3.15.4.tar.gz", hash = "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb"}, ] [package.extras] @@ -29,13 +29,13 @@ typing = ["typing-extensions (>=4.8)"] [[package]] name = "fsspec" -version = "2024.6.0" +version = "2024.6.1" description = "File-system specification" optional = false python-versions = ">=3.8" files = [ - {file = "fsspec-2024.6.0-py3-none-any.whl", hash = "sha256:58d7122eb8a1a46f7f13453187bfea4972d66bf01618d37366521b1998034cee"}, - {file = "fsspec-2024.6.0.tar.gz", hash = "sha256:f579960a56e6d8038a9efc8f9c77279ec12e6299aa86b0769a7e9c46b94527c2"}, + {file = "fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e"}, + {file = "fsspec-2024.6.1.tar.gz", hash = "sha256:fad7d7e209dd4c1208e3bbfda706620e0da5142bebbd9c384afb95b07e798e49"}, ] [package.extras] @@ -364,13 +364,13 @@ files = [ [[package]] name = "nvidia-nvjitlink-cu12" -version = "12.5.40" +version = "12.5.82" description = "Nvidia JIT LTO Library" optional = false python-versions = ">=3" files = [ - {file = "nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_x86_64.whl", hash = "sha256:d9714f27c1d0f0895cd8915c07a87a1d0029a0aa36acaf9156952ec2a8a12189"}, - {file = "nvidia_nvjitlink_cu12-12.5.40-py3-none-win_amd64.whl", hash = "sha256:c3401dc8543b52d3a8158007a0c1ab4e9c768fcbd24153a48c86972102197ddd"}, + {file = "nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f9b37bc5c8cf7509665cb6ada5aaa0ce65618f2332b7d3e78e9790511f111212"}, + {file = "nvidia_nvjitlink_cu12-12.5.82-py3-none-win_amd64.whl", hash = "sha256:e782564d705ff0bf61ac3e1bf730166da66dd2fe9012f111ede5fc49b64ae697"}, ] [[package]] diff --git a/examples/transitions/MODULE.bazel b/examples/transitions/MODULE.bazel index cf15ac3..571d826 100644 --- a/examples/transitions/MODULE.bazel +++ b/examples/transitions/MODULE.bazel @@ -1,5 +1,5 @@ -bazel_dep(name = "rules_python", version = "0.32.2") -bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "rules_python", version = "0.33.2") +bazel_dep(name = "platforms", version = "0.0.10") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") diff --git a/examples/workspace/README.md b/examples/workspace/README.md index 22cda72..e21fc53 100644 --- a/examples/workspace/README.md +++ b/examples/workspace/README.md @@ -4,12 +4,15 @@ Explicitly set `--enable_bzlmod=false` to disable bzlmod. Add rules_ophiuchus to `WORKSPACE` file as ``` -load("@rules_ophiuchus//python:poetry_parse.bzl", "poetry_parse") load("@rules_ophiuchus//python:repositories.bzl", install_poetry_dependencies = "install_dependencies") -install_poetry_dependencies() +install_poetry_dependencies("black_mamba", "3.11") + +load("@rules_ophiuchus//python:poetry_parse.bzl", "poetry_parse") + poetry_parse( name = "poetry", lock = "//:poetry.lock", ) ``` +where `black_mamba` and 3.11 are toolchain name and version for lock files processing. diff --git a/examples/workspace/WORKSPACE b/examples/workspace/WORKSPACE index 35ff15a..6f8e3cd 100644 --- a/examples/workspace/WORKSPACE +++ b/examples/workspace/WORKSPACE @@ -1,12 +1,28 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_python", + sha256 = "e3f1cc7a04d9b09635afb3130731ed82b5f58eadc8233d4efb59944d92ffc06f", + strip_prefix = "rules_python-0.33.2", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.33.2/rules_python-0.33.2.tar.gz", +) + +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") + +py_repositories() + +python_register_toolchains("black_mamba", "3.11") + local_repository( name = "rules_ophiuchus", path = "../..", ) -load("@rules_ophiuchus//python:poetry_parse.bzl", "poetry_parse") load("@rules_ophiuchus//python:repositories.bzl", install_poetry_dependencies = "install_dependencies") -install_poetry_dependencies() +install_poetry_dependencies("black_mamba", "3.11") + +load("@rules_ophiuchus//python:poetry_parse.bzl", "poetry_parse") poetry_parse( name = "poetry", diff --git a/examples/workspace_rules_python/.bazelrc b/examples/workspace_multi/.bazelrc similarity index 100% rename from examples/workspace_rules_python/.bazelrc rename to examples/workspace_multi/.bazelrc diff --git a/examples/workspace_rules_python/.bazelversion b/examples/workspace_multi/.bazelversion similarity index 100% rename from examples/workspace_rules_python/.bazelversion rename to examples/workspace_multi/.bazelversion diff --git a/examples/workspace_rules_python/BUILD b/examples/workspace_multi/BUILD similarity index 72% rename from examples/workspace_rules_python/BUILD rename to examples/workspace_multi/BUILD index 7339c03..d0c7323 100644 --- a/examples/workspace_rules_python/BUILD +++ b/examples/workspace_multi/BUILD @@ -1,8 +1,8 @@ -load("@python//3.12:defs.bzl", py_test_3_12 = "py_test") -load("@python//3.11:defs.bzl", py_test_3_11 = "py_test") -load("@python//3.10:defs.bzl", py_test_3_10 = "py_test") -load("@python//3.9:defs.bzl", py_test_3_9 = "py_test") -load("@python//3.8:defs.bzl", py_test_3_8 = "py_test") +load("@copperhead//3.10:defs.bzl", py_test_3_10 = "py_test") +load("@copperhead//3.11:defs.bzl", py_test_3_11 = "py_test") +load("@copperhead//3.12:defs.bzl", py_test_3_12 = "py_test") +load("@copperhead//3.8:defs.bzl", py_test_3_8 = "py_test") +load("@copperhead//3.9:defs.bzl", py_test_3_9 = "py_test") [ py_test( diff --git a/examples/workspace_rules_python/README.md b/examples/workspace_multi/README.md similarity index 62% rename from examples/workspace_rules_python/README.md rename to examples/workspace_multi/README.md index 874fd3b..849034b 100644 --- a/examples/workspace_rules_python/README.md +++ b/examples/workspace_multi/README.md @@ -4,12 +4,21 @@ Explicitly set `--enable_bzlmod=false` to disable bzlmod. Add rules_ophiuchus to `WORKSPACE` file as ``` -load("@rules_ophiuchus//python:poetry_parse.bzl", "poetry_parse") load("@rules_ophiuchus//python:repositories.bzl", install_poetry_dependencies = "install_dependencies") -install_poetry_dependencies() +install_poetry_dependencies("copperhead_3_11", "3.11") + +load("@rules_ophiuchus//python:poetry_parse.bzl", "poetry_parse") + poetry_parse( - name = "poetry", + name = "poetry_repo1", + lock = "//:poetry.lock", +) + +poetry_parse( + name = "poetry_repo2", lock = "//:poetry.lock", ) ``` + +where `copperhead_3_11` and 3.11 are toolchain name and version for lock files processing. diff --git a/examples/workspace_rules_python/WORKSPACE b/examples/workspace_multi/WORKSPACE similarity index 78% rename from examples/workspace_rules_python/WORKSPACE rename to examples/workspace_multi/WORKSPACE index 22a4c08..623a9c7 100644 --- a/examples/workspace_rules_python/WORKSPACE +++ b/examples/workspace_multi/WORKSPACE @@ -2,9 +2,9 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "rules_python", - sha256 = "4912ced70dc1a2a8e4b86cec233b192ca053e82bc72d877b98e126156e8f228d", - strip_prefix = "rules_python-0.32.2", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.32.2/rules_python-0.32.2.tar.gz", + sha256 = "e3f1cc7a04d9b09635afb3130731ed82b5f58eadc8233d4efb59944d92ffc06f", + strip_prefix = "rules_python-0.33.2", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.33.2/rules_python-0.33.2.tar.gz", ) load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_multi_toolchains") @@ -20,7 +20,7 @@ python_versions = [ ] python_register_multi_toolchains( - name = "python", + name = "copperhead", default_version = python_versions[0], python_versions = python_versions, register_coverage_tool = True, @@ -31,10 +31,11 @@ local_repository( path = "../..", ) -load("@rules_ophiuchus//python:poetry_parse.bzl", "poetry_parse") load("@rules_ophiuchus//python:repositories.bzl", install_poetry_dependencies = "install_dependencies") -install_poetry_dependencies() +install_poetry_dependencies("copperhead_3_11", "3.11") + +load("@rules_ophiuchus//python:poetry_parse.bzl", "poetry_parse") poetry_parse( name = "poetry_repo1", diff --git a/examples/workspace_rules_python/poetry.lock b/examples/workspace_multi/poetry.lock similarity index 100% rename from examples/workspace_rules_python/poetry.lock rename to examples/workspace_multi/poetry.lock diff --git a/examples/workspace_rules_python/pyproject.toml b/examples/workspace_multi/pyproject.toml similarity index 100% rename from examples/workspace_rules_python/pyproject.toml rename to examples/workspace_multi/pyproject.toml diff --git a/examples/workspace_rules_python/test.py b/examples/workspace_multi/test.py similarity index 100% rename from examples/workspace_rules_python/test.py rename to examples/workspace_multi/test.py diff --git a/python/extensions.bzl b/python/extensions.bzl index abd55e0..4cfa61e 100644 --- a/python/extensions.bzl +++ b/python/extensions.bzl @@ -1,4 +1,3 @@ -load("@rules_ophiuchus//python:repositories.bzl", "install_dependencies") load("@rules_ophiuchus//python:poetry_parse.bzl", "poetry_parse") def _poetry_impl(module_ctx): @@ -24,13 +23,3 @@ poetry = module_extension( ), }, ) - -def _internal_deps_impl(module_ctx): - install_dependencies() - -internal_deps = module_extension( - implementation = _internal_deps_impl, - tag_classes = { - "install": tag_class(dict()), - }, -) diff --git a/python/internal_deps.bzl b/python/internal_deps.bzl new file mode 100644 index 0000000..bf658c8 --- /dev/null +++ b/python/internal_deps.bzl @@ -0,0 +1,11 @@ +load("@rules_ophiuchus//python:repositories.bzl", "install_dependencies") + +def _internal_deps_impl(module_ctx): + install_dependencies("python_3_11", "3.11") + +internal_deps = module_extension( + implementation = _internal_deps_impl, + tag_classes = { + "install": tag_class(dict()), + }, +) diff --git a/python/poetry.bzl b/python/poetry.bzl index 789136b..b654f26 100644 --- a/python/poetry.bzl +++ b/python/poetry.bzl @@ -1,7 +1,9 @@ -load("@rules_ophiuchus_poetry_deps//:defs.bzl", _python = "python") load("//python/private:poetry_deps.bzl", _get_imports = "get_imports") def _poetry_update_impl(ctx): + python_toolchain = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"] + runtime_info = python_toolchain.py3_runtime + script = """#!{python} import runpy @@ -23,7 +25,7 @@ if __name__ == "__main__": sys.argv = [sys.argv[0], "lock", f"--directory={{dir}}"{update}, *sys.argv[1:]] runpy.run_module("poetry", run_name="__main__", alter_sys=True) """.format( - python = _python, + python = runtime_info.interpreter.short_path, deps = repr(["../{}".format(path) for path in _get_imports(ctx.attr._poetry_deps).to_list()]), toml = ctx.attr.toml.files.to_list().pop().short_path, lock = ctx.attr.lock.files.to_list().pop().short_path, @@ -32,7 +34,7 @@ if __name__ == "__main__": output = ctx.actions.declare_file(ctx.label.name + ".update") ctx.actions.write(output, script, is_executable = True) - runfiles = ctx.runfiles(ctx.attr.toml.files.to_list() + ctx.attr.lock.files.to_list()) + runfiles = ctx.runfiles(transitive_files = depset(transitive = [ctx.attr.toml.files, ctx.attr.lock.files, runtime_info.files])) runfiles = runfiles.merge(ctx.attr._poetry_deps.default_runfiles) return [ @@ -48,4 +50,7 @@ poetry_update = rule( "update": attr.bool(default = True), "_poetry_deps": attr.label(default = "@rules_ophiuchus_poetry_deps//:pkg"), }, + toolchains = [ + "@bazel_tools//tools/python:toolchain_type", + ], ) diff --git a/python/poetry_deps.bzl b/python/poetry_deps.bzl index cee9046..32fc581 100644 --- a/python/poetry_deps.bzl +++ b/python/poetry_deps.bzl @@ -1,10 +1,8 @@ -load("@rules_python//python:defs.bzl", StarPyInfo = "PyInfo") load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//lib:versions.bzl", "versions") load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") -load("//python/private:poetry_deps.bzl", _derive_environment_markers = "derive_environment_markers", _include_dep = "include_dep") -load("//python/private:poetry_deps.bzl", _get_imports = "get_imports", _get_transitive_sources = "get_transitive_sources") -load("//python/private:poetry_deps.bzl", _DEFAULT_PLATFORMS = "DEFAULT_PLATFORMS") +load("@rules_python//python:defs.bzl", StarPyInfo = "PyInfo") +load("//python/private:poetry_deps.bzl", _DEFAULT_PLATFORMS = "DEFAULT_PLATFORMS", _derive_environment_markers = "derive_environment_markers", _get_imports = "get_imports", _get_transitive_sources = "get_transitive_sources", _include_dep = "include_dep") PYTHON_BINARY = ["bin/python3", "python/py3wrapper.sh"] @@ -47,7 +45,7 @@ def _package_impl(ctx): # Get Python target toolchain and corresponding tags py_toolchain = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"] py_runtime_info = py_toolchain.py3_runtime - runtime_tag, tags = _derive_environment_markers(py_runtime_info.interpreter.path, ctx.attr.platforms, ctx.attr.host_platform) + runtime_tag, tags = _derive_environment_markers(py_runtime_info.interpreter.path, ctx.attr.platforms, ctx.attr.system_platform) python_version = tags["python_version"] platform_tags = tags["platform_tags"] @@ -120,11 +118,10 @@ def _package_impl(ctx): deps = [dep for dep in ctx.attr.deps if _include_dep(dep, ctx.attr.markers, tags)] transitive_imports = [_get_imports(dep) for dep in deps] transitive_depsets = [_get_transitive_sources(dep) for dep in deps] - runfiles = [output] + [item for dep in transitive_depsets for item in dep.to_list()] files = depset([output], transitive = transitive_depsets) imports = depset([output.short_path.replace("../", "")], transitive = transitive_imports) return [ - DefaultInfo(files = files, runfiles = ctx.runfiles(files = runfiles)), + DefaultInfo(files = files, runfiles = ctx.runfiles(transitive_files = files)), PyInfo(transitive_sources = files, imports = imports), ] + ([] if PyInfo == StarPyInfo else [StarPyInfo(transitive_sources = files, imports = imports)]) @@ -145,7 +142,7 @@ package = rule( "Default value corresponds to platforms defined at " + "https://github.com/bazelbuild/rules_python/blob/23cf6b66/python/versions.bzl#L231-L277", ), - "host_platform": attr.string(doc = "The host platform environment markers as a JSON string"), + "system_platform": attr.string(doc = "The system platform environment markers as a JSON string"), "_poetry_deps": attr.label(default = ":poetry_deps", cfg = "exec", executable = True), }, toolchains = [ diff --git a/python/poetry_deps.py b/python/poetry_deps.py index 4d869b0..13f7a2d 100644 --- a/python/poetry_deps.py +++ b/python/poetry_deps.py @@ -119,8 +119,7 @@ def install(args): "--no-dependencies", "--disable-pip-version-check", "--use-pep517", - # "--quiet", - "--verbose", + "--quiet", ] if args.cc_toolchain is not None: diff --git a/python/private/poetry_deps.bzl b/python/private/poetry_deps.bzl index f05d934..f64db5c 100644 --- a/python/private/poetry_deps.bzl +++ b/python/private/poetry_deps.bzl @@ -1,6 +1,7 @@ +load("@rules_ophiuchus_defs//:defs.bzl", _python_toolchain_prefix = "python_toolchain_prefix", _python_version = "python_version") +load("@rules_python//python:defs.bzl", StarPyInfo = "PyInfo") load("@rules_python//python:versions.bzl", _MINOR_MAPPING = "MINOR_MAPPING") load("//python:markers.bzl", "evaluate", "parse") -load("@rules_python//python:defs.bzl", StarPyInfo = "PyInfo") # Environment Markers https://peps.python.org/pep-0508/#environment-markers # @@ -19,25 +20,18 @@ DEFAULT_PLATFORMS = { "x86_64-unknown-linux-gnu": """{"os_name": "posix", "platform_machine": "x86_64", "platform_system": "Linux", "platform_tags": ["linux_x86_64", "manylinux2014_x86_64", "manylinux_2_12_x86_64", "manylinux_2_17_x86_64", "manylinux_2_27_x86_64", "manylinux_2_28_x86_64"], "sys_platform": "linux"}""", } -def _collect_version(parts): - version = [] - for index in range(len(parts)): - if not parts[index].isdigit(): - break - - version.append(parts[index]) - - return ".".join(version) - def _get_python_version(interpreter): - parts = interpreter.split("_") - for index in range(len(parts)): - if parts[index].endswith("python3"): - return "3." + _collect_version(parts[index + 1:]) - elif parts[index].endswith("python"): - return _collect_version(parts[index + 1:]) + parts = interpreter.replace(".", "_").split(_python_toolchain_prefix.replace(".", "_")) + for part in parts: + tokens = [token for token in part.split("_") if token] + for index in range(len(tokens)): + if not tokens[index].isdigit(): + break + version = ".".join(tokens[:index]) + if version: + return version - return "host" + return _python_version def derive_environment_markers(interpreter, interpreter_markers, host_tags): python_version = _get_python_version(interpreter) diff --git a/python/private/poetry_parse.bzl b/python/private/poetry_parse.bzl index cf6f3d7..c4bdbc8 100644 --- a/python/private/poetry_parse.bzl +++ b/python/private/poetry_parse.bzl @@ -1,3 +1,5 @@ +load("@rules_ophiuchus_defs//:defs.bzl", _python_host = "python_host") + def dfs_cycles(graph): # Detect pip dependency cycles u->...->v...->w->v and remove edges w->v # Graph DFS with path tracking @@ -24,7 +26,7 @@ def dfs_cycles(graph): def normalize_dep_name(dep_name): return dep_name.strip().strip('"').strip("'").replace("_", "-").replace(".", "-").lower() -def parse_lock_file(data, platforms = None, generate_extras = True, host_platform_tags = None): +def parse_lock_file(data, platforms = None, generate_extras = True, system_platform_tags = None): _MARKERS = "markers = " _SOURCE_URL = "url = " _SOURCE_TYPE = "type = " @@ -112,7 +114,7 @@ package( name = "{name}", constraint = "{name}=={version}",{description} files = {{{files} - }},{deps}{markers}{source_urls}{extra_index_urls}{platforms}{host_platform_tags} + }},{deps}{markers}{source_urls}{extra_index_urls}{platforms}{system_platform_tags} visibility = [{visibility}], ) """.format( @@ -125,7 +127,7 @@ package( source_urls = "\n source_urls = [\n{}\n ],".format("\n".join([" " + url + "," for url in source_urls])) if source_urls else "", extra_index_urls = "\n extra_index_urls = [{}],".format(", ".join(extra_index_urls)) if extra_index_urls else "", platforms = "\n platforms = {},".format(platforms) if platforms else "", - host_platform_tags = "\n host_platform = {},".format(host_platform_tags) if host_platform_tags else "", + system_platform_tags = "\n system_platform = {},".format(system_platform_tags) if system_platform_tags else "", visibility = ", ".join(['"{}"'.format(vis) for vis in visibility]), ) @@ -144,7 +146,8 @@ package( return result -def get_host_platform_tags(rctx): +def get_system_platform_tags(rctx): + # DEPRECATED: the sytem interpreter must not be used # Return a JSON string with a dict values as described in # https://peps.python.org/pep-0508/#environment-markers and # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag @@ -175,14 +178,21 @@ print(json.dumps(json.dumps(dict( return "{}" def _poetry_parse_impl(rctx): - header = "# Autogenerated file _poetry_parse_impl at rules_ophiuchus/python/private/poetry_parse.bzl:175" - rules_repository = str(rctx.path(rctx.attr._self)).split("/")[-4] + interpreter = rctx.path(rctx.attr._python_host) + command = "import tomllib; print('{0}', 1); print('ok'); print(tomllib.load(open('{0}', 'rb')))".format(rctx.path(rctx.attr.lock)) + exec_result = rctx.execute([interpreter, "-c", command], quiet = True) + if exec_result.return_code: + fail("Parsing {} failed with exit code {}\n{}\n".format(rctx.attr.lock, exec_result.return_code, exec_result.stderr)) + + self = str(rctx.path(rctx.attr._self)).split("/external/")[-1] + header = "# Autogenerated file by _poetry_parse_impl in {}".format(self) + rules_repository = self.split("/", 1)[0] rules_repository = ("@@" if "~" in rules_repository else "@") + rules_repository - host_platform_tags_variable = "host_platform_tags" + system_platform_tags_variable = "system_platform_tags" prefix = '''{header}\n\nload("{name}//python:poetry_deps.bzl", "package")'''.format(header = header, name = rules_repository) - host_platform_tags = """{k} = {v}""".format(k = host_platform_tags_variable, v = get_host_platform_tags(rctx)) - lock_file_content = parse_lock_file(rctx.read(rctx.attr.lock), rctx.attr.platforms, rctx.attr.generate_extras, host_platform_tags_variable) - rctx.file("BUILD", "{}\n\n{}\n\n{}".format(prefix, host_platform_tags, lock_file_content)) + system_platform_tags = """{k} = {v}""".format(k = system_platform_tags_variable, v = get_system_platform_tags(rctx)) + lock_file_content = parse_lock_file(rctx.read(rctx.attr.lock), rctx.attr.platforms, rctx.attr.generate_extras, system_platform_tags_variable) + rctx.file("BUILD", "{}\n\n{}\n\n{}".format(prefix, system_platform_tags, lock_file_content)) rctx.file("WORKSPACE") poetry_parse = repository_rule( @@ -198,6 +208,10 @@ poetry_parse = repository_rule( "platforms": attr.string_dict( doc = "The mapping of interpter substrings to Python platform tags and environment markers as a JSON string", ), + "_python_host": attr.label( + allow_single_file = True, + default = _python_host, + ), "_self": attr.label( allow_single_file = True, default = ":poetry_parse.bzl", diff --git a/python/repositories.bzl b/python/repositories.bzl index 06d6444..292093c 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -13,21 +13,25 @@ _INTERNAL_DEPS = [ ), ] -def _poetry_deps_repo_impl(ctx): - poetry_version = "1.8.3" - - # Intentionally use a host default interpreter as the repository only used in host tooling targets - # This may lead to inconsistency if the repository will be used with a different toolchain - python = ctx.which("python.exe" if "win" in ctx.os.name else "python3") - if python: - ctx.execute( - [python, "-m", "pip", "install", "poetry=={}".format(poetry_version), "--target", ctx.attr.output], - environment = { - "PYTHONPATH": ":".join([str(ctx.path(dep).dirname) for dep in ctx.attr.deps]), - }, - ) +_POETRY_VERSION = "1.8.3" + +# TODO: drop it +def _poetry_deps_repo_impl(rctx): + interpreter = rctx.path(rctx.attr.python_host) + + rctx.execute( + [interpreter, "-m", "pip", "install", "poetry=={}".format(_POETRY_VERSION), "--target", rctx.attr.output], + environment = { + "PYTHONPATH": ":".join([str(rctx.path(dep).dirname) for dep in rctx.attr.deps]), + }, + ) - build_file_content = """py_library( + generator = "{} by {}".format(rctx.attr.generator_name, rctx.attr.generator_function) + rctx.file("BUILD", """# Generated by {generator} + +exports_files(["defs.bzl"]) + +py_library( name = "pkg", srcs = glob(include=["{output}/**/*.py"]), data = glob(include=["{output}/**/*"], exclude=[ @@ -39,25 +43,53 @@ def _poetry_deps_repo_impl(ctx): ]), visibility = ["//visibility:public"], imports = ["{output}"], -)""".format(output = ctx.attr.output) - - else: - build_file_content = "# Poetry deps require an installed host Python 3 interpreter" # TODO: switch to a host interpreter from rules_python - - ctx.file("BUILD", build_file_content) - ctx.file("defs.bzl", 'python = "{}"'.format(python)) +)""".format(generator = generator, output = rctx.attr.output)) +# TODO: drop it poetry_deps_repo = repository_rule( implementation = _poetry_deps_repo_impl, attrs = { "deps": attr.label_list(), "output": attr.string(), + "python_host": attr.label(), }, ) -def install_dependencies(auth_patterns = {}, netrc = ""): +def _internal_definitions_repo_impl(rctx): + generator = "{} by {}".format(rctx.attr.generator_name, rctx.attr.generator_function) + + rctx.file("BUILD", "") + rctx.file("defs.bzl", """# Generated by {generator} +python_host = "{python_host}" +python_version = "{python_version}" +python_toolchain_prefix = "{python_toolchain_prefix}" +""".format( + generator = generator, + python_host = rctx.attr.python_host, + python_version = rctx.attr.python_version, + python_toolchain_prefix = rctx.attr.python_toolchain_prefix, + )) + +internal_definitions_repo = repository_rule( + implementation = _internal_definitions_repo_impl, + attrs = { + "python_host": attr.label(), + "python_version": attr.string(), + "python_toolchain_prefix": attr.string(), + }, +) + +def install_dependencies(toolchain_prefix, python_version, auth_patterns = {}, netrc = ""): prefix = "rules_ophiuchus_" + internal_definitions_repo( + name = prefix + "defs", + # Ref: https://github.com/bazelbuild/rules_python/blob/084b877c/python/repositories.bzl#L653-L658 + python_host = "@{name}_host//:python".format(name = toolchain_prefix), + python_version = python_version, + python_toolchain_prefix = toolchain_prefix.split("_3_")[0], + ) + for (name, url, sha256, patches) in _INTERNAL_DEPS: maybe( http_archive, @@ -84,9 +116,11 @@ def install_dependencies(auth_patterns = {}, netrc = ""): """, ) + # TODO: Switch to internal poetry.lock file poetry_deps_repo( name = prefix + "poetry_deps", output = "site-packages", + python_host = "@{name}_host//:python".format(name = toolchain_prefix), deps = [ "@{}pip//:BUILD.bazel".format(prefix), ],