From 27949c6df412281b2be021b74460be53f1f8af2a Mon Sep 17 00:00:00 2001 From: Kyle Edwards <4302519+difyrrwrzd@users.noreply.github.com> Date: Fri, 31 May 2024 10:46:11 -0400 Subject: [PATCH] Use rapids-build-backend (#145) Contributes to rapidsai/build-planning#31. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Jake Awe (https://github.com/AyodeAwe) - Dante Gama Dessavre (https://github.com/dantegd) - James Lamb (https://github.com/jameslamb) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cuvs/pull/145 --- .pre-commit-config.yaml | 2 +- build.sh | 2 +- ci/build_python.sh | 3 -- ci/build_wheel.sh | 33 +----------- ci/release/update-version.sh | 8 +-- .../all_cuda-118_arch-aarch64.yaml | 5 +- .../all_cuda-118_arch-x86_64.yaml | 5 +- .../all_cuda-122_arch-aarch64.yaml | 5 +- .../all_cuda-122_arch-x86_64.yaml | 5 +- conda/recipes/cuvs/meta.yaml | 1 + dependencies.yaml | 52 ++++++++++++++----- python/cuvs/cuvs/_version.py | 20 +++++-- python/cuvs/cuvs/test/test_version.py | 12 +++++ python/cuvs/pyproject.toml | 25 +++++---- 14 files changed, 102 insertions(+), 76 deletions(-) create mode 100644 python/cuvs/cuvs/test/test_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b1de648..72e7dff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,7 +93,7 @@ repos: args: ["--toml", "pyproject.toml"] exclude: (?x)^(^CHANGELOG.md$) - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.8.0 + rev: v1.13.11 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/build.sh b/build.sh index 822b845..678df7b 100755 --- a/build.sh +++ b/build.sh @@ -391,7 +391,7 @@ fi if (( ${NUMARGS} == 0 )) || hasArg python; then SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL}" \ - python -m pip install --no-build-isolation --no-deps -vvv ${REPODIR}/python/cuvs + python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true ${REPODIR}/python/cuvs fi # Build the cuvs Rust bindings diff --git a/ci/build_python.sh b/ci/build_python.sh index c50a413..82ecba4 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -21,12 +21,9 @@ package_dir="python" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) export RAPIDS_PACKAGE_VERSION=${version} echo "${version}" > VERSION -sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${package_name}/_version.py" - # TODO: Remove `--no-test` flags once importing on a CPU # node works correctly rapids-conda-retry mambabuild \ diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 4f9f96d..29d9ce4 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -10,40 +10,9 @@ underscore_package_name=$(echo "${package_name}" | tr "-" "_") source rapids-configure-sccache source rapids-date-string -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) - RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -# Patch project metadata files to include the CUDA version suffix and version override. -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${underscore_package_name}/_version.py" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "s/pylibraft(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} - - -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} -fi +rapids-generate-version > VERSION cd "${package_dir}" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 0082461..69fbb74 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -58,15 +58,15 @@ DEPENDENCIES=( ) for FILE in dependencies.yaml conda/environments/*.yaml; do for DEP in "${DEPENDENCIES[@]}"; do - sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}\.*/g" ${FILE}; + sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE}; done - sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_SHORT_TAG_PEP440}\.*/g" ${FILE}; + sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE}; done for FILE in python/*/pyproject.toml; do for DEP in "${DEPENDENCIES[@]}"; do - sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*\"/g" ${FILE} + sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" ${FILE} done - sed_runner "/\"ucx-py==/ s/==.*\"/==${NEXT_UCX_PY_SHORT_TAG_PEP440}.*\"/g" ${FILE} + sed_runner "/\"ucx-py==/ s/==.*\"/==${NEXT_UCX_PY_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" ${FILE} done for FILE in .github/workflows/*.yaml; do diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 3540039..9a50bb8 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -43,11 +43,12 @@ dependencies: - nvcc_linux-aarch64=11.8 - pre-commit - pydata-sphinx-theme -- pylibraft==24.8.* +- pylibraft==24.8.*,>=0.0.0a0 - pytest-cov - pytest==7.* +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - recommonmark -- rmm==24.8.* +- rmm==24.8.*,>=0.0.0a0 - rust - scikit-build-core>=0.7.0 - scikit-learn diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 51170b4..a0581ff 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -43,11 +43,12 @@ dependencies: - nvcc_linux-64=11.8 - pre-commit - pydata-sphinx-theme -- pylibraft==24.8.* +- pylibraft==24.8.*,>=0.0.0a0 - pytest-cov - pytest==7.* +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - recommonmark -- rmm==24.8.* +- rmm==24.8.*,>=0.0.0a0 - rust - scikit-build-core>=0.7.0 - scikit-learn diff --git a/conda/environments/all_cuda-122_arch-aarch64.yaml b/conda/environments/all_cuda-122_arch-aarch64.yaml index b430fa8..9b151e4 100644 --- a/conda/environments/all_cuda-122_arch-aarch64.yaml +++ b/conda/environments/all_cuda-122_arch-aarch64.yaml @@ -39,11 +39,12 @@ dependencies: - numpydoc - pre-commit - pydata-sphinx-theme -- pylibraft==24.8.* +- pylibraft==24.8.*,>=0.0.0a0 - pytest-cov - pytest==7.* +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - recommonmark -- rmm==24.8.* +- rmm==24.8.*,>=0.0.0a0 - rust - scikit-build-core>=0.7.0 - scikit-learn diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index 1091337..d4b04da 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -39,11 +39,12 @@ dependencies: - numpydoc - pre-commit - pydata-sphinx-theme -- pylibraft==24.8.* +- pylibraft==24.8.*,>=0.0.0a0 - pytest-cov - pytest==7.* +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - recommonmark -- rmm==24.8.* +- rmm==24.8.*,>=0.0.0a0 - rust - scikit-build-core>=0.7.0 - scikit-learn diff --git a/conda/recipes/cuvs/meta.yaml b/conda/recipes/cuvs/meta.yaml index 349923b..b0f0c3f 100644 --- a/conda/recipes/cuvs/meta.yaml +++ b/conda/recipes/cuvs/meta.yaml @@ -55,6 +55,7 @@ requirements: - libcuvs {{ version }} - python x.x - rmm ={{ minor_version }} + - rapids-build-backend>=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.7.0 - setuptools run: diff --git a/dependencies.yaml b/dependencies.yaml index 97dae89..22325d5 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -7,6 +7,7 @@ files: arch: [x86_64, aarch64] includes: - build + - rapids_build - build_py_cuvs - cuda - cuda_version @@ -46,13 +47,13 @@ files: - docs - py_version - rust - - build + - rapids_build - cuda rust: output: none includes: - cuda_version - - build + - rapids_build - cuda - rust py_build_py_cuvs: @@ -62,6 +63,14 @@ files: table: build-system includes: - build + py_rapids_build_py_cuvs: + output: pyproject + pyproject_dir: python/cuvs + extras: + table: tool.rapids-build-backend + key: requires + includes: + - rapids_build - build_py_cuvs py_run_py_cuvs: output: pyproject @@ -88,6 +97,17 @@ channels: - nvidia dependencies: build: + common: + - output_types: [conda, requirements, pyproject] + packages: + - rapids-build-backend>=0.3.0,<0.4.0.dev0 + - output_types: [conda] + packages: + - scikit-build-core>=0.7.0 + - output_types: [requirements, pyproject] + packages: + - scikit-build-core[pyproject]>=0.7.0 + rapids_build: common: - output_types: [conda, requirements, pyproject] packages: @@ -99,10 +119,6 @@ dependencies: - c-compiler - cxx-compiler - nccl>=2.9.9 - - scikit-build-core>=0.7.0 - - output_types: [requirements, pyproject] - packages: - - scikit-build-core[pyproject]>=0.7.0 specific: - output_types: conda matrices: @@ -141,8 +157,8 @@ dependencies: common: - output_types: [conda] packages: - - &rmm_conda rmm==24.8.* - - &pylibraft_conda pylibraft==24.8.* + - &rmm_conda rmm==24.8.*,>=0.0.0a0 + - &pylibraft_conda pylibraft==24.8.*,>=0.0.0a0 - dlpack>=0.8,<1.0 - output_types: requirements packages: @@ -157,19 +173,23 @@ dependencies: cuda: "12.*" packages: - &cuda_python12 cuda-python>=12.0,<13.0a0 - - matrix: # All CUDA 11 versions + - matrix: + cuda: "11.*" packages: - &cuda_python11 cuda-python>=11.7.1,<12.0a0 + - matrix: + packages: + - &cuda_python cuda-python - output_types: [requirements, pyproject] matrices: - matrix: {cuda: "12.*"} packages: - - &rmm_cu12 rmm-cu12==24.8.* - - &pylibraft_cu12 pylibraft-cu12==24.8.* + - &rmm_cu12 rmm-cu12==24.8.*,>=0.0.0a0 + - &pylibraft_cu12 pylibraft-cu12==24.8.*,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - &rmm_cu11 rmm-cu11==24.8.* - - &pylibraft_cu11 pylibraft-cu11==24.8.* + - &rmm_cu11 rmm-cu11==24.8.*,>=0.0.0a0 + - &pylibraft_cu11 pylibraft-cu11==24.8.*,>=0.0.0a0 - {matrix: null, packages: [*rmm_conda, *pylibraft_conda] } checks: common: @@ -373,9 +393,13 @@ dependencies: cuda: "12.*" packages: - *cuda_python12 - - matrix: # All CUDA 11 versions + - matrix: + cuda: "11.*" packages: - *cuda_python11 + - matrix: + packages: + - *cuda_python - output_types: [requirements, pyproject] matrices: - matrix: {cuda: "12.*"} diff --git a/python/cuvs/cuvs/_version.py b/python/cuvs/cuvs/_version.py index 72ab982..6436ee0 100644 --- a/python/cuvs/cuvs/_version.py +++ b/python/cuvs/cuvs/_version.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,22 @@ # limitations under the License. # - import importlib.resources __version__ = ( - importlib.resources.files("cuvs").joinpath("VERSION").read_text().strip() + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() ) -__git_commit__ = "" +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/cuvs/cuvs/test/test_version.py b/python/cuvs/cuvs/test/test_version.py new file mode 100644 index 0000000..9f2c0dd --- /dev/null +++ b/python/cuvs/cuvs/test/test_version.py @@ -0,0 +1,12 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import cuvs + + +def test_version_constants_are_populated(): + # __git_commit__ will only be non-empty in a built distribution + assert isinstance(cuvs.__git_commit__, str) + + # __version__ should always be non-empty + assert isinstance(cuvs.__version__, str) + assert len(cuvs.__version__) > 0 diff --git a/python/cuvs/pyproject.toml b/python/cuvs/pyproject.toml index a9660a8..4fda501 100644 --- a/python/cuvs/pyproject.toml +++ b/python/cuvs/pyproject.toml @@ -15,15 +15,10 @@ [build-system] requires = [ - "cmake>=3.26.4", - "cuda-python>=11.7.1,<12.0a0", - "cython>=3.0.0", - "ninja", - "pylibraft==24.8.*", - "rmm==24.8.*", + "rapids-build-backend>=0.3.0,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -build-backend = "scikit_build_core.build" +build-backend = "rapids_build_backend.build" [project] name = "cuvs" @@ -36,9 +31,9 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cuda-python>=11.7.1,<12.0a0", + "cuda-python", "numpy>=1.23,<2.0a0", - "pylibraft==24.8.*", + "pylibraft==24.8.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -121,6 +116,18 @@ provider = "scikit_build_core.metadata.regex" input = "cuvs/VERSION" regex = "(?P.*)" +[tool.rapids-build-backend] +requires = [ + "cmake>=3.26.4", + "cuda-python", + "cython>=3.0.0", + "ninja", + "pylibraft==24.8.*,>=0.0.0a0", + "rmm==24.8.*,>=0.0.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" + [tool.pytest.ini_options] filterwarnings = [ "error",