Use rapids-build-backend (#145)

Contributes to rapidsai/build-planning#31.

Authors:
  - Kyle Edwards (https://github.com/KyleFromNVIDIA)

Approvers:
  - Jake Awe (https://github.com/AyodeAwe)
  - Dante Gama Dessavre (https://github.com/dantegd)
  - James Lamb (https://github.com/jameslamb)
  - Bradley Dice (https://github.com/bdice)

URL: rapidsai/cuvs#145

.pre-commit-config.yaml

@@ -93,7 +93,7 @@ repos:
                 args: ["--toml", "pyproject.toml"]
                 exclude: (?x)^(^CHANGELOG.md$)
       - repo: https://github.com/rapidsai/dependency-file-generator
-        rev: v1.8.0
+        rev: v1.13.11
         hooks:
             - id: rapids-dependency-file-generator
               args: ["--clean"]

build.sh

@@ -391,7 +391,7 @@ fi
 if (( ${NUMARGS} == 0 )) || hasArg python; then
     SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \
         SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL}" \
-        python -m pip install --no-build-isolation --no-deps -vvv ${REPODIR}/python/cuvs
+        python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true ${REPODIR}/python/cuvs
 fi
 
 # Build the cuvs Rust bindings

ci/build_python.sh

@@ -21,12 +21,9 @@ package_dir="python"
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
 version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
 export RAPIDS_PACKAGE_VERSION=${version}
 echo "${version}" > VERSION
 
-sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${package_name}/_version.py"
-
 # TODO: Remove `--no-test` flags once importing on a CPU
 # node works correctly
 rapids-conda-retry mambabuild \

ci/build_wheel.sh

@@ -10,40 +10,9 @@ underscore_package_name=$(echo "${package_name}" | tr "-" "_")
 source rapids-configure-sccache
 source rapids-date-string
 
-version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
-
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# This is the version of the suffix with a preceding hyphen. It's used
-# everywhere except in the final wheel name.
-PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
-
-# Patch project metadata files to include the CUDA version suffix and version override.
-pyproject_file="${package_dir}/pyproject.toml"
-version_file="${package_dir}/${underscore_package_name}/_version.py"
-
-sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
-echo "${version}" > VERSION
-sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file}
-
-# For nightlies we want to ensure that we're pulling in alphas as well. The
-# easiest way to do so is to augment the spec with a constraint containing a
-# min alpha version that doesn't affect the version bounds but does allow usage
-# of alpha versions for that dependency without --pre
-alpha_spec=''
-if ! rapids-is-release-build; then
-    alpha_spec=',>=0.0.0a0'
-fi
-
-sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file}
-sed -r -i "s/pylibraft(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file}
-
-
-if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
-    sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file}
-    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
-fi
+rapids-generate-version > VERSION
 
 cd "${package_dir}"
 

ci/release/update-version.sh

@@ -58,15 +58,15 @@ DEPENDENCIES=(
 )
 for FILE in dependencies.yaml conda/environments/*.yaml; do
   for DEP in "${DEPENDENCIES[@]}"; do
-    sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}\.*/g" ${FILE};
+    sed_runner "/-.* ${DEP}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE};
   done
-  sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_SHORT_TAG_PEP440}\.*/g" ${FILE};
+  sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_SHORT_TAG_PEP440}\.*,>=0.0.0a0/g" ${FILE};
 done
 for FILE in python/*/pyproject.toml; do
   for DEP in "${DEPENDENCIES[@]}"; do
-    sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*\"/g" ${FILE}
+    sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" ${FILE}
   done
-  sed_runner "/\"ucx-py==/ s/==.*\"/==${NEXT_UCX_PY_SHORT_TAG_PEP440}.*\"/g" ${FILE}
+  sed_runner "/\"ucx-py==/ s/==.*\"/==${NEXT_UCX_PY_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" ${FILE}
 done
 
 for FILE in .github/workflows/*.yaml; do

conda/environments/all_cuda-118_arch-aarch64.yaml

@@ -43,11 +43,12 @@ dependencies:
 - nvcc_linux-aarch64=11.8
 - pre-commit
 - pydata-sphinx-theme
-- pylibraft==24.8.*
+- pylibraft==24.8.*,>=0.0.0a0
 - pytest-cov
 - pytest==7.*
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
-- rmm==24.8.*
+- rmm==24.8.*,>=0.0.0a0
 - rust
 - scikit-build-core>=0.7.0
 - scikit-learn

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -43,11 +43,12 @@ dependencies:
 - nvcc_linux-64=11.8
 - pre-commit
 - pydata-sphinx-theme
-- pylibraft==24.8.*
+- pylibraft==24.8.*,>=0.0.0a0
 - pytest-cov
 - pytest==7.*
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
-- rmm==24.8.*
+- rmm==24.8.*,>=0.0.0a0
 - rust
 - scikit-build-core>=0.7.0
 - scikit-learn

conda/environments/all_cuda-122_arch-aarch64.yaml

@@ -39,11 +39,12 @@ dependencies:
 - numpydoc
 - pre-commit
 - pydata-sphinx-theme
-- pylibraft==24.8.*
+- pylibraft==24.8.*,>=0.0.0a0
 - pytest-cov
 - pytest==7.*
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
-- rmm==24.8.*
+- rmm==24.8.*,>=0.0.0a0
 - rust
 - scikit-build-core>=0.7.0
 - scikit-learn

conda/environments/all_cuda-122_arch-x86_64.yaml

@@ -39,11 +39,12 @@ dependencies:
 - numpydoc
 - pre-commit
 - pydata-sphinx-theme
-- pylibraft==24.8.*
+- pylibraft==24.8.*,>=0.0.0a0
 - pytest-cov
 - pytest==7.*
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
-- rmm==24.8.*
+- rmm==24.8.*,>=0.0.0a0
 - rust
 - scikit-build-core>=0.7.0
 - scikit-learn

conda/recipes/cuvs/meta.yaml

@@ -55,6 +55,7 @@ requirements:
     - libcuvs {{ version }}
     - python x.x
     - rmm ={{ minor_version }}
+    - rapids-build-backend>=0.3.0,<0.4.0.dev0
     - scikit-build-core >=0.7.0
     - setuptools
   run:

dependencies.yaml

@@ -7,6 +7,7 @@ files:
       arch: [x86_64, aarch64]
     includes:
       - build
+      - rapids_build
       - build_py_cuvs
       - cuda
       - cuda_version
@@ -46,13 +47,13 @@ files:
       - docs
       - py_version
       - rust
-      - build
+      - rapids_build
       - cuda
   rust:
     output: none
     includes:
       - cuda_version
-      - build
+      - rapids_build
       - cuda
       - rust
   py_build_py_cuvs:
@@ -62,6 +63,14 @@ files:
       table: build-system
     includes:
       - build
+  py_rapids_build_py_cuvs:
+    output: pyproject
+    pyproject_dir: python/cuvs
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
+    includes:
+      - rapids_build
       - build_py_cuvs
   py_run_py_cuvs:
     output: pyproject
@@ -88,6 +97,17 @@ channels:
   - nvidia
 dependencies:
   build:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - rapids-build-backend>=0.3.0,<0.4.0.dev0
+      - output_types: [conda]
+        packages:
+          - scikit-build-core>=0.7.0
+      - output_types: [requirements, pyproject]
+        packages:
+          - scikit-build-core[pyproject]>=0.7.0
+  rapids_build:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
@@ -99,10 +119,6 @@ dependencies:
           - c-compiler
           - cxx-compiler
           - nccl>=2.9.9
-          - scikit-build-core>=0.7.0
-      - output_types: [requirements, pyproject]
-        packages:
-          - scikit-build-core[pyproject]>=0.7.0
     specific:
       - output_types: conda
         matrices:
@@ -141,8 +157,8 @@ dependencies:
     common:
       - output_types: [conda]
         packages:
-          - &rmm_conda rmm==24.8.*
-          - &pylibraft_conda pylibraft==24.8.*
+          - &rmm_conda rmm==24.8.*,>=0.0.0a0
+          - &pylibraft_conda pylibraft==24.8.*,>=0.0.0a0
           - dlpack>=0.8,<1.0
       - output_types: requirements
         packages:
@@ -157,19 +173,23 @@ dependencies:
               cuda: "12.*"
             packages:
               - &cuda_python12 cuda-python>=12.0,<13.0a0
-          - matrix: # All CUDA 11 versions
+          - matrix:
+              cuda: "11.*"
             packages:
               - &cuda_python11 cuda-python>=11.7.1,<12.0a0
+          - matrix:
+            packages:
+              - &cuda_python cuda-python
       - output_types: [requirements, pyproject]
         matrices:
           - matrix: {cuda: "12.*"}
             packages:
-              - &rmm_cu12 rmm-cu12==24.8.*
-              - &pylibraft_cu12 pylibraft-cu12==24.8.*
+              - &rmm_cu12 rmm-cu12==24.8.*,>=0.0.0a0
+              - &pylibraft_cu12 pylibraft-cu12==24.8.*,>=0.0.0a0
           - matrix: {cuda: "11.*"}
             packages:
-              - &rmm_cu11 rmm-cu11==24.8.*
-              - &pylibraft_cu11 pylibraft-cu11==24.8.*
+              - &rmm_cu11 rmm-cu11==24.8.*,>=0.0.0a0
+              - &pylibraft_cu11 pylibraft-cu11==24.8.*,>=0.0.0a0
           - {matrix: null, packages: [*rmm_conda, *pylibraft_conda] }
   checks:
     common:
@@ -373,9 +393,13 @@ dependencies:
               cuda: "12.*"
             packages:
               - *cuda_python12
-          - matrix: # All CUDA 11 versions
+          - matrix:
+              cuda: "11.*"
             packages:
               - *cuda_python11
+          - matrix:
+            packages:
+              - *cuda_python
       - output_types: [requirements, pyproject]
         matrices:
           - matrix: {cuda: "12.*"}

python/cuvs/cuvs/_version.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,10 +13,22 @@
 # limitations under the License.
 #
 
-
 import importlib.resources
 
 __version__ = (
-    importlib.resources.files("cuvs").joinpath("VERSION").read_text().strip()
+    importlib.resources.files(__package__)
+    .joinpath("VERSION")
+    .read_text()
+    .strip()
 )
-__git_commit__ = ""
+try:
+    __git_commit__ = (
+        importlib.resources.files(__package__)
+        .joinpath("GIT_COMMIT")
+        .read_text()
+        .strip()
+    )
+except FileNotFoundError:
+    __git_commit__ = ""
+
+__all__ = ["__git_commit__", "__version__"]

python/cuvs/cuvs/test/test_version.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+import cuvs
+
+
+def test_version_constants_are_populated():
+    # __git_commit__ will only be non-empty in a built distribution
+    assert isinstance(cuvs.__git_commit__, str)
+
+    # __version__ should always be non-empty
+    assert isinstance(cuvs.__version__, str)
+    assert len(cuvs.__version__) > 0

python/cuvs/pyproject.toml

@@ -15,15 +15,10 @@
 [build-system]
 
 requires = [
-    "cmake>=3.26.4",
-    "cuda-python>=11.7.1,<12.0a0",
-    "cython>=3.0.0",
-    "ninja",
-    "pylibraft==24.8.*",
-    "rmm==24.8.*",
+    "rapids-build-backend>=0.3.0,<0.4.0.dev0",
     "scikit-build-core[pyproject]>=0.7.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
-build-backend = "scikit_build_core.build"
+build-backend = "rapids_build_backend.build"
 
 [project]
 name = "cuvs"
@@ -36,9 +31,9 @@ authors = [
 license = { text = "Apache 2.0" }
 requires-python = ">=3.9"
 dependencies = [
-    "cuda-python>=11.7.1,<12.0a0",
+    "cuda-python",
     "numpy>=1.23,<2.0a0",
-    "pylibraft==24.8.*",
+    "pylibraft==24.8.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 classifiers = [
     "Intended Audience :: Developers",
@@ -121,6 +116,18 @@ provider = "scikit_build_core.metadata.regex"
 input = "cuvs/VERSION"
 regex = "(?P<value>.*)"
 
+[tool.rapids-build-backend]
+requires = [
+    "cmake>=3.26.4",
+    "cuda-python",
+    "cython>=3.0.0",
+    "ninja",
+    "pylibraft==24.8.*,>=0.0.0a0",
+    "rmm==24.8.*,>=0.0.0a0",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+build-backend = "scikit_build_core.build"
+dependencies-file = "../../dependencies.yaml"
+
 [tool.pytest.ini_options]
 filterwarnings = [
     "error",