From 13a5f474c02399a975b9f315ab6010d933bdf077 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 24 Jul 2024 10:03:56 -0500 Subject: [PATCH 1/2] split up CUDA-suffixed dependencies in dependencies.yaml (#1364) Contributes to https://github.com/rapidsai/build-planning/issues/31 In short, RAPIDS DLFW builds want to produce wheels with unsuffixed dependencies, e.g. `cudf` depending on `rmm`, not `rmm-cu12`. This PR is part of a series across all of RAPIDS to try to support that type of build by setting up CUDA-suffixed and CUDA-unsuffixed dependency lists in `dependencies.yaml`. For more details, see: * https://github.com/rapidsai/build-planning/issues/31#issuecomment-2245815818 * https://github.com/rapidsai/cudf/pull/16183 ## Notes for Reviewers ### Why target 24.08? This is targeting 24.08 because: 1. it should be very low-risk 2. getting these changes into 24.08 prevents the need to carry around patches for every library in DLFW builds using RAPIDS 24.08 Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/dask-cuda/pull/1364 --- dependencies.yaml | 22 ++++++++++++++++++++-- pyproject.toml | 1 + 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 910edc08..f547df6b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -190,16 +190,34 @@ dependencies: matrices: # kvikio should be added to the CUDA-version-specific matrices once there are wheels available # ref: https://github.com/rapidsai/kvikio/pull/369 - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - cudf-cu12==24.8.*,>=0.0.0a0 - dask-cudf-cu12==24.8.*,>=0.0.0a0 - ucx-py-cu12==0.39.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "false" + packages: + - *cudf_conda + - *dask_cudf_conda + - *ucx_py_conda + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - cudf-cu11==24.8.*,>=0.0.0a0 - dask-cudf-cu11==24.8.*,>=0.0.0a0 - ucx-py-cu11==0.39.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "false" + packages: + - *cudf_conda + - *dask_cudf_conda + - *ucx_py_conda - matrix: packages: - *cudf_conda diff --git a/pyproject.toml b/pyproject.toml index 126efba6..b6c431d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,6 +134,7 @@ filterwarnings = [ build-backend = "setuptools.build_meta" dependencies-file = "dependencies.yaml" disable-cuda = true +matrix-entry = "cuda_suffixed=true" [tool.setuptools] license-files = ["LICENSE"] From 064e2544e3545a4f7478b41a1c7459212a7768ce Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 24 Jul 2024 18:31:32 -0500 Subject: [PATCH 2/2] consolidate cuda_suffixed=false blocks in dependencies.yaml, fix update-version.sh (#1367) Contributes to https://github.com/rapidsai/build-planning/issues/31. Follow-up to #1364. Implements some of the suggestions made in https://github.com/rapidsai/cudf/pull/16183 (after #1364 was already merged): * removing `cuda_suffixed: "false"` blocks in `dependencies.yaml` wherever they're identical to each other and the fallback matrix * changing `dependencies.yaml` anchors with names like `*_conda` to `*_unsuffixed`, to reflect the fact that they're not conda-specific * checking that `update-version.sh` catches all changes to versions ## Notes for Reviewers ### How I tested this Looked for `update-versions.sh` issues manually like this: ```shell git fetch upstream --tags ci/release/update-version.sh '24.10.0' git grep -E '24\.8|24\.08|0\.39' ``` The did find a few problems (like UCX dependency versions not being updated). This fixes those issues. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/dask-cuda/pull/1367 --- ci/release/update-version.sh | 17 +++++++++++++++-- dependencies.yaml | 30 ++++++++---------------------- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index ac834e5e..a9fe1d02 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -45,10 +45,23 @@ DEPENDENCIES=( kvikio rapids-dask-dependency ) -for FILE in dependencies.yaml conda/environments/*.yaml; do - for DEP in "${DEPENDENCIES[@]}"; do +for DEP in "${DEPENDENCIES[@]}"; do + for FILE in dependencies.yaml conda/environments/*.yaml; do sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}" done + sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" pyproject.toml +done + +UCX_DEPENDENCIES=( + distributed-ucxx + ucx-py + ucxx +) +for DEP in "${UCX_DEPENDENCIES[@]}"; do + for FILE in dependencies.yaml conda/environments/*.yaml; do + sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_UCXPY_VERSION}.*,>=0.0.0a0/g" "${FILE}" + done + sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_UCXPY_VERSION}.*,>=0.0.0a0\"/g" pyproject.toml done # CI files diff --git a/dependencies.yaml b/dependencies.yaml index f547df6b..c3b62965 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -168,11 +168,11 @@ dependencies: - pytest-cov - output_types: [conda] packages: - - &cudf_conda cudf==24.8.*,>=0.0.0a0 - - &dask_cudf_conda dask-cudf==24.8.*,>=0.0.0a0 + - &cudf_unsuffixed cudf==24.8.*,>=0.0.0a0 + - &dask_cudf_unsuffixed dask-cudf==24.8.*,>=0.0.0a0 - distributed-ucxx==0.39.*,>=0.0.0a0 - - &kvikio_conda kvikio==24.8.*,>=0.0.0a0 - - &ucx_py_conda ucx-py==0.39.*,>=0.0.0a0 + - &kvikio_unsuffixed kvikio==24.8.*,>=0.0.0a0 + - &ucx_py_unsuffixed ucx-py==0.39.*,>=0.0.0a0 - ucx-proc=*=gpu - ucxx==0.39.*,>=0.0.0a0 specific: @@ -197,13 +197,6 @@ dependencies: - cudf-cu12==24.8.*,>=0.0.0a0 - dask-cudf-cu12==24.8.*,>=0.0.0a0 - ucx-py-cu12==0.39.*,>=0.0.0a0 - - matrix: - cuda: "12.*" - cuda_suffixed: "false" - packages: - - *cudf_conda - - *dask_cudf_conda - - *ucx_py_conda - matrix: cuda: "11.*" cuda_suffixed: "true" @@ -211,16 +204,9 @@ dependencies: - cudf-cu11==24.8.*,>=0.0.0a0 - dask-cudf-cu11==24.8.*,>=0.0.0a0 - ucx-py-cu11==0.39.*,>=0.0.0a0 - - matrix: - cuda: "11.*" - cuda_suffixed: "false" - packages: - - *cudf_conda - - *dask_cudf_conda - - *ucx_py_conda - matrix: packages: - - *cudf_conda - - *dask_cudf_conda - - *kvikio_conda - - *ucx_py_conda + - *cudf_unsuffixed + - *dask_cudf_unsuffixed + - *kvikio_unsuffixed + - *ucx_py_unsuffixed