From eae05881d8ff5c5c9476e11bcd0bc531760e8efa Mon Sep 17 00:00:00 2001 From: Ray Douglass <3107146+raydouglass@users.noreply.github.com> Date: Wed, 26 Jul 2023 10:14:49 -0400 Subject: [PATCH] Remove the env packages (#666) Closes #119 Removes: - `rapids-build-env` - `rapids-doc-env` - `rapids-notebook-env` Also updates the `version.yaml` files to remove unnecessary packages. Authors: - Ray Douglass (https://github.com/raydouglass) Approvers: - Bradley Dice (https://github.com/bdice) - https://github.com/jakirkham - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/integration/pull/666 --- README.md | 9 +- ci/build_python.sh | 24 --- conda/recipes/README.md | 17 +- conda/recipes/rapids-build-env/meta.yaml | 164 -------------------- conda/recipes/rapids-doc-env/meta.yaml | 56 ------- conda/recipes/rapids-notebook-env/meta.yaml | 72 --------- conda/recipes/versions.yaml | 158 ------------------- 7 files changed, 6 insertions(+), 494 deletions(-) delete mode 100644 conda/recipes/rapids-build-env/meta.yaml delete mode 100644 conda/recipes/rapids-doc-env/meta.yaml delete mode 100644 conda/recipes/rapids-notebook-env/meta.yaml diff --git a/README.md b/README.md index 5972e16b..bd41fc2f 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,9 @@ #
  Integration -RAPIDS - combined conda package & integration tests for all of RAPIDS libraries +RAPIDS - combined conda package for all of RAPIDS libraries ## RAPIDS Meta-packages -The conda recipe in the `conda` folder provides the RAPIDS meta-packages, which -when installed will provide the latest RAPIDS libraries for the given version or -setup `build`, `doc`, or `notebook` environments for RAPIDS. +The conda recipe in the `conda` folder provides the RAPIDS meta-packages, which when installed will provide the latest RAPIDS libraries for the given version. -See the [README](conda/recipes/README.md) for more information about the -meta-packages and how to update versions. +See the [README](conda/recipes/README.md) for more information about the meta-packages and how to update versions. diff --git a/ci/build_python.sh b/ci/build_python.sh index dc7bc468..6512095c 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -23,28 +23,4 @@ rapids-mamba-retry mambabuild \ --variant-config-files "${CONDA_CONFIG_FILE}" \ conda/recipes/rapids -rapids-logger "Build rapids-build-env" - -rapids-mamba-retry mambabuild \ - --no-test \ - --variant-config-files "${CONDA_CONFIG_FILE}" \ - conda/recipes/rapids-build-env - -rapids-logger "Build rapids-notebook-env" - -rapids-mamba-retry mambabuild \ - --no-test \ - --variant-config-files "${CONDA_CONFIG_FILE}" \ - conda/recipes/rapids-notebook-env - -if [ "$(uname -m)" != "aarch64" ]; then - - rapids-logger "Build rapids-doc-env" - - rapids-mamba-retry mambabuild \ - --no-test \ - --variant-config-files "${CONDA_CONFIG_FILE}" \ - conda/recipes/rapids-doc-env -fi - rapids-upload-conda-to-s3 python diff --git a/conda/recipes/README.md b/conda/recipes/README.md index e2d8cca6..e6c3778a 100644 --- a/conda/recipes/README.md +++ b/conda/recipes/README.md @@ -26,17 +26,6 @@ Package Name | Purpose `rapids` | Provide a one package install for all RAPIDS libraries, version matched to a RAPIDS release `rapids-xgboost` | Defines the version of `xgboost` used for a RAPIDS release -### Environment Packages - -The environment meta-packages are used for setting up `build`, `doc`, and -`notebook` environments for users and in our RAPIDS [containers](https://github.com/rapidsai/build). - -Package Name | Purpose ---- | --- -`rapids-build-env` | Installs all `conda` build dependencies to build & test RAPIDS libraries from source -`rapids-doc-env` | Installs all tools needed to build RAPIDS documentation -`rapids-notebook-env` | Installs a Jupyter Notebook server and other dependencies to run RAPIDS example notebooks; used in the `runtime` [stable](https://hub.docker.com/r/rapidsai/rapidsai/tags?page=1&name=runtime) and [nightly](https://hub.docker.com/r/rapidsai/rapidsai-nightly/tags?page=1&name=runtime) RAPIDS containers. - ## Managing Versions Packages without version restrictions do not need to use the following process @@ -64,7 +53,7 @@ the `VERSIONING_NAME` added to the file. - `PACKAGE_NAME` - is the conda package name - `VERSIONING_NAME` - is the conda package name with `-` replaced with `_` and a suffix of `_version` added - - For example + - For example - `cupy` would become `cupy_version` - `scikit-learn` would become `scikit_learn_version` @@ -89,7 +78,7 @@ scikit-learn {{ scikit_learn_version }} #### Modifying Versions File -In `conda/recipes` is `versions.yaml` - These are versions used by the `ci/axis/build.yaml` for testing in PRs and conda builds. +In `conda/recipes` is `versions.yaml` - These are versions used by CI for testing in PRs and conda builds. In this file we specify the version for the newly created `VERSIONING_NAME`. @@ -105,7 +94,7 @@ standard `conda` version specifiers: ##### TIP - Correct version specs **NOTE:** `=2.5.*` is not a valid version spec. Please use `=2.5` instead -which will be interperted as `=2.5.*`. Otherwise `conda build` throws a +which will be interpreted as `=2.5.*`. Otherwise `conda build` throws a warning message with the definition of `.*`. For example: ``` diff --git a/conda/recipes/rapids-build-env/meta.yaml b/conda/recipes/rapids-build-env/meta.yaml deleted file mode 100644 index d1f8dd6f..00000000 --- a/conda/recipes/rapids-build-env/meta.yaml +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. - -{% set rapids_version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set py_version = environ['CONDA_PY'] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -### -# Versions referenced below are set in `conda/recipe/*versions.yaml` except for -# those set above (e.g. `cuda_version`) -### - -package: - name: rapids-build-env - version: {{ rapids_version }} - -source: - git_url: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - -requirements: - host: - - python - run: - - asvdb - - autoconf - - automake - - benchmark {{ benchmark_version }} - - black {{ black_version }} - - conda-forge::blas - - boost - - boost-cpp {{ boost_cpp_version }} - - boto3 - - cachetools - - conda-forge::clang {{ clang_version }} # [x86_64] - - conda-forge::clang-tools {{ clang_version }} # [x86_64] - - cmake {{ cmake_version }} - - cmakelang {{ cmakelang_version }} - - cmake_setuptools {{ cmake_setuptools_version }} - - conda {{ conda_version }} - - conda-build {{ conda_build_version }} - - conda-verify {{ conda_verify_version }} - - cubinlinker # CUDA enhanced compat. - - cuda-python {{ cuda11_cuda_python_version }} - - cudatoolkit ={{ cuda_major }}.* - - cupy {{ cupy_version }} - - c-compiler - - cxx-compiler - - gcc_impl_{{ target_platform }} {{ gcc_version }} - - cython {{ cython_version }} - - dask {{ dask_version }} - - dask-core {{ dask_version }} - - dask-ml - - datashader {{ datashader_version }} - - distributed {{ distributed_version }} - - dlpack {{ dlpack_version }} - - double-conversion {{ double_conversion_version }} - - faiss-proc=*=cuda - - libfaiss {{ faiss_version }} - - fastavro {{ fastavro_version }} - - feather-format - - flake8 {{ flake8_version }} - - fmt {{ fmt_version }} - - fsspec {{ fsspec_version }} - - gcsfs {{ gcsfs_version }} - - geopandas {{ geopandas_version }} - - git - - gmock {{ gmock_version }} - - graphviz - - gtest {{ gtest_version }} - - h5py - - hdbscan - - holoviews {{ holoviews_version }} - - httpretty - - hypothesis - - isort {{ isort_version }} - - lapack - - libarrow {{ arrow_version }} - - libcypher-parser - - liblapack - - librdkafka {{ librdkafka_version }} - - libtool - - libwebp - - lightgbm - - make - - mimesis {{ mimesis_version }} - - mock - - moto {{ moto_version }} - - mypy {{ mypy_version }} - - nccl {{ nccl_version }} - - networkx {{ networkx_version }} - - ninja - - nltk - - numba {{ numba_version }} - - numpy {{ numpy_version }} - - nvtx {{ nvtx_version }} - - openslide - - packaging - - pandas {{ pandas_version }} - - panel {{ panel_version }} - - pillow {{ pillow_version }} - - pip - - pkg-config - - pre-commit - - protobuf {{ protobuf_version }} - - psutil - - ptxcompiler # CUDA enhanced compat. See https://github.com/rapidsai/ptxcompiler - - pyarrow {{ arrow_version }} - - pydeck {{ pydeck_version }} - - pydocstyle {{ pydocstyle_version }} - - pynvml - - pyorc - - pyppeteer {{ pyppeteer_version }} - - pyproj {{ pyproj_version }} - - pytest - - pytest-asyncio {{ pytest_asyncio_version }} - - pytest-benchmark - - pytest-cases - - pytest-cov - - pytest-timeout - - pytest-xdist - - python - - python-confluent-kafka {{ python_confluent_kafka_version }} - - python-louvain - - python-snappy - - rapidjson {{ rapidjson_version }} - - ripgrep - - s3fs {{ s3fs_version }} - - setuptools {{ setuptools_version }} - - scikit-build {{ scikit_build_version }} - - scikit-image {{ scikit_image_version }} - - scikit-learn {{ scikit_learn_version }} - - scipy {{ scipy_version }} - - shellcheck - - spdlog {{ spdlog_version }} - - statsmodels - - streamz - - sysroot_{{ target_platform }} {{ sysroot_version }} - - thriftpy2 - - tomli # [py<311] - - transformers {{ transformers_version }} - - treelite {{ treelite_version }} - - twine - - types-cachetools - - typing_extensions - - conda-forge::ucx-proc=*=gpu - - conda-forge::ucx {{ ucx_version }} - - umap-learn - - versioneer {{ versioneer_version }} - - werkzeug {{ werkzeug_version }} # Temporary transient dependency pinning to avoid URL-LIB3 + moto timeouts - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_file: LICENSE - summary: 'RAPIDS Build Dependencies and Tool Environment Installer' - description: | - Meta-package for installing build dependencies and tools needed to build all RAPIDS libraries. - doc_url: https://docs.rapids.ai/ - dev_url: https://github.com/rapidsai/ diff --git a/conda/recipes/rapids-doc-env/meta.yaml b/conda/recipes/rapids-doc-env/meta.yaml deleted file mode 100644 index c1c51ad2..00000000 --- a/conda/recipes/rapids-doc-env/meta.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. - -{% set rapids_version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} -{% set py_version = environ['CONDA_PY'] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -### -# Versions referenced below are set in `conda/recipe/*versions.yaml` except for -# those set above (e.g. `cuda_version`) -### - -package: - name: rapids-doc-env - version: {{ rapids_version }} - -source: - git_url: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - -requirements: - host: - - python - - pip - run: - - beautifulsoup4 - - breathe - - doxygen {{ doxygen_version }} - - html5lib - - jupyter_sphinx - - markdown {{ markdown_version }} - - nbsphinx {{ nbsphinx_version }} - - numpydoc - - pandoc - - pydata-sphinx-theme {{ pydata_sphinx_theme_version }} - - recommonmark - - sphinx - - sphinx_rtd_theme - - sphinxcontrib-websupport - - sphinx-click - - sphinx-copybutton - - sphinx-markdown-tables - - sphinx-autobuild - - myst-nb - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_file: LICENSE - summary: 'RAPIDS Documentation Dependencies and Tool Environment Installer' - description: | - Meta-package for installing documentation build dependencies and tools needed to build RAPIDS docs. - doc_url: https://docs.rapids.ai/ - dev_url: https://github.com/rapidsai/ diff --git a/conda/recipes/rapids-notebook-env/meta.yaml b/conda/recipes/rapids-notebook-env/meta.yaml deleted file mode 100644 index 9c73ef0b..00000000 --- a/conda/recipes/rapids-notebook-env/meta.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. - -{% set rapids_version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set py_version = environ['CONDA_PY'] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -### -# Versions referenced below are set in `conda/recipe/*versions.yaml` except for -# those set above (e.g. `cuda_version`) -### - -package: - name: rapids-notebook-env - version: {{ rapids_version }} - -source: - git_url: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - -requirements: - host: - - python - run: - - bokeh {{ bokeh_version }} - - colorcet - - conda-forge::blas - - cudatoolkit ={{ cuda_major }}.* - - cupy {{ cupy_version }} - - cython {{ cython_version }} - - dask {{ dask_version }} - - dask-core {{ dask_version }} - - dask-labextension - - dask-ml - - datashader {{ datashader_version }} - - distributed {{ distributed_version }} - - filterpy - - holoviews - - ipython {{ ipython_version }} - - ipykernel {{ ipykernel_version }} - - ipywidgets - - jupyter-server-proxy - - jupyterlab {{ jupyterlab_version }} - - jupyterlab-favorites - - jupyter-packaging {{ jupyter_packaging_version }} - - jupyterlab_widgets - - matplotlib-base - - networkx {{ networkx_version }} - - nodejs {{ nodejs_version }} - - openslide - - pytest - - pillow {{ pillow_version }} - - s3fs {{ s3fs_version }} - - scikit-image {{ scikit_image_version }} - - scikit-learn {{ scikit_learn_version }} - - scipy {{ scipy_version }} - - seaborn - - umap-learn - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_file: LICENSE - summary: 'RAPIDS Notebook Dependencies and Tool Environment Installer' - description: | - Meta-package for installing notebook dependencies and tools needed to run RAPIDS notebooks. - doc_url: https://docs.rapids.ai/ - dev_url: https://github.com/rapidsai/ diff --git a/conda/recipes/versions.yaml b/conda/recipes/versions.yaml index 248354cc..c54b1edc 100644 --- a/conda/recipes/versions.yaml +++ b/conda/recipes/versions.yaml @@ -1,181 +1,23 @@ -### -# This file is used by `/ci/cpu/build*.sh` for NIGHTLY builds -### - -# Versions for `rapids` meta-pkg -dask_xgboost_version: - - '=0.2.0.dev28' - # Versions for `rapids-xgboost` meta-pkg xgboost_version: # Minor version is appended in meta.yaml - '=1.7.5dev.rapidsai' -# Versions for conda -conda_version: - - '=4.12.0' -conda_build_version: - - '=3.22.0' -conda_verify_version: - - '=3.1.1' - -# conda compilers -gcc_version: - - 11 -sysroot_version: - - "2.17" - -# Shared versions across meta-pkgs -arrow_version: - - '=11' -benchmark_version: - - '=1.5.1' -black_version: - - '=22.3.0' -bokeh_version: - - '>=2.4.2,<=2.5' -boost_cpp_version: - - '=1.74.0' -clang_version: - - '=11.1.0' -cmake_version: - - '>=3.23.1,!=3.25.0' -cmakelang_version: - - '=0.6.13' -cmake_setuptools_version: - - '>=0.1.3' cuda11_cuda_python_version: - '>=11.7.1,<12.0a' cuda12_cuda_python_version: - '>=12.0.0,<13.0a' cupy_version: - '>=12.0.0' -cython_version: - - '>=0.29.17,<0.30' -dask_version: - - '>=2023.5.1' -datashader_version: - - '>=0.15' -distributed_version: - - '>=2023.5.1' -dlpack_version: - - '>=0.5,<0.6.0a0' -double_conversion_version: - - '=3.1.5' -doxygen_version: - - '>=1.8.12,<=1.8.20' -faiss_version: - - '=1.7.2' -fastavro_version: - - '>=0.22.0' -flake8_version: - - '=3.8' -fmt_version: - - '>=9.1.0,<10' -fsspec_version: - - '>=0.9.0' -geopandas_version: - - '>=0.11.0' -gcsfs_version: - - '>=0.8.0' -google_cloud_cpp_version: - - '>=1.25.0,<1.30' -gmock_version: - - '>=1.13.0' -gtest_version: - - '>=1.13.0' -holoviews_version: - - '>=1.14.8,<=1.15.3' -ipython_version: - - '=7.31.1' -ipykernel_version: - - '!=6.21.0,!=6.21.1' -isort_version: - - '=5.12.0' -jupyterlab_version: - - '>=3.1.4,<4.0a0' -jupyter_packaging_version: - - '>=0.7.0,<0.8' -librdkafka_version: - - '>=1.9.0,<1.10.0a0' -markdown_version: - - '>=3.4.1' -mimesis_version: - - '>=4.1.0' -moto_version: - - '>=3.1.6' -mypy_version: - - '0.971' -mysql_connector_cpp_version: - - '8.0.23' -nbsphinx_version: - - '>=0.8.6' nccl_version: - '>=2.9.9,<3.0a0' networkx_version: - '>=2.5.1' -nlohmann_json_version: - - '3.9.1' -nodejs_version: - - '>=14' numba_version: - '>=0.57' numpy_version: - '>=1.21' nvtx_version: - '>=0.2.1,<0.3' -openjdk_version: - - '>=8.0' -pandas_version: - - '>=1.3,<1.6.0dev0' -panel_version: - - '>=0.14.0,<=0.14.1' -pillow_version: - - '>=9.0.0' -pydeck_version: - - '>=0.3, <=0.5.0' -pydocstyle_version: - - '>=6.0.0, <=7.0.0' -python_confluent_kafka_version: - - '>=1.9.0,<1.10.0a0' -pytorch_version: - - '>=1.6,<1.12.0' -protobuf_version: - - '>=4.21.6,<4.22' -pydata_sphinx_theme_version: - - '>=0.6.3' -pyproj_version: - - '>=2.4,<=3.4' -pyppeteer_version: - - '>=0.2.6' -pytest_asyncio_version: - - '=0.20.*' -rapidjson_version: - - '=1.1.0' -s3fs_version: - - '>=2022.3.0' -scikit_build_version: - - '=0.13.1' -scikit_image_version: - - '>=0.19.0,<1.0a0' -scikit_learn_version: - - '=1.2' -# when scipy is updated, remove upper bound on networkx ver. -scipy_version: - - '>=1.6.0' -setuptools_version: - - '>65' -spdlog_version: - - '>=1.11.0,<1.12' -sphinx_markdown_tables_version: - - '=0.0.14=pyh9f0ad1d_1' -treelite_version: - - '=3.2.0' -transformers_version: - - '<=4.10.3' ucx_version: - '>=1.12.1' -versioneer_version: - - '>=0.24' -werkzeug_version: - - '<2.2.0'