From 7d56b5e93df7330020ae4956c402be12955cc52b Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Mon, 6 Nov 2023 15:02:56 -0800 Subject: [PATCH] build wheels on CI (#619) This MR includes the commits from #617, so that one should be merged first. The purpose of this MR is to build and test wheels on RAPIDS CI infrastructure. Previously, cuCIM was using a custom manual wheel building process based on running: ``` ./run build_python_package ``` which used `dockcross`/`cibuildwheel`. We would then manually upload the resulting wheel to PyPI using `twine`. The changes in this MR are to enable automated wheel builds for future RAPIDS releases in a similar way to other RAPIDS projects. ### Misc issues/questions for discussion - The `build_wheels.sh` script installs the YASM and the openslide libraries via system package managers. This is currently hard-coded in that script rather than using the `dependencies.yaml` mechanism being used elsewhere. Is this reasonable or is there a better way? - Following cuML's example, I added a separate `ci/release/apply_wheel_modifications.sh` script that updates the pyproject.toml as needed (to fixup version string and the CUDA version strings in the cucim package name and CuPy dependency) - I updated the cuCIM test suite to make a handful of tests that relied on `imagecodecs` to be skipped if it is not installed. This is because I did not find a pre-compiled wheel for it on aarch64 and when it tried to compile from source instead, it failed due to various missing system packages needed for the compilation. - Unlike most RAPIDS projects, which seem to use scikit-build, this project performs a separate CMake build via `run build_local` commands and copies the C++ libraries into the Python source tree. The `pyproject.toml` then just does a standard pure-python build, but will package the previously compiled libraries along with the other files. - The wheel tests are currently taking approximately 35-50 minutes to run. It should be possible to reduce that time by an order of magnitude if the CuPy kernels are cached across runs. Does any RAPIDS projects cache the compiled CuPy kernels in this way? (cached kernels end up in `~/.cupy/kernel_cache/` by default, although an alternative location can be specified via `CUPY_CACHE_DIR`) Authors: - Gregory Lee (https://github.com/grlee77) - https://github.com/jakirkham Approvers: - Bradley Dice (https://github.com/bdice) - Divye Gala (https://github.com/divyegala) - Ray Douglass (https://github.com/raydouglass) - https://github.com/jakirkham URL: https://github.com/rapidsai/cucim/pull/619 --- .github/workflows/build.yaml | 19 +++++++ .github/workflows/pr.yaml | 16 ++++++ .github/workflows/test.yaml | 9 ++++ .pre-commit-config.yaml | 1 - CMakeLists.txt | 2 + CONTRIBUTING.md | 16 +++--- VERSION | 2 +- ci/build_cpp.sh | 4 +- ci/build_python.sh | 13 ++++- ci/build_wheel.sh | 51 +++++++++++++++++++ ci/release/update-version.sh | 9 ++-- ci/test_wheel.sh | 18 +++++++ ci/wheel_smoke_test.py | 21 ++++++++ .../all_cuda-118_arch-x86_64.yaml | 2 + .../all_cuda-120_arch-x86_64.yaml | 2 + conda/recipes/cucim/meta.yaml | 4 +- conda/recipes/libcucim/meta.yaml | 4 +- cpp/cmake/deps/openslide.cmake | 4 +- cpp/plugins/cucim.kit.cumed/CMakeLists.txt | 4 +- cpp/plugins/cucim.kit.cumed/VERSION | 1 - cpp/plugins/cucim.kit.cuslide/CMakeLists.txt | 4 +- cpp/plugins/cucim.kit.cuslide/VERSION | 1 - .../cmake/deps/openslide.cmake | 4 +- .../src/cuslide/jpeg2k/gen_color_table.py | 14 +++-- dependencies.yaml | 23 ++++----- python/CMakeLists.txt | 2 + python/cucim/VERSION | 1 - python/cucim/pyproject.toml | 18 +++---- python/cucim/src/cucim/VERSION | 1 + python/cucim/src/cucim/__init__.py | 13 ++--- python/cucim/src/cucim/__init__.pyi | 12 +++-- python/cucim/src/cucim/_version.py | 20 ++++++++ python/cucim/src/cucim/clara/__init__.py | 11 +--- .../skimage/measure/tests/test_blur_effect.py | 8 ++- .../test_masked_phase_cross_correlation.py | 6 ++- .../skimage/transform/tests/test_warps.py | 3 -- .../clara/test_read_region_memory_usage.py | 3 ++ .../unit/clara/converter/test_converter.py | 6 +++ .../tests/unit/clara/test_image_cache.py | 5 ++ .../unit/clara/test_load_image_metadata.py | 5 ++ .../tests/unit/clara/test_tiff_read_region.py | 3 ++ 41 files changed, 280 insertions(+), 85 deletions(-) create mode 100755 ci/build_wheel.sh create mode 100755 ci/test_wheel.sh create mode 100644 ci/wheel_smoke_test.py delete mode 100644 cpp/plugins/cucim.kit.cumed/VERSION delete mode 100644 cpp/plugins/cucim.kit.cuslide/VERSION delete mode 100644 python/cucim/VERSION create mode 120000 python/cucim/src/cucim/VERSION create mode 100644 python/cucim/src/cucim/_version.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9e00fdce7..1a028553d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,3 +66,22 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + wheel-build: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel.sh + wheel-publish: + needs: wheel-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-23.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: cucim diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c1c7b60ff..d098ff326 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,6 +17,8 @@ jobs: - conda-python-build - conda-python-tests - docs-build + - wheel-build + - wheel-tests secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-23.12 checks: @@ -50,3 +52,17 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" + wheel-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 + with: + build_type: pull-request + script: ci/build_wheel.sh + wheel-tests: + needs: wheel-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 + with: + build_type: pull-request + script: ci/test_wheel.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9a5e0428a..8ac82418e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,3 +22,12 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + wheel-tests: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/test_wheel.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0290c0e8f..e12e83ae3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,6 @@ repos: rev: 23.10.1 hooks: - id: black - files: (python|legate)/.* args: ["--config", "python/cucim/pyproject.toml"] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.1.3 diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ab2e178c..ac1aff40e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,8 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION and BUILD unset(VERSION CACHE) file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION) +# strip alpha version info +string(REGEX REPLACE "a.*$" "" VERSION ${VERSION}) set(PROJECT_VERSION_BUILD dev) # Append local cmake module path diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 16c29fcb6..a05dee929 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -263,19 +263,21 @@ conda install -y -c ${CONDA_BLD_DIR} -c conda-forge \ ## Building a package (for distribution. Including a wheel package for pip) -**Build** - -You can execute the following command to build a wheel file for pip. +**Wheel Build** +If you are using CUDA 12.x, please update pyproject.toml as follows before building the wheel ```bash -./run build_package +sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/cucim/pyproject.toml ``` +This will switch the CuPy dependency to one based on CUDA 12.x instead of 11.x. -The command would use `./temp` folder as a local build folder and build a distribution package into `dist` folder using [dockcross](https://github.com/dockcross/dockcross)'s manylinux2014 docker image. +The wheel can then be built using: -`./run build_package` will reuse local `./temp` folder to reduce the build time. +```bash +python -m pip wheel python/cucim/ -w dist -vvv --no-deps --disable-pip-version-check +``` -If C++ code or dependent packages are updated so the build is failing somehow, please retry it after deleting the `temp` folder under the repository root. +**Note:** It is possible to build the wheel in this way even without compiling the C++ library first, but in that case the `cucim.clara` module will not be importable. **Install** diff --git a/VERSION b/VERSION index a193fff41..bb94af9dd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -23.12.00 +23.12.00a56 diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 1cc80d198..b0ef4a446 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -9,8 +9,10 @@ export CMAKE_GENERATOR=Ninja rapids-print-env +version=$(rapids-generate-version) + rapids-logger "Begin cpp build" -rapids-conda-retry mambabuild conda/recipes/libcucim +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libcucim rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index ae607aeeb..58d8a8e00 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -9,13 +9,24 @@ export CMAKE_GENERATOR=Ninja rapids-print-env +package_name="cucim" +package_dir="python/cucim" +package_src_dir="${package_dir}/src/${package_name}" + +version=$(rapids-generate-version) + +commit=$(git rev-parse HEAD) + +echo "${version}" > VERSION +sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_src_dir}/_version.py" + rapids-logger "Begin py build" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) # TODO: Remove `--no-test` flag once importing on a CPU # node works correctly -rapids-conda-retry mambabuild \ +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ conda/recipes/cucim diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh new file mode 100755 index 000000000..a39ebf502 --- /dev/null +++ b/ci/build_wheel.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="cucim" +package_dir="python/cucim" +package_src_dir="${package_dir}/src/${package_name}" + +CMAKE_BUILD_TYPE="release" + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="${package_dir}/pyproject.toml" + +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" +# update package name to have the cuda suffix +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +echo "${version}" > VERSION +sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_src_dir}/_version.py" + +if [[ ${PACKAGE_CUDA_SUFFIX} == "-cu12" ]]; then + # change pyproject.toml to use CUDA 12.x version of cupy + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} +fi + +# Install pip build dependencies (not yet using pyproject.toml) +rapids-dependency-file-generator \ + --file_key "py_build" \ + --output "requirements" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee build_requirements.txt +pip install -r build_requirements.txt + +# First build the C++ lib using CMake via the run script +./run build_local all ${CMAKE_BUILD_TYPE} + +cd "${package_dir}" + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 37cee5a67..3669c2dda 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -34,12 +34,9 @@ function sed_runner() { sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/source/conf.py sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf.py -sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cucim/pyproject.toml -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" VERSION -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" python/cucim/VERSION -sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cucim/src/cucim/__init__.pyi -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cuslide/VERSION -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cumed/VERSION +# Centralized version file update +echo "${NEXT_FULL_TAG}" > VERSION + sed_runner "s#\[Version ${CURRENT_LONG_TAG}\](release_notes/v${CURRENT_LONG_TAG}.md)#\[Version ${NEXT_FULL_TAG}\](release_notes/v${NEXT_FULL_TAG}.md)#g" python/cucim/docs/index.md sed_runner "s/v${CURRENT_LONG_TAG}/v${NEXT_FULL_TAG}/g" python/cucim/docs/getting_started/index.md sed_runner "s#cucim.kit.cuslide@${CURRENT_LONG_TAG}.so#cucim.kit.cuslide@${NEXT_FULL_TAG}.so#g" python/cucim/docs/getting_started/index.md diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh new file mode 100755 index 000000000..2fe2c1fb9 --- /dev/null +++ b/ci/test_wheel.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eou pipefail + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="cucim_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install $(echo ./dist/cucim*.whl)[test] + +# Run smoke tests for aarch64 pull requests +if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then + python ./ci/wheel_smoke_test.py +else + # TODO: revisit enabling imagecodecs package during testing + python -m pytest ./python/cucim +fi diff --git a/ci/wheel_smoke_test.py b/ci/wheel_smoke_test.py new file mode 100644 index 000000000..e86a8888e --- /dev/null +++ b/ci/wheel_smoke_test.py @@ -0,0 +1,21 @@ +import cupy as cp + +import cucim +import cucim.skimage + + +if __name__ == "__main__": + # verify that all top-level modules are available + assert cucim.is_available("clara") + assert cucim.is_available("core") + assert cucim.is_available("skimage") + + # generate a synthetic image and apply a filter + img = cucim.skimage.data.binary_blobs(length=512, n_dim=2) + assert isinstance(img, cp.ndarray) + assert img.dtype.kind == "b" + assert img.shape == (512, 512) + + eroded = cucim.skimage.morphology.binary_erosion( + img, cp.ones((3, 3), dtype=bool) + ) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 19d15f9a6..c038513af 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -24,6 +24,7 @@ dependencies: - libnvjpeg-dev=11.6.0.55 - libnvjpeg=11.6.0.55 - libwebp-base +- matplotlib-base - nbsphinx - ninja - numpy>=1.21.3 @@ -31,6 +32,7 @@ dependencies: - nvcc_linux-64=11.8 - openslide-python>=1.1.2 - pip +- pooch>=1.6.0 - pre-commit - psutil>=5.8.0 - pydata-sphinx-theme diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 982a2c627..18f61bf9f 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -24,12 +24,14 @@ dependencies: - libnvjpeg-dev - libnvjpeg-static - libwebp-base +- matplotlib-base - nbsphinx - ninja - numpy>=1.21.3 - numpydoc - openslide-python>=1.1.2 - pip +- pooch>=1.6.0 - pre-commit - psutil>=5.8.0 - pydata-sphinx-theme diff --git a/conda/recipes/cucim/meta.yaml b/conda/recipes/cucim/meta.yaml index 609ca09c5..dbb34a906 100644 --- a/conda/recipes/cucim/meta.yaml +++ b/conda/recipes/cucim/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2021-2023, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} {% set py_version = environ['CONDA_PY'] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} @@ -12,7 +12,7 @@ package: version: {{ version }} source: - git_url: ../../.. + path: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} diff --git a/conda/recipes/libcucim/meta.yaml b/conda/recipes/libcucim/meta.yaml index 4140bbf1d..666a56d90 100644 --- a/conda/recipes/libcucim/meta.yaml +++ b/conda/recipes/libcucim/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2021-2023, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} {% set date_string = environ['RAPIDS_DATE_STRING'] %} @@ -11,7 +11,7 @@ package: version: {{ version }} source: - git_url: ../../.. + path: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} diff --git a/cpp/cmake/deps/openslide.cmake b/cpp/cmake/deps/openslide.cmake index 651acd5ff..8e44e33a5 100644 --- a/cpp/cmake/deps/openslide.cmake +++ b/cpp/cmake/deps/openslide.cmake @@ -22,7 +22,9 @@ if (NOT TARGET deps::openslide) set(OPENSLIDE_LIB_PATH "$ENV{CONDA_PREFIX}/lib/libopenslide.so") elseif (EXISTS /usr/lib/x86_64-linux-gnu/libopenslide.so) set(OPENSLIDE_LIB_PATH /usr/lib/x86_64-linux-gnu/libopenslide.so) - else () # CentOS 6 + elseif (EXISTS /usr/lib/aarch64-linux-gnu/libopenslide.so) + set(OPENSLIDE_LIB_PATH /usr/lib/aarch64-linux-gnu/libopenslide.so) + else () # CentOS (x86_64) set(OPENSLIDE_LIB_PATH /usr/lib64/libopenslide.so) endif () diff --git a/cpp/plugins/cucim.kit.cumed/CMakeLists.txt b/cpp/plugins/cucim.kit.cumed/CMakeLists.txt index 4073ecfd5..dc775742d 100644 --- a/cpp/plugins/cucim.kit.cumed/CMakeLists.txt +++ b/cpp/plugins/cucim.kit.cumed/CMakeLists.txt @@ -23,7 +23,9 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION unset(VERSION CACHE) -file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION) +file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION VERSION) +# strip alpha version info +string(REGEX REPLACE "a.*$" "" VERSION ${VERSION}) # Append local cmake module path list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules") diff --git a/cpp/plugins/cucim.kit.cumed/VERSION b/cpp/plugins/cucim.kit.cumed/VERSION deleted file mode 100644 index a193fff41..000000000 --- a/cpp/plugins/cucim.kit.cumed/VERSION +++ /dev/null @@ -1 +0,0 @@ -23.12.00 diff --git a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt index 37dbca4f7..047b5ee61 100644 --- a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt +++ b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt @@ -23,7 +23,9 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION unset(VERSION CACHE) -file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION) +file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION VERSION) +# strip alpha version info +string(REGEX REPLACE "a.*$" "" VERSION ${VERSION}) # Append local cmake module path list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules") diff --git a/cpp/plugins/cucim.kit.cuslide/VERSION b/cpp/plugins/cucim.kit.cuslide/VERSION deleted file mode 100644 index a193fff41..000000000 --- a/cpp/plugins/cucim.kit.cuslide/VERSION +++ /dev/null @@ -1 +0,0 @@ -23.12.00 diff --git a/cpp/plugins/cucim.kit.cuslide/cmake/deps/openslide.cmake b/cpp/plugins/cucim.kit.cuslide/cmake/deps/openslide.cmake index 651acd5ff..8e44e33a5 100644 --- a/cpp/plugins/cucim.kit.cuslide/cmake/deps/openslide.cmake +++ b/cpp/plugins/cucim.kit.cuslide/cmake/deps/openslide.cmake @@ -22,7 +22,9 @@ if (NOT TARGET deps::openslide) set(OPENSLIDE_LIB_PATH "$ENV{CONDA_PREFIX}/lib/libopenslide.so") elseif (EXISTS /usr/lib/x86_64-linux-gnu/libopenslide.so) set(OPENSLIDE_LIB_PATH /usr/lib/x86_64-linux-gnu/libopenslide.so) - else () # CentOS 6 + elseif (EXISTS /usr/lib/aarch64-linux-gnu/libopenslide.so) + set(OPENSLIDE_LIB_PATH /usr/lib/aarch64-linux-gnu/libopenslide.so) + else () # CentOS (x86_64) set(OPENSLIDE_LIB_PATH /usr/lib64/libopenslide.so) endif () diff --git a/cpp/plugins/cucim.kit.cuslide/src/cuslide/jpeg2k/gen_color_table.py b/cpp/plugins/cucim.kit.cuslide/src/cuslide/jpeg2k/gen_color_table.py index 9f442af7c..9d9de21f3 100644 --- a/cpp/plugins/cucim.kit.cuslide/src/cuslide/jpeg2k/gen_color_table.py +++ b/cpp/plugins/cucim.kit.cuslide/src/cuslide/jpeg2k/gen_color_table.py @@ -177,8 +177,12 @@ def gen_b_cb(): def gen_list(values: list, width: int, align: int = 8): text = [] for i in range(0, len(values), width): - text.append(", ".join(("{:>"+str(align)+"}").format(item) - for item in values[i:i + width])) + text.append( + ", ".join( + ("{:>" + str(align) + "}").format(item) + for item in values[i : i + width] + ) + ) return ",\n ".join(text) @@ -189,14 +193,16 @@ def main(output_file_name: str) -> int: b_cb = gen_list(list(gen_b_cb()), 10, 4) with open(output_file_name, "w") as f: - f.write(TEMPLATE % - {"r_cr": r_cr, "g_cb": g_cb, "g_cr": g_cr, "b_cb": b_cb}) + f.write( + TEMPLATE % {"r_cr": r_cr, "g_cb": g_cb, "g_cr": g_cr, "b_cb": b_cb} + ) return 0 if __name__ == "__main__": import sys + if len(sys.argv) != 2: print("Usage: gen_color_table.py ") sys.exit(1) diff --git a/dependencies.yaml b/dependencies.yaml index fddf3add9..b7ceeef02 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -245,32 +245,27 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - GPUtil>=1.4.0 - - imagecodecs>=2021.6.8 - - openslide-python>=1.1.2 - psutil>=5.8.0 - pytest-cov>=2.12.1 - pytest-lazy-fixture>=0.6.3 - pytest-xdist - pytest>=6.2.4 - tifffile>=2022.7.28 + - pooch>=1.6.0 # needed to download scikit-image sample data - output_types: [conda] packages: + - imagecodecs>=2021.6.8 + - matplotlib-base + - openslide-python>=1.1.2 - pip - pip: - opencv-python-headless>=4.6 - output_types: [requirements, pyproject] packages: + # temporarily remove imagecodecs / openslide-python from wheel tests + # # skip packages on arm64 that don't provide a wheel + # - imagecodecs>=2021.6.8; platform_machine=='x86_64' + # - openslide-python>=1.1.2; platform_machine=='x86_64' + - matplotlib - opencv-python-headless>=4.6 - # All dependencies below this point are specific to `cucim.clara` and - # are not needed for either `cucim.core` or `cucim.skimage`, so are - # listed as optional. They are needed in order to run the full test - # suite, including the `cucim.clara` package. - click - - jbig - - libwebp-base - - xz - - zlib - - zstd - # Not sure where these go, if anywhere: - # - openslide - # - xorg-libxcb diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 2a3277457..adbbde7e9 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -24,6 +24,8 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION unset(VERSION CACHE) file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../VERSION VERSION) +# strip alpha version info +string(REGEX REPLACE "a.*$" "" VERSION ${VERSION}) # Append local cmake module path list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules") diff --git a/python/cucim/VERSION b/python/cucim/VERSION deleted file mode 100644 index a193fff41..000000000 --- a/python/cucim/VERSION +++ /dev/null @@ -1 +0,0 @@ -23.12.00 diff --git a/python/cucim/pyproject.toml b/python/cucim/pyproject.toml index 8af8dca84..1136ee877 100644 --- a/python/cucim/pyproject.toml +++ b/python/cucim/pyproject.toml @@ -10,7 +10,7 @@ requires = [ [project] name = "cucim" -version = "23.12.00" +dynamic = ["version"] description = "cuCIM - an extensible toolkit designed to provide GPU accelerated I/O, computer vision & image processing primitives for N-Dimensional images with a focus on biomedical imaging." # TODO: tried also adding CHANGELOG.md as in setup.py's long_description, but ruff complained about it readme = { file = "README.md", content-type = "text/markdown" } @@ -62,20 +62,15 @@ Tracker = "https://github.com/rapidsai/cucim/issues" test = [ "GPUtil>=1.4.0", "click", - "imagecodecs>=2021.6.8", - "jbig", - "libwebp-base", + "matplotlib", "opencv-python-headless>=4.6", - "openslide-python>=1.1.2", + "pooch>=1.6.0", "psutil>=5.8.0", "pytest-cov>=2.12.1", "pytest-lazy-fixture>=0.6.3", "pytest-xdist", "pytest>=6.2.4", "tifffile>=2022.7.28", - "xz", - "zlib", - "zstd", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. developer = [ "black", @@ -97,15 +92,16 @@ cucim = "cucim.clara.cli:main" [tool.setuptools] license-files = ["LICENSE"] -# By default, include-package-data is true in pyproject.toml, so you do -# NOT have to specify this line. include-package-data = true +[tool.setuptools.dynamic] +version = {file = "src/cucim/VERSION"} + [tool.setuptools.packages.find] where = ["src"] [tool.setuptools.package-data] -mypkg = ["*.pyi", "*.h", "*.cu"] +mypkg = ["*.pyi", "*.h", "*.cu", "VERSION"] [tool.pytest.ini_options] # If a pytest section is found in one of the possible config files diff --git a/python/cucim/src/cucim/VERSION b/python/cucim/src/cucim/VERSION new file mode 120000 index 000000000..a4e948506 --- /dev/null +++ b/python/cucim/src/cucim/VERSION @@ -0,0 +1 @@ +../../../../VERSION \ No newline at end of file diff --git a/python/cucim/src/cucim/__init__.py b/python/cucim/src/cucim/__init__.py index ed21dc97d..b655754be 100644 --- a/python/cucim/src/cucim/__init__.py +++ b/python/cucim/src/cucim/__init__.py @@ -40,6 +40,8 @@ submodules = [] submod_attrs = {} +from ._version import __git_commit__, __version__ + try: import cupy @@ -49,18 +51,13 @@ pass try: - from .clara import CuImage, __version__, cli + from .clara import CuImage, cli _is_clara_available = True submodules += ["clara"] submod_attrs["clara"] = ["CuImage", "cli"] except ImportError: - from ._version import get_versions - - __version__ = get_versions()["version"] - del get_versions - del _version - + pass import lazy_loader as lazy @@ -68,7 +65,7 @@ def __dir__(): - return __lazy_dir__() + ["__version__", "is_available"] + return __lazy_dir__() + ["__git_commit__", "__version__", "is_available"] def is_available(module_name: str = "") -> bool: diff --git a/python/cucim/src/cucim/__init__.pyi b/python/cucim/src/cucim/__init__.pyi index 8c7b467f9..646c41b65 100644 --- a/python/cucim/src/cucim/__init__.pyi +++ b/python/cucim/src/cucim/__init__.pyi @@ -13,6 +13,8 @@ # limitations under the License. # +from ._version import __git_commit__, __version__ + submodules = [] try: @@ -25,11 +27,15 @@ except ImportError: pass try: - from .clara import CuImage, __version__, cli # noqa: F401 + from .clara import CuImage, cli # noqa: F401 _is_clara_available = True submodules += ["clara"] except ImportError: - __version__ = "23.12.00" + pass -__all__ = submodules + ["__version__", "is_available"] # noqa: F822 +__all__ = submodules + [ # noqa: F822 + "__git_commit__", + "__version__", + "is_available", +] diff --git a/python/cucim/src/cucim/_version.py b/python/cucim/src/cucim/_version.py new file mode 100644 index 000000000..9abb33848 --- /dev/null +++ b/python/cucim/src/cucim/_version.py @@ -0,0 +1,20 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files("cucim").joinpath("VERSION").read_text().strip() +) +__git_commit__ = "" diff --git a/python/cucim/src/cucim/clara/__init__.py b/python/cucim/src/cucim/clara/__init__.py index 4c9fea2f6..537d6b064 100644 --- a/python/cucim/src/cucim/clara/__init__.py +++ b/python/cucim/src/cucim/clara/__init__.py @@ -18,15 +18,7 @@ from . import cli, converter # import hidden methods -from ._cucim import ( - CuImage, - DLDataType, - DLDataTypeCode, - __version__, - cache, - filesystem, - io, -) +from ._cucim import CuImage, DLDataType, DLDataTypeCode, cache, filesystem, io __all__ = [ "cli", @@ -37,7 +29,6 @@ "io", "cache", "converter", - "__version__", ] diff --git a/python/cucim/src/cucim/skimage/measure/tests/test_blur_effect.py b/python/cucim/src/cucim/skimage/measure/tests/test_blur_effect.py index 1fe103b9c..8b4afe337 100644 --- a/python/cucim/src/cucim/skimage/measure/tests/test_blur_effect.py +++ b/python/cucim/src/cucim/skimage/measure/tests/test_blur_effect.py @@ -46,8 +46,12 @@ def test_blur_effect_channel_axis(): def test_blur_effect_3d(): """Test that the blur metric works on a 3D image.""" - cells3d = pytest.importorskip("skimage.data.cells3d") - image_3d = cp.array(cells3d()[:, 1, :, :]) # grab just the nuclei + data = pytest.importorskip("skimage.data") + if not hasattr(data, "cells3d"): + pytest.skip( + "cells3d data not available in this version of scikit-image" + ) + image_3d = cp.array(data.cells3d()[:, 1, :, :]) # grab just the nuclei B0 = blur_effect(image_3d) B1 = blur_effect(gaussian(image_3d, sigma=1)) B2 = blur_effect(gaussian(image_3d, sigma=4)) diff --git a/python/cucim/src/cucim/skimage/registration/tests/test_masked_phase_cross_correlation.py b/python/cucim/src/cucim/skimage/registration/tests/test_masked_phase_cross_correlation.py index e83c638fa..79c0c8255 100644 --- a/python/cucim/src/cucim/skimage/registration/tests/test_masked_phase_cross_correlation.py +++ b/python/cucim/src/cucim/skimage/registration/tests/test_masked_phase_cross_correlation.py @@ -75,8 +75,10 @@ def test_masked_registration_random_masks(): def test_masked_registration_3d_contiguous_mask(): """masked_register_translation should be able to register translations between volumes with contiguous masks.""" - brain = pytest.importorskip("skimage.data.brain") - ref_vol = cp.array(brain()[:, ::2, ::2]) + data = pytest.importorskip("skimage.data") + if not hasattr(data, "brain"): + pytest.skip("brain data not available in this version of scikit-image") + ref_vol = cp.array(data.brain()[:, ::2, ::2]) offset = (1, -5, 10) diff --git a/python/cucim/src/cucim/skimage/transform/tests/test_warps.py b/python/cucim/src/cucim/skimage/transform/tests/test_warps.py index 22a5aad60..403cc12ea 100644 --- a/python/cucim/src/cucim/skimage/transform/tests/test_warps.py +++ b/python/cucim/src/cucim/skimage/transform/tests/test_warps.py @@ -81,7 +81,6 @@ def shift(xy): assert_array_almost_equal(outx, refx) -@cp.testing.with_requires("cupy>=9.0.0b2") def test_warp_matrix(): x = cp.zeros((5, 5), dtype=cp.float64) x[2, 2] = 1 @@ -116,7 +115,6 @@ def test_warp_nd(): assert_array_almost_equal(outx, refx) -@cp.testing.with_requires("cupy>=9.0.0b2") def test_warp_clip(): x = cp.zeros((5, 5), dtype=cp.float64) x[2, 2] = 1 @@ -573,7 +571,6 @@ def test_warp_identity(): assert cp.all(0 == warped_rgb_img[:, :, 1]) -@cp.testing.with_requires("cupy>=9.0.0b2") def test_warp_coords_example(): image = cp.array(astronaut().astype(cp.float32)) assert 3 == image.shape[2] diff --git a/python/cucim/tests/performance/clara/test_read_region_memory_usage.py b/python/cucim/tests/performance/clara/test_read_region_memory_usage.py index a0137444c..12aacab1a 100644 --- a/python/cucim/tests/performance/clara/test_read_region_memory_usage.py +++ b/python/cucim/tests/performance/clara/test_read_region_memory_usage.py @@ -17,6 +17,9 @@ from ...util.io import open_image_cucim +# skip if imagecodecs package not available (needed by ImageGenerator utility) +pytest.importorskip("imagecodecs") + def test_read_region_cuda_memleak(testimg_tiff_stripe_4096x4096_256_jpeg): import GPUtil diff --git a/python/cucim/tests/unit/clara/converter/test_converter.py b/python/cucim/tests/unit/clara/converter/test_converter.py index 6e54640ea..8198e78f7 100644 --- a/python/cucim/tests/unit/clara/converter/test_converter.py +++ b/python/cucim/tests/unit/clara/converter/test_converter.py @@ -16,6 +16,12 @@ import os from pathlib import Path +import pytest + +# skip if imagecodecs and openslide packages are not available +pytest.importorskip("imagecodecs") +pytest.importorskip("openslide") + def test_image_converter_stripe_4096x4096_256_jpeg( tmp_path, testimg_tiff_stripe_4096x4096_256_jpeg diff --git a/python/cucim/tests/unit/clara/test_image_cache.py b/python/cucim/tests/unit/clara/test_image_cache.py index f6f7d199f..1b8987765 100644 --- a/python/cucim/tests/unit/clara/test_image_cache.py +++ b/python/cucim/tests/unit/clara/test_image_cache.py @@ -13,6 +13,11 @@ # limitations under the License. # +import pytest + +# skip if imagecodecs package not available (needed by ImageGenerator utility) +pytest.importorskip("imagecodecs") + def test_get_nocache(): from cucim import CuImage diff --git a/python/cucim/tests/unit/clara/test_load_image_metadata.py b/python/cucim/tests/unit/clara/test_load_image_metadata.py index 971e4e171..cc434c4bd 100644 --- a/python/cucim/tests/unit/clara/test_load_image_metadata.py +++ b/python/cucim/tests/unit/clara/test_load_image_metadata.py @@ -15,8 +15,13 @@ import math +import pytest + from ...util.io import open_image_cucim +# skip if imagecodecs package not available (needed by ImageGenerator utility) +pytest.importorskip("imagecodecs") + def test_load_image_metadata(testimg_tiff_stripe_32x24_16): import numpy as np diff --git a/python/cucim/tests/unit/clara/test_tiff_read_region.py b/python/cucim/tests/unit/clara/test_tiff_read_region.py index 5b122fb50..f379847a5 100644 --- a/python/cucim/tests/unit/clara/test_tiff_read_region.py +++ b/python/cucim/tests/unit/clara/test_tiff_read_region.py @@ -18,6 +18,9 @@ from ...util.io import open_image_cucim +# skip if imagecodecs package not available (needed by ImageGenerator utility) +pytest.importorskip("imagecodecs") + def test_tiff_stripe_inner(testimg_tiff_stripe_32x24_16): cucim_img = open_image_cucim(testimg_tiff_stripe_32x24_16)