From 9382a3096cdb08c70423b2378808a71ed6ab5552 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Wed, 25 Oct 2023 15:56:22 -0400 Subject: [PATCH] remove outdated package/release/docs commands from the script The build_local command is needed and is used by conda build scripts --- run | 417 ------------------------------------------------------------ 1 file changed, 417 deletions(-) diff --git a/run b/run index ddcb283ac..ca4cfd47d 100755 --- a/run +++ b/run @@ -598,249 +598,6 @@ copy_nvjpeg_files_() { fi } -build_python_package_desc() { echo 'Build Python package - -Note: This command does not remove `dist` folder before building. -' -} -build_python_package() { - local ret=0 - local old_opt="$(shopt -op errexit);$(shopt -op nounset)" # save old shopts - set -eu - - # Copy cufile SDK from host system to temp/cuda - copy_gds_files_ - # Copy nvjpeg SDK from host system to temp/cuda - copy_nvjpeg_files_ - - run_command ${TOP}/dockcross-manylinux2014-x64 ./run build_python_package_ - ret=$? - - eval "${old_opt}" # restore old shopts - - return ${ret} -} - -repair_wheel_() { - local wheel="$1" - local dest="${2:-./}" - local PLAT="${3:-manylinux2014_x86_64}" - - - if ! auditwheel show "$wheel"; then - echo "Skipping non-platform wheel ${wheel}" - else - $(head -1 $(which auditwheel) | cut -d'!' -f2) $TOP/scripts/auditwheel_repair.py repair --plat "${PLAT}" -w ${dest} "${wheel}" - fi -} - -build_python_package_() { - local SRC_ROOT=${1:-${TOP}} - local BUILD_ROOT=${2:-${TOP}/temp} - local DEST_ROOT=${3:-${TOP}/dist} - local CUCIM_SDK_PATH=${4:-${BUILD_ROOT}/libcucim} - - local old_opt="$(shopt -op errexit);$(shopt -op nounset)" # save old shopts - local major_version="$(cat ${TOP}/VERSION | cut -d. -f1)" # major version number - set -eu - - # Clear CMakeCache.txt to use the latest options - run_command rm -f ${BUILD_ROOT}/libcucim/CMakeCache.txt - run_command rm -f ${BUILD_ROOT}/cuslide/CMakeCache.txt - run_command rm -f ${BUILD_ROOT}/cumed/CMakeCache.txt - run_command rm -f ${BUILD_ROOT}/cucim/CMakeCache.txt - - # Create a folder for .whl file that repair_wheel_ is not applied. - TEMP_PYPKG_DIR=${BUILD_ROOT}/py_pkg # $(mktemp -d) - c_echo b "TEMP_PYPKG_DIR=${TEMP_PYPKG_DIR}" - mkdir -p $TEMP_PYPKG_DIR - rm -rf $TEMP_PYPKG_DIR/* - # trap 'rm -rf ${TEMP_PYPKG_DIR}' EXIT - - local CMAKE_CMD=${CMAKE_CMD:-cmake} - local CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} - local NUM_THREADS=${NUM_THREADS:-$(nproc)} - - local PLAT=manylinux2014_x86_64 - - local pybins - - mkdir -p ${DEST_ROOT} - - # Remove existing library files at build root - rm -rf ${BUILD_ROOT}/libcucim/lib/* - rm -rf ${BUILD_ROOT}/libcucim/install/lib*/lib* - rm -rf ${BUILD_ROOT}/cuslide/lib/* - rm -rf ${BUILD_ROOT}/cuslide/install/lib*/cucim* - rm -rf ${BUILD_ROOT}/cumed/lib/* - rm -rf ${BUILD_ROOT}/cumed/install/lib*/cucim* - rm -rf ${BUILD_ROOT}/cucim/lib/cucim/*.so* - rm -rf ${BUILD_ROOT}/cucim/install/lib/*.so* - - # Build libcucim - ${CMAKE_CMD} -S ${SRC_ROOT} -B ${BUILD_ROOT}/libcucim \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_PREFIX=${CUCIM_SDK_PATH}/install - ${CMAKE_CMD} --build ${BUILD_ROOT}/libcucim --config ${CMAKE_BUILD_TYPE} --target cucim -- -j ${NUM_THREADS} - ${CMAKE_CMD} --build ${BUILD_ROOT}/libcucim --config ${CMAKE_BUILD_TYPE} --target install -- -j ${NUM_THREADS} - - # Build cuslide plugin - ${CMAKE_CMD} -S ${SRC_ROOT}/cpp/plugins/cucim.kit.cuslide -B ${BUILD_ROOT}/cuslide \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_PREFIX=${BUILD_ROOT}/cuslide/install \ - -DCUCIM_SDK_PATH=${CUCIM_SDK_PATH} - ${CMAKE_CMD} --build ${BUILD_ROOT}/cuslide --config ${CMAKE_BUILD_TYPE} --target cucim.kit.cuslide -- -j ${NUM_THREADS} - ${CMAKE_CMD} --build ${BUILD_ROOT}/cuslide --config ${CMAKE_BUILD_TYPE} --target install -- -j ${NUM_THREADS} - - # Build cumed plugin - ${CMAKE_CMD} -S ${SRC_ROOT}/cpp/plugins/cucim.kit.cumed -B ${BUILD_ROOT}/cumed \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_PREFIX=${BUILD_ROOT}/cumed/install \ - -DCUCIM_SDK_PATH=${CUCIM_SDK_PATH} - ${CMAKE_CMD} --build ${BUILD_ROOT}/cumed --config ${CMAKE_BUILD_TYPE} --target cucim.kit.cumed -- -j ${NUM_THREADS} - ${CMAKE_CMD} --build ${BUILD_ROOT}/cumed --config ${CMAKE_BUILD_TYPE} --target install -- -j ${NUM_THREADS} - - # Build Python bind - pybins="$(echo /opt/python/*/bin)" - if [ "${pybins}" = "/opt/python/*/bin" ]; then - pybins="$(dirname $(which python3))" # for building at host - fi - for PYBIN in ${pybins}; do - local python_library=$(${PYBIN}/python3 -c "import distutils.sysconfig as sysconfig; import os; print(os.path.join(sysconfig.get_config_var('LIBDIR'), sysconfig.get_config_var('LDLIBRARY')))") - local python_include_dir=$(${PYBIN}/python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") - - ${CMAKE_CMD} -S ${SRC_ROOT}/python -B ${BUILD_ROOT}/cucim \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DCMAKE_INSTALL_PREFIX=${BUILD_ROOT}/cucim/install \ - -DCUCIM_SDK_PATH=${CUCIM_SDK_PATH} \ - -DPYTHON_EXECUTABLE=${PYBIN}/python3 \ - -DPYTHON_LIBRARY=${python_library} \ - -DPYTHON_INCLUDE_DIR=${python_include_dir} - ${CMAKE_CMD} --build ${BUILD_ROOT}/cucim --config ${CMAKE_BUILD_TYPE} --target cucim -- -j ${NUM_THREADS} - ${CMAKE_CMD} --build ${BUILD_ROOT}/cucim --config ${CMAKE_BUILD_TYPE} --target install -- -j ${NUM_THREADS} - done - - # Remove existing library files at python/cucim/src/cucim/clara - rm -f ${SRC_ROOT}/python/cucim/src/cucim/clara/*.so* - - # Copy .so files to pybind's build folder - # (it uses -P to copy symbolic links as they are) - cp -P ${BUILD_ROOT}/libcucim/install/lib*/lib* ${BUILD_ROOT}/cucim/install/lib/ - cp -P ${BUILD_ROOT}/cuslide/install/lib*/cucim* ${BUILD_ROOT}/cucim/install/lib/ - cp -P ${BUILD_ROOT}/cumed/install/lib*/cucim* ${BUILD_ROOT}/cucim/install/lib/ - - # Copy .so files from pybind's build folder to cucim Python source folder - # Since wheel file doesn't support symbolic link (https://github.com/pypa/wheel/issues/203), - # we don't need to copy symbolic links. Instead copy only libcucim.so.${major_version} (without symbolic link) - #find ${BUILD_ROOT}/cucim/install/lib -maxdepth 1 -type f -exec cp {} ${SRC_ROOT}/python/cucim/src/cucim/clara/ \; - cp ${BUILD_ROOT}/cucim/install/lib/_cucim.*.so ${SRC_ROOT}/python/cucim/src/cucim/clara/ - cp ${BUILD_ROOT}/cucim/install/lib/cucim.*.so ${SRC_ROOT}/python/cucim/src/cucim/clara/ - cp ${BUILD_ROOT}/cucim/install/lib/libcucim.so.${major_version} ${SRC_ROOT}/python/cucim/src/cucim/clara/ - find ${BUILD_ROOT}/cucim/install/lib -maxdepth 1 -type f -name "lib*.so" -exec cp {} ${SRC_ROOT}/python/cucim/src/cucim/clara/ \; - - cd ${SRC_ROOT}/python/cucim - - # Remove build folder - rm -rf ${SRC_ROOT}/python/cucim/build - # Compile wheels (one python binary is enough) - pybins="$(echo /opt/python/*/bin)" - [ ! -e /opt/python/cp39-cp39/bin ] && pybins="$(dirname $(which python3))" # for building at host - - pybins="$(echo /opt/python/*/bin)" - if [ "${pybins}" = "/opt/python/*/bin" ]; then # if multiple python binaries not found - pybins="$(dirname $(which python3))" # for building at host - else - pybins=/opt/python/cp39-cp39/bin # use Python 3.9 for executing `pip wheel` - fi - - for PYBIN in ${pybins}; do # /opt/python/*/bin - # run_command "${PYBIN}/python3" setup.py bdist_wheel -p $PLAT -d ${TEMP_PYPKG_DIR} - # no pip equivalent of -p for platform? - run_command "${PYBIN}/python3" -m pip wheel . --wheel-dir ${TEMP_PYPKG_DIR} -vv - done - - # Do not bundle external shared libraries for now. - # (CUDA-related libraries cannot be redistributed without EULA messages confirmed by user) - # Here, we just copy to dist folder. - # cp ${TEMP_PYPKG_DIR}/*.whl ${DEST_ROOT}/ - - # Bundle external shared libraries into the wheels - for whl in ${TEMP_PYPKG_DIR}/*.whl; do - repair_wheel_ "$whl" "${DEST_ROOT}/" "${PLAT}" - done - - # run_command rm -rf ${TEMP_PYPKG_DIR} - # trap -- EXIT - - # Copy cpp packages and examples - mkdir -p ${DEST_ROOT}/examples/cpp - cp -P -r ${BUILD_ROOT}/libcucim/install ${DEST_ROOT}/ - cp -P -r ${BUILD_ROOT}/cuslide/install/lib/*.so ${DEST_ROOT}/install/lib/ - cp -P -r ${BUILD_ROOT}/cumed/install/lib/*.so ${DEST_ROOT}/install/lib/ - cp -r ${SRC_ROOT}/examples/cpp/tiff_image ${DEST_ROOT}/examples/cpp/ - - cp ${BUILD_ROOT}/libcucim/CMakeLists.txt.examples.release ${DEST_ROOT}/examples/cpp/CMakeLists.txt - - # # Install packages and test - # for PYBIN in /opt/python/*/bin/; do - # "${PYBIN}/pip" install python-manylinux-demo --no-index -f /io/wheelhouse - # (cd "$HOME"; "${PYBIN}/nosetests" pymanylinuxdemo) - # done - # python setup.py bdist_wheel -p manylinux2014-x86_64 - - eval "${old_opt}" # restore old shopts -} - -build_package_desc() { echo 'Build package for release (& gen_docs) -' -} -build_package() { - local SRC_ROOT=${1:-${TOP}} - local BUILD_ROOT=${2:-${TOP}/temp} - local DEST_ROOT=${3:-${TOP}/dist} - local CUCIM_SDK_PATH=${4:-${BUILD_ROOT}/libcucim} - - # Clean dist folder - mkdir -p ${DEST_ROOT} - [ -n "${DEST_ROOT}" ] && run_command sudo rm -rf ${DEST_ROOT}/* - - build_python_package ${SRC_ROOT} ${BUILD_ROOT} ${DEST_ROOT} ${CUCIM_SDK_PATH} - gen_docs ${DEST_ROOT}/docs - copy_data_ ${SRC_ROOT} ${BUILD_ROOT} ${DEST_ROOT} - - # Copy the built wheel file into ${TOP}/notebooks - run_command cp ${DEST_ROOT}/*.whl ${TOP}/notebooks/ -} - -copy_data_() { - c_echo W "Copy necessary files for packaging..." - local SRC_ROOT=${1:-${TOP}} - local BUILD_ROOT=${2:-${TOP}/temp} - local DEST_ROOT=${3:-${TOP}/dist} - - # Create notebooks folder (add notebooks and scripts) - run_command mkdir -p ${DEST_ROOT}/notebooks/static_images - run_command cp $(git ls-files ${SRC_ROOT}/notebooks/*.ipynb) ${DEST_ROOT}/notebooks/ - run_command cp ${SRC_ROOT}/notebooks/static_images/*.png ${DEST_ROOT}/notebooks/static_images/ - - # Create docker folder - run_command mkdir -p ${DEST_ROOT}/docker - run_command cp ${SRC_ROOT}/docker/*-jupyter{,-gds,.txt} ${DEST_ROOT}/docker/ - run_command cp ${SRC_ROOT}/docker/*-claratrain{,.txt} ${DEST_ROOT}/docker/ - run_command cp ${SRC_ROOT}/docker/*-cmake ${DEST_ROOT}/docker/ - run_command cp ${SRC_ROOT}/docker/cufile.json ${DEST_ROOT}/docker/ # Copy cufile.json - - # Copy main script - run_command cp ${SRC_ROOT}/scripts/run-dist ${DEST_ROOT}/run - - # Create .dockerignore to speed up docker run - echo "notebooks" > ${DEST_ROOT}/.dockerignore - - # Copy license files - run_command cp -r ${SRC_ROOT}/3rdparty ${SRC_ROOT}/LICENSE* ${DEST_ROOT}/ -} - - #================================================================================== # Section: Test #================================================================================== @@ -1151,61 +908,6 @@ gen_docs_dev() { popd > /dev/null } -publish_docs_desc() { echo 'Publish generated documents to the server - -Publish generated documents to $GITLAB_PUBLISH_PROJECT_URL -The web page is available at the following: -- $GITLAB_PUBLISH_PROJECT_URL - -Arguments: - $1 - If specified, force creating a tag with the specified tag name. - Use "latest" tag to make public documents up to date. - -Returns: - None - - Exit code: - exit code returned from publishing documents -' -} -publish_docs() { - local release_version="$(cat ${TOP}/VERSION)" - local release_tag="${1:-v${release_version}}" - - # Import secrets - import_env_vars_ || return 1 - - c_echo W "Publishing documents to Gitlab Pages..." - TEMP_DOCS_DIR=$(mktemp -d) - c_echo r "TEMP_DOCS_DIR=${TEMP_DOCS_DIR}" - trap 'rm -rf ${TEMP_DOCS_DIR}' EXIT - - pushd ${TEMP_DOCS_DIR} > /dev/null - - git clone ${GITLAB_PUBLISH_GIT_URL} --branch init --single-branch - cd ${GITLAB_PUBLISH_PROJECT_NAME} - mkdir -p dist/docs - cp -rf ${TOP}/dist/docs/* dist/docs/ - git checkout -b pages - git add dist/docs - git commit -am "Upload home page v$(cat ${TOP}/VERSION)" - git tag -f ${release_tag} - git push -f origin pages - git push -f origin ${release_tag} - popd > /dev/null - - # Create tag if custom (such as 'latest') tag is specified - if [ -n "${1:-}" ]; then - git tag -f "$1" - git push -f origin "$1" - fi - - run_command rm -rf ${TEMP_DOCS_DIR} - trap -- EXIT - - c_echo W "Checkout the published webpage!" -} - #================================================================================== # Section: Release #================================================================================== @@ -1248,113 +950,6 @@ import_env_vars_() { # Import environment variables . ${TOP}/.env } -release_package_desc() { echo 'Release generated packages to the server - -Release generated package files to $GITLAB_PUBLISH_PROJECT_URL -This leverages "release" environment in tox and uses - https://gitlab.com/alelec/gitlab-release -for uploading files to the server. - -Returns: - None - - Exit code: - exit code returned from releasing packages -' -} -release_package() { - local release_version="$(cat ${TOP}/VERSION)" - local release_tag="v${release_version}" - local wheel_file_name="cucim-${release_version}-py3-none-manylinux2014_x86_64.whl" - local package_file_name="cuCIM-${release_tag}-linux.tar.gz" - local package_data - local release_note_md="${TOP}/python/cucim/docs/release_notes/${release_tag}.md" - - # Import secrets - import_env_vars_ || return 1 - - # Publish docs first to create a release tag - publish_docs - - old_opt="$(shopt -op errexit);$(shopt -op nounset)" # save old shopts - set -eu - trap 'eval "${old_opt}"; unset old_opt' EXIT - - pushd ${TOP}/python/cucim/ > /dev/null - - # https://github.com/gitlabhq/gitlabhq/blob/master/doc/api/releases/index.md - # https://docs.gitlab.com/ee/api/releases/index.html#delete-a-release - - # install jq if not exists - if ! command -v jq > /dev/null; then - c_echo G "jq" W " doesn't exists. Installing " G "jq" W "..." - sudo apt-get install -y jq - hash -r - fi - - if curl --silent --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" "${GITLAB_PUBLISH_SERVER}/api/v4/projects/${GITLAB_PUBLISH_PROJECT_ID}/releases/${release_tag}" | jq '.name' -e > /dev/null; then - read -n 1 -r -p "$(c_str R "Do you want to delete existing release " G "${release_tag}" R " (y/n)?")" - echo - if [[ $REPLY =~ ^[Yy]$ ]] - then - c_echo W "Removing release " G "${release_tag}..." - curl --request DELETE --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" "${GITLAB_PUBLISH_SERVER}/api/v4/projects/${GITLAB_PUBLISH_PROJECT_ID}/releases/${release_tag}" | jq - else - return - fi - fi - - run_command cd ${TOP}/dist - run_command rm -rf ${package_file_name} - run_command tar -czvf ${package_file_name} --exclude='*.gz' ./* ./.dockerignore # --exclude="*.whl" - package_url="${GITLAB_PUBLISH_SERVER}$(curl --request POST \ - --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" \ - --form "file=@${package_file_name}" \ - "${GITLAB_PUBLISH_SERVER}/api/v4/projects/${GITLAB_PUBLISH_PROJECT_ID}/uploads" \ - | jq '.full_path' -r)" - - wheel_url="${GITLAB_PUBLISH_SERVER}$(curl --request POST \ - --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" \ - --form "file=@${wheel_file_name}" \ - "${GITLAB_PUBLISH_SERVER}/api/v4/projects/${GITLAB_PUBLISH_PROJECT_ID}/uploads" \ - | jq '.full_path' -r)" - - package_data="$(echo " -import json -release_note = open('${release_note_md}').read() -data = {'tag_name': '${release_tag}', - 'description': release_note, - 'assets': { - 'links': [ - {'name': '${package_file_name}', - 'url': '${package_url}' - }, - {'name': '${wheel_file_name}', - 'url': '${wheel_url}' - } - ] - } - } -print(json.dumps(data)) -" | python -)" - c_echo b "${package_data}" - - # Force-push tag - git tag -f ${release_tag} && git push -f origin ${release_tag} - - # https://docs.gitlab.com/ee/api/releases/#create-a-release - curl --header 'Content-Type: application/json' --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" \ - --data "${package_data}" \ - --request POST "${GITLAB_PUBLISH_SERVER}/api/v4/projects/${GITLAB_PUBLISH_PROJECT_ID}/releases" | jq - - popd > /dev/null - - # tox -c ${TOP}/python/cucim -e release -- ${TOP}/run release_package_to_gitlab - - eval "${old_opt}" # restore old shopts - trap -- EXIT - unset old_opt -} parse_args() { local OPTIND @@ -1420,21 +1015,9 @@ main() { print_cmd_help_messages "${ARGS[@]}" exit 0 ;; - package) - build_package "${ARGS[@]}" - ;; notebooks) launch_notebooks "${ARGS[@]}" ;; - docs) - gen_docs "${ARGS[@]}" - ;; - docs:dev) - gen_docs_dev "${ARGS[@]}" - ;; - release) - release_package "${ARGS[@]}" - ;; ''|main) print_usage ;;