diff --git a/.github/workflows/houdini.yml b/.github/workflows/houdini.yml index 81f9709e7d..6bb74eac4d 100644 --- a/.github/workflows/houdini.yml +++ b/.github/workflows/houdini.yml @@ -64,7 +64,7 @@ jobs: ${{ needs.checksecret.outputs.HOUDINI_SECRETS == 'true' || github.repository_owner == 'AcademySoftwareFoundation' }} runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }} - name: hou:${{ matrix.config.hou }}-vfx:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }} + name: hou:${{ matrix.config.hou_hash }}-vfx:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }} container: image: aswf/ci-base:${{ matrix.config.image }} env: @@ -73,10 +73,11 @@ jobs: strategy: matrix: config: - #- { cxx: clang++, image: '2022', hou: '20_0', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' } - - { cxx: clang++, image: '2021', hou: '19_5', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' } - #- { cxx: clang++, image: '2022', hou: '20_0', build: 'Debug', components: 'core,hou' } - #- { cxx: g++, image: '2022', hou: '20_0', build: 'Release', components: 'core,hou' } + - { cxx: clang++, image: '2022', hou_hash: '20_0-newabi', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' } + - { cxx: clang++, image: '2022', hou_hash: '20_0-oldabi', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' } + - { cxx: clang++, image: '2021', hou_hash: '19_5', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' } + - { cxx: clang++, image: '2022', hou_hash: '20_0-newabi', build: 'Debug', components: 'core,hou' } + - { cxx: g++, image: '2022', hou_hash: '20_0-newabi', build: 'Release', components: 'core,hou' } fail-fast: false steps: - uses: actions/checkout@v3 @@ -93,14 +94,14 @@ jobs: uses: actions/cache@v3 with: path: /tmp/ccache - key: linux-vfx-hou${{ matrix.config.hou }}-${{ matrix.config.image }}-${{ matrix.config.cxx }}-${{ steps.timestamp.outputs.timestamp }} - restore-keys: linux-vfx-hou${{ matrix.config.hou }}-${{ matrix.config.image }}-${{ matrix.config.cxx }}- + key: linux-vfx-hou${{ matrix.config.hou_hash }}-${{ matrix.config.image }}-${{ matrix.config.cxx }}-${{ steps.timestamp.outputs.timestamp }} + restore-keys: linux-vfx-hou${{ matrix.config.hou_hash }}-${{ matrix.config.image }}-${{ matrix.config.cxx }}- - name: fetch_houdini uses: actions/cache/restore@v3 with: path: hou - key: dummy-houdini${{ matrix.config.hou }}-${{ steps.timestamp.outputs.timestamp }} - restore-keys: vdb-v5-houdini${{ matrix.config.hou }}- + key: dummy-houdini${{ matrix.config.hou_hash }}-${{ steps.timestamp.outputs.timestamp }} + restore-keys: vdb-v5-houdini${{ matrix.config.hou_hash }}- - name: validate_houdini run: test -f "hou/hou.tar.gz" # Make sure the cache is copied, not moved, as the cache action always posts the cache. @@ -113,10 +114,17 @@ jobs: cd $HOME/houdini_install && tar -xzf hou.tar.gz && cd - - name: build run: | - export HFS="$HOME/houdini_install/hou" - export HDSO="${HFS}/dsolib" - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib64:${HDSO}" - ./ci/build.sh -v --build-type=Release --components="core,hou" --cargs=\"-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DDISABLE_CMAKE_SEARCH_PATHS=ON -DOPENVDB_BUILD_HOUDINI_ABITESTS=OFF -DOPENVDB_HOUDINI_INSTALL_PREFIX=/tmp\ -DTBB_INCLUDEDIR=/usr/local/include -DTBB_LIBRARYDIR=/usr/local/lib\" + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib64" + ./ci/build.sh -v \ + --build-type=Release \ + --components="${{ matrix.config.components }}" \ + --cargs=\" \ + -DHOUDINI_ROOT=$HOME/houdini_install/hou \ + -DOPENVDB_BUILD_HOUDINI_ABITESTS=OFF \ + -DOPENVDB_HOUDINI_INSTALL_PREFIX=/tmp \ + -DDISABLE_CMAKE_SEARCH_PATHS=ON \ + -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON \ + \" - name: test run: cd build && ctest -V # Keep ccache light by stripping out any caches not accessed in the last day diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index 867e440398..ad4bd4d341 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -59,7 +59,7 @@ jobs: github.event.inputs.type == 'all' || github.event.inputs.type == 'houdini') runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }} - name: linux-houdini:${{ matrix.config.houdini_version }} + name: linux-houdini:${{ matrix.config.hou_hash }} env: CXX: clang++ HOUDINI_CLIENT_ID: ${{ secrets.HOUDINI_CLIENT_ID }} @@ -67,8 +67,9 @@ jobs: strategy: matrix: config: - - { houdini_version: '19.5', houdini_version_str: '19_5' } - #- { houdini_version: '20.0', houdini_version_str: '20_0' } + - { houdini_version: '19.5', platform: 'linux_x86_64_gcc9.3', hou_hash: '19_5' } + - { houdini_version: '20.0', platform: 'linux_x86_64_gcc9.3', hou_hash: '20_0-oldabi' } + - { houdini_version: '20.0', platform: 'linux_x86_64_gcc11.2', hou_hash: '20_0-newabi' } fail-fast: false container: image: aswf/ci-base:2023 @@ -88,7 +89,7 @@ jobs: id: timestamp run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - name: download_houdini - run: ./ci/download_houdini.sh ${{ matrix.config.houdini_version }} ON + run: ./ci/download_houdini.sh ${{ matrix.config.houdini_version }} ${{ matrix.config.platform }} --prod - name: install_houdini run: | mkdir $HOME/houdini_install @@ -98,7 +99,7 @@ jobs: uses: actions/cache/save@v3 with: path: hou - key: vdb-v5-houdini${{ matrix.config.houdini_version_str }}-${{ steps.timestamp.outputs.timestamp }} + key: vdb-v5-houdini${{ matrix.config.hou_hash }}-${{ steps.timestamp.outputs.timestamp }} ############################################################################# ########################### Core Library Extras ############################# diff --git a/ci/download_houdini.py b/ci/download_houdini.py old mode 100644 new mode 100755 index 7b65253d76..94a5f27b45 --- a/ci/download_houdini.py +++ b/ci/download_houdini.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python +#!/usr/bin/env python3 # # Copyright Contributors to the OpenVDB Project # SPDX-License-Identifier: MPL-2.0 @@ -19,16 +19,16 @@ import requests import hashlib import os +import argparse +import copy -# this argument is for the major.minor version of Houdini to download (such as 15.0, 15.5, 16.0) -version = sys.argv[1] -only_production = True if sys.argv[2] == 'ON' else False -user_client_id = os.getenv('HOUDINI_CLIENT_ID') -user_client_secret_key = os.getenv('HOUDINI_SECRET_KEY') - -if not re.match('[0-9][0-9]\.[0-9]$', version): - raise IOError('Invalid Houdini Version "%s", expecting in the form "major.minor" such as "16.0"' % version) - +# For progress bar printing +try: + from tqdm import tqdm + has_tqdm = True +except: + has_tqdm = False + pass # Code that provides convenient Python wrappers to call into the API: @@ -138,33 +138,72 @@ def __init__(self, http_code, message): self.http_code = http_code -service = service( - access_token_url="https://www.sidefx.com/oauth2/application_token", - client_id=user_client_id, - client_secret_key=user_client_secret_key, - endpoint_url="https://www.sidefx.com/api/", - ) +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Download a Houdini Installation') + parser.add_argument('version', type=str, help='Major.Minor version of Houdini to download') + parser.add_argument('platform', type=str, help='Platform target') + parser.add_argument('--prod', action='store_true', help='Only download production builds') + parser.add_argument('--list', action='store_true', help='Just list the available builds and exit.') + args = parser.parse_args() + + version = args.version + platform = args.platform + only_production = args.prod + + user_client_id = os.getenv('HOUDINI_CLIENT_ID') + user_client_secret_key = os.getenv('HOUDINI_SECRET_KEY') + + if not re.match('[0-9][0-9]\.[0-9]$', version): + raise IOError('Invalid Houdini Version "%s", expecting in the form "major.minor" such as "16.0"' % version) -releases_list = service.download.get_daily_builds_list( - product='houdini', version=version, platform='linux', only_production=only_production) + service = service( + access_token_url="https://www.sidefx.com/oauth2/application_token", + client_id=user_client_id, + client_secret_key=user_client_secret_key, + endpoint_url="https://www.sidefx.com/api/", + ) -latest_release = service.download.get_daily_build_download( - product='houdini', version=version, platform='linux', build=releases_list[0]['build']) + releases_list = service.download.get_daily_builds_list( + product='houdini', version=version, platform=platform, only_production=only_production) -# Download the file as hou.tar.gz -local_filename = 'hou.tar.gz' -response = requests.get(latest_release['download_url'], stream=True) -if response.status_code == 200: - with open(local_filename, 'wb') as f: + print('Available builds:') + for rel in releases_list: + rel = copy.deepcopy(rel) + if 'third_party_libraries' in rel: + # Don't print these + del rel['third_party_libraries'] + print(rel) + + if args.list: + sys.exit(0) + + print('Selecting build: ' + releases_list[0]['build']) + + latest_release = service.download.get_daily_build_download( + product='houdini', version=version, platform=platform, build=releases_list[0]['build']) + + # Download the file as hou.tar.gz + local_filename = 'hou.tar.gz' + response = requests.get(latest_release['download_url'], stream=True) + if response.status_code == 200: response.raw.decode_content = True - shutil.copyfileobj(response.raw, f) -else: - raise Exception('Error downloading file!') - -# Verify the file checksum is matching -file_hash = hashlib.md5() -with open(local_filename, 'rb') as f: - for chunk in iter(lambda: f.read(4096), b''): - file_hash.update(chunk) -if file_hash.hexdigest() != latest_release['hash']: - raise Exception('Checksum does not match!') + if has_tqdm: + file_size = int(response.headers.get('Content-Length', 0)) + desc = "(Unknown total file size)" if file_size == 0 else "" + with tqdm.wrapattr(response.raw, "read", total=file_size, desc=desc) as r_raw: + with open(local_filename, 'wb') as f: + shutil.copyfileobj(r_raw, f) + else: + with open(local_filename, 'wb') as f: + shutil.copyfileobj(response.raw, f) + else: + raise Exception('Error downloading file!') + + # Verify the file checksum is matching + file_hash = hashlib.md5() + with open(local_filename, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b''): + file_hash.update(chunk) + if file_hash.hexdigest() != latest_release['hash']: + raise Exception('Checksum does not match!') diff --git a/cmake/OpenVDBHoudiniSetup.cmake b/cmake/OpenVDBHoudiniSetup.cmake index 2de38b53f6..abf605b415 100644 --- a/cmake/OpenVDBHoudiniSetup.cmake +++ b/cmake/OpenVDBHoudiniSetup.cmake @@ -335,68 +335,67 @@ if(NOT OPENVDB_HOUDINI_ABI) endif() # ------------------------------------------------------------------------ -# Configure GCC CXX11 ABI +# Configure libstc++ CXX11 ABI # ------------------------------------------------------------------------ -if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - if((CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 5.1) OR - (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.1)) - message(STATUS "GCC >= 5.1 detected. Configuring GCC CXX11 ABI for Houdini compatibility...") - - execute_process(COMMAND echo "#include " - COMMAND ${CMAKE_CXX_COMPILER} "-x" "c++" "-E" "-dM" "-" - COMMAND grep "-F" "_GLIBCXX_USE_CXX11_ABI" - TIMEOUT 10 - RESULT_VARIABLE QUERIED_GCC_CXX11_ABI_SUCCESS - OUTPUT_VARIABLE _GCC_CXX11_ABI) - - set(GLIBCXX_USE_CXX11_ABI "UNKNOWN") - - if(NOT QUERIED_GCC_CXX11_ABI_SUCCESS) - string(FIND ${_GCC_CXX11_ABI} "_GLIBCXX_USE_CXX11_ABI 0" GCC_OLD_CXX11_ABI) - string(FIND ${_GCC_CXX11_ABI} "_GLIBCXX_USE_CXX11_ABI 1" GCC_NEW_CXX11_ABI) - if(NOT (${GCC_OLD_CXX11_ABI} EQUAL -1)) - set(GLIBCXX_USE_CXX11_ABI 0) - endif() - if(NOT (${GCC_NEW_CXX11_ABI} EQUAL -1)) - set(GLIBCXX_USE_CXX11_ABI 1) - endif() - endif() +if(UNIX AND NOT APPLE) + # Assume we're using libstdc++ + message(STATUS "Configuring CXX11 ABI for Houdini compatibility...") + + execute_process(COMMAND echo "#include " + COMMAND ${CMAKE_CXX_COMPILER} "-x" "c++" "-E" "-dM" "-" + COMMAND grep "-F" "_GLIBCXX_USE_CXX11_ABI" + TIMEOUT 10 + RESULT_VARIABLE QUERIED_GCC_CXX11_ABI_SUCCESS + OUTPUT_VARIABLE _GCC_CXX11_ABI) + + set(GLIBCXX_USE_CXX11_ABI "UNKNOWN") - # Try and query the Houdini CXX11 ABI. Allow it to be provided by users to - # override this logic should Houdini's CMake ever change - - if(NOT DEFINED HOUDINI_CXX11_ABI) - get_target_property(houdini_interface_compile_options - Houdini INTERFACE_COMPILE_OPTIONS) - set(HOUDINI_CXX11_ABI "UNKNOWN") - if("-D_GLIBCXX_USE_CXX11_ABI=0" IN_LIST houdini_interface_compile_options) - set(HOUDINI_CXX11_ABI 0) - elseif("-D_GLIBCXX_USE_CXX11_ABI=1" IN_LIST houdini_interface_compile_options) - set(HOUDINI_CXX11_ABI 1) - endif() + if(NOT QUERIED_GCC_CXX11_ABI_SUCCESS) + string(FIND ${_GCC_CXX11_ABI} "_GLIBCXX_USE_CXX11_ABI 0" GCC_OLD_CXX11_ABI) + string(FIND ${_GCC_CXX11_ABI} "_GLIBCXX_USE_CXX11_ABI 1" GCC_NEW_CXX11_ABI) + if(NOT (${GCC_OLD_CXX11_ABI} EQUAL -1)) + set(GLIBCXX_USE_CXX11_ABI 0) endif() + if(NOT (${GCC_NEW_CXX11_ABI} EQUAL -1)) + set(GLIBCXX_USE_CXX11_ABI 1) + endif() + endif() - message(STATUS " GCC CXX11 ABI : ${GLIBCXX_USE_CXX11_ABI}") - message(STATUS " Houdini CXX11 ABI : ${HOUDINI_CXX11_ABI}") + # Try and query the Houdini CXX11 ABI. Allow it to be provided by users to + # override this logic should Houdini's CMake ever change - if(${HOUDINI_CXX11_ABI} STREQUAL "UNKNOWN") - message(WARNING "Unable to determine Houdini CXX11 ABI. Assuming newer ABI " - "has been used.") + if(NOT DEFINED HOUDINI_CXX11_ABI) + get_target_property(houdini_interface_compile_options + Houdini INTERFACE_COMPILE_OPTIONS) + set(HOUDINI_CXX11_ABI "UNKNOWN") + if("-D_GLIBCXX_USE_CXX11_ABI=0" IN_LIST houdini_interface_compile_options) + set(HOUDINI_CXX11_ABI 0) + elseif("-D_GLIBCXX_USE_CXX11_ABI=1" IN_LIST houdini_interface_compile_options) set(HOUDINI_CXX11_ABI 1) endif() + endif() - if(${GLIBCXX_USE_CXX11_ABI} EQUAL ${HOUDINI_CXX11_ABI}) - message(STATUS " Current CXX11 ABI matches Houdini configuration " - "(_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI}).") - else() - message(WARNING "A potential mismatch was detected between the CXX11 ABI " - "of GCC and Houdini. The following ABI configuration will be used: " - "-D_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI}. See: " - "https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html and " - "https://vfxplatform.com/#footnote-gcc6 for more information.") - endif() + message(STATUS " GNU CXX11 ABI : ${GLIBCXX_USE_CXX11_ABI}") + message(STATUS " Houdini CXX11 ABI : ${HOUDINI_CXX11_ABI}") + + if(${HOUDINI_CXX11_ABI} STREQUAL "UNKNOWN") + message(WARNING "Unable to determine Houdini CXX11 ABI. Assuming newer ABI " + "has been used.") + set(HOUDINI_CXX11_ABI 1) + endif() - add_definitions(-D_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI}) + if(${GLIBCXX_USE_CXX11_ABI} EQUAL ${HOUDINI_CXX11_ABI}) + message(STATUS " Current CXX11 ABI matches Houdini configuration " + "(_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI}).") + else() + message(WARNING "A potential mismatch was detected between the CXX11 ABI " + "of libstdc++ and Houdini. The following ABI configuration will be used: " + "-D_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI}. See: " + "https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html and " + "https://vfxplatform.com/#footnote-gcc6 for more information.") endif() + + add_definitions(-D_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI}) endif() +