Added Houdini 20 CI for old and new CXX ABI and improved CXX11 ABI logic

Signed-off-by: Nick Avramoussis <[email protected]>
AcademySoftwareFoundation · Dec 6, 2023 · 2e6ae06 · 2e6ae06
1 parent 9153c12
commit 2e6ae06
Show file tree

Hide file tree

Showing 4 changed files with 154 additions and 107 deletions.
diff --git a/.github/workflows/houdini.yml b/.github/workflows/houdini.yml
@@ -64,7 +64,7 @@ jobs:
       ${{ needs.checksecret.outputs.HOUDINI_SECRETS == 'true' ||
           github.repository_owner == 'AcademySoftwareFoundation' }}
     runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
-    name: hou:${{ matrix.config.hou }}-vfx:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }}
+    name: hou:${{ matrix.config.hou_hash }}-vfx:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }}
     container:
       image: aswf/ci-base:${{ matrix.config.image }}
     env:
@@ -73,10 +73,11 @@ jobs:
     strategy:
       matrix:
         config:
-          #- { cxx: clang++, image: '2022', hou: '20_0', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
-          - { cxx: clang++, image: '2021', hou: '19_5', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
-          #- { cxx: clang++, image: '2022', hou: '20_0', build: 'Debug',   components: 'core,hou' }
-          #- { cxx: g++,     image: '2022', hou: '20_0', build: 'Release', components: 'core,hou' }
+          - { cxx: clang++, image: '2022', hou_hash: '20_0-newabi', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
+          - { cxx: clang++, image: '2022', hou_hash: '20_0-oldabi', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
+          - { cxx: clang++, image: '2021', hou_hash: '19_5',        build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
+          - { cxx: clang++, image: '2022', hou_hash: '20_0-newabi', build: 'Debug',   components: 'core,hou' }
+          - { cxx: g++,     image: '2022', hou_hash: '20_0-newabi', build: 'Release', components: 'core,hou' }
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
@@ -93,14 +94,14 @@ jobs:
       uses: actions/cache@v3
       with:
         path: /tmp/ccache
-        key: linux-vfx-hou${{ matrix.config.hou }}-${{ matrix.config.image }}-${{ matrix.config.cxx }}-${{ steps.timestamp.outputs.timestamp }}
-        restore-keys: linux-vfx-hou${{ matrix.config.hou }}-${{ matrix.config.image }}-${{ matrix.config.cxx }}-
+        key: linux-vfx-hou${{ matrix.config.hou_hash }}-${{ matrix.config.image }}-${{ matrix.config.cxx }}-${{ steps.timestamp.outputs.timestamp }}
+        restore-keys: linux-vfx-hou${{ matrix.config.hou_hash }}-${{ matrix.config.image }}-${{ matrix.config.cxx }}-
     - name: fetch_houdini
       uses: actions/cache/restore@v3
       with:
         path: hou
-        key: dummy-houdini${{ matrix.config.hou }}-${{ steps.timestamp.outputs.timestamp }}
-        restore-keys: vdb-v5-houdini${{ matrix.config.hou }}-
+        key: dummy-houdini${{ matrix.config.hou_hash }}-${{ steps.timestamp.outputs.timestamp }}
+        restore-keys: vdb-v5-houdini${{ matrix.config.hou_hash }}-
     - name: validate_houdini
       run: test -f "hou/hou.tar.gz"
       # Make sure the cache is copied, not moved, as the cache action always posts the cache.
@@ -113,10 +114,17 @@ jobs:
         cd $HOME/houdini_install && tar -xzf hou.tar.gz && cd -
     - name: build
       run: |
-        export HFS="$HOME/houdini_install/hou"
-        export HDSO="${HFS}/dsolib"
-        export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib64:${HDSO}"
-        ./ci/build.sh -v --build-type=Release --components="core,hou" --cargs=\"-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DDISABLE_CMAKE_SEARCH_PATHS=ON -DOPENVDB_BUILD_HOUDINI_ABITESTS=OFF -DOPENVDB_HOUDINI_INSTALL_PREFIX=/tmp\ -DTBB_INCLUDEDIR=/usr/local/include -DTBB_LIBRARYDIR=/usr/local/lib\"
+        export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib64"
+        ./ci/build.sh -v \
+          --build-type=Release \
+          --components="${{ matrix.config.components }}" \
+          --cargs=\" \
+            -DHOUDINI_ROOT=$HOME/houdini_install/hou \
+            -DOPENVDB_BUILD_HOUDINI_ABITESTS=OFF \
+            -DOPENVDB_HOUDINI_INSTALL_PREFIX=/tmp \
+            -DDISABLE_CMAKE_SEARCH_PATHS=ON \
+            -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON \
+            \"
     - name: test
       run: cd build && ctest -V
     # Keep ccache light by stripping out any caches not accessed in the last day

diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml
@@ -59,16 +59,17 @@ jobs:
       github.event.inputs.type == 'all' ||
       github.event.inputs.type == 'houdini')
     runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
-    name: linux-houdini:${{ matrix.config.houdini_version }}
+    name: linux-houdini:${{ matrix.config.hou_hash }}
     env:
       CXX: clang++
       HOUDINI_CLIENT_ID: ${{ secrets.HOUDINI_CLIENT_ID }}
       HOUDINI_SECRET_KEY: ${{ secrets.HOUDINI_SECRET_KEY }}
     strategy:
       matrix:
         config:
-          - { houdini_version: '19.5', houdini_version_str: '19_5' }
-          #- { houdini_version: '20.0', houdini_version_str: '20_0' }
+          - { houdini_version: '19.5', platform: 'linux_x86_64_gcc9.3',  hou_hash: '19_5' }
+          - { houdini_version: '20.0', platform: 'linux_x86_64_gcc9.3',  hou_hash: '20_0-oldabi' }
+          - { houdini_version: '20.0', platform: 'linux_x86_64_gcc11.2', hou_hash: '20_0-newabi' }
       fail-fast: false
     container:
       image: aswf/ci-base:2023
@@ -88,7 +89,7 @@ jobs:
       id: timestamp
       run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
     - name: download_houdini
-      run: ./ci/download_houdini.sh ${{ matrix.config.houdini_version }} ON
+      run: ./ci/download_houdini.sh ${{ matrix.config.houdini_version }} ${{ matrix.config.platform }} --prod
     - name: install_houdini
       run: |
         mkdir $HOME/houdini_install
@@ -98,7 +99,7 @@ jobs:
       uses: actions/cache/save@v3
       with:
         path: hou
-        key: vdb-v5-houdini${{ matrix.config.houdini_version_str }}-${{ steps.timestamp.outputs.timestamp }}
+        key: vdb-v5-houdini${{ matrix.config.hou_hash }}-${{ steps.timestamp.outputs.timestamp }}
 
   #############################################################################
   ########################### Core Library Extras #############################

diff --git a/ci/download_houdini.py b/ci/download_houdini.py
@@ -1,4 +1,4 @@
-#!/usr/local/bin/python
+#!/usr/bin/env python3
 #
 # Copyright Contributors to the OpenVDB Project
 # SPDX-License-Identifier: MPL-2.0
@@ -19,16 +19,16 @@
 import requests
 import hashlib
 import os
+import argparse
+import copy
 
-# this argument is for the major.minor version of Houdini to download (such as 15.0, 15.5, 16.0)
-version = sys.argv[1]
-only_production = True if sys.argv[2] == 'ON' else False
-user_client_id = os.getenv('HOUDINI_CLIENT_ID')
-user_client_secret_key = os.getenv('HOUDINI_SECRET_KEY')
-
-if not re.match('[0-9][0-9]\.[0-9]$', version):
-    raise IOError('Invalid Houdini Version "%s", expecting in the form "major.minor" such as "16.0"' % version)
-
+# For progress bar printing
+try:
+    from tqdm import tqdm
+    has_tqdm = True
+except:
+    has_tqdm = False
+    pass
 
 # Code that provides convenient Python wrappers to call into the API:
 
@@ -138,33 +138,72 @@ def __init__(self, http_code, message):
         self.http_code = http_code
 
 
-service = service(
-        access_token_url="https://www.sidefx.com/oauth2/application_token",
-        client_id=user_client_id,
-        client_secret_key=user_client_secret_key,
-        endpoint_url="https://www.sidefx.com/api/",
-    )
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description='Download a Houdini Installation')
+    parser.add_argument('version', type=str, help='Major.Minor version of Houdini to download')
+    parser.add_argument('platform', type=str, help='Platform target')
+    parser.add_argument('--prod', action='store_true', help='Only download production builds')
+    parser.add_argument('--list', action='store_true', help='Just list the available builds and exit.')
+    args = parser.parse_args()
+
+    version = args.version
+    platform = args.platform
+    only_production = args.prod
+
+    user_client_id = os.getenv('HOUDINI_CLIENT_ID')
+    user_client_secret_key = os.getenv('HOUDINI_SECRET_KEY')
+
+    if not re.match('[0-9][0-9]\.[0-9]$', version):
+        raise IOError('Invalid Houdini Version "%s", expecting in the form "major.minor" such as "16.0"' % version)
 
-releases_list = service.download.get_daily_builds_list(
-        product='houdini', version=version, platform='linux', only_production=only_production)
+    service = service(
+            access_token_url="https://www.sidefx.com/oauth2/application_token",
+            client_id=user_client_id,
+            client_secret_key=user_client_secret_key,
+            endpoint_url="https://www.sidefx.com/api/",
+        )
 
-latest_release = service.download.get_daily_build_download(
-        product='houdini', version=version, platform='linux', build=releases_list[0]['build'])
+    releases_list = service.download.get_daily_builds_list(
+            product='houdini', version=version, platform=platform, only_production=only_production)
 
-# Download the file as hou.tar.gz
-local_filename = 'hou.tar.gz'
-response = requests.get(latest_release['download_url'], stream=True)
-if response.status_code == 200:
-    with open(local_filename, 'wb') as f:
+    print('Available builds:')
+    for rel in releases_list:
+        rel = copy.deepcopy(rel)
+        if 'third_party_libraries' in rel:
+            # Don't print these
+            del rel['third_party_libraries']
+        print(rel)
+
+    if args.list:
+        sys.exit(0)
+
+    print('Selecting build: ' + releases_list[0]['build'])
+
+    latest_release = service.download.get_daily_build_download(
+            product='houdini', version=version, platform=platform, build=releases_list[0]['build'])
+
+    # Download the file as hou.tar.gz
+    local_filename = 'hou.tar.gz'
+    response = requests.get(latest_release['download_url'], stream=True)
+    if response.status_code == 200:
         response.raw.decode_content = True
-        shutil.copyfileobj(response.raw, f)
-else:
-    raise Exception('Error downloading file!')
-
-# Verify the file checksum is matching
-file_hash = hashlib.md5()
-with open(local_filename, 'rb') as f:
-    for chunk in iter(lambda: f.read(4096), b''):
-        file_hash.update(chunk)
-if file_hash.hexdigest() != latest_release['hash']:
-    raise Exception('Checksum does not match!')
+        if has_tqdm:
+            file_size = int(response.headers.get('Content-Length', 0))
+            desc = "(Unknown total file size)" if file_size == 0 else ""
+            with tqdm.wrapattr(response.raw, "read", total=file_size, desc=desc) as r_raw:
+                with open(local_filename, 'wb') as f:
+                    shutil.copyfileobj(r_raw, f)
+        else:
+            with open(local_filename, 'wb') as f:
+                shutil.copyfileobj(response.raw, f)
+    else:
+        raise Exception('Error downloading file!')
+
+    # Verify the file checksum is matching
+    file_hash = hashlib.md5()
+    with open(local_filename, 'rb') as f:
+        for chunk in iter(lambda: f.read(4096), b''):
+            file_hash.update(chunk)
+    if file_hash.hexdigest() != latest_release['hash']:
+        raise Exception('Checksum does not match!')
diff --git a/cmake/OpenVDBHoudiniSetup.cmake b/cmake/OpenVDBHoudiniSetup.cmake
@@ -335,68 +335,67 @@ if(NOT OPENVDB_HOUDINI_ABI)
 endif()
 
 # ------------------------------------------------------------------------
-#  Configure GCC CXX11 ABI
+#  Configure libstc++ CXX11 ABI
 # ------------------------------------------------------------------------
 
-if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  if((CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 5.1) OR
-     (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 5.1))
-    message(STATUS "GCC >= 5.1 detected. Configuring GCC CXX11 ABI for Houdini compatibility...")
-
-    execute_process(COMMAND echo "#include <string>"
-      COMMAND ${CMAKE_CXX_COMPILER} "-x" "c++" "-E" "-dM" "-"
-      COMMAND grep "-F" "_GLIBCXX_USE_CXX11_ABI"
-      TIMEOUT 10
-      RESULT_VARIABLE QUERIED_GCC_CXX11_ABI_SUCCESS
-      OUTPUT_VARIABLE _GCC_CXX11_ABI)
-
-    set(GLIBCXX_USE_CXX11_ABI "UNKNOWN")
-
-    if(NOT QUERIED_GCC_CXX11_ABI_SUCCESS)
-      string(FIND ${_GCC_CXX11_ABI} "_GLIBCXX_USE_CXX11_ABI 0" GCC_OLD_CXX11_ABI)
-      string(FIND ${_GCC_CXX11_ABI} "_GLIBCXX_USE_CXX11_ABI 1" GCC_NEW_CXX11_ABI)
-      if(NOT (${GCC_OLD_CXX11_ABI} EQUAL -1))
-        set(GLIBCXX_USE_CXX11_ABI 0)
-      endif()
-      if(NOT (${GCC_NEW_CXX11_ABI} EQUAL -1))
-        set(GLIBCXX_USE_CXX11_ABI 1)
-      endif()
-    endif()
+if(UNIX AND NOT APPLE)
+  # Assume we're using libstdc++
+  message(STATUS "Configuring CXX11 ABI for Houdini compatibility...")
+
+  execute_process(COMMAND echo "#include <string>"
+    COMMAND ${CMAKE_CXX_COMPILER} "-x" "c++" "-E" "-dM" "-"
+    COMMAND grep "-F" "_GLIBCXX_USE_CXX11_ABI"
+    TIMEOUT 10
+    RESULT_VARIABLE QUERIED_GCC_CXX11_ABI_SUCCESS
+    OUTPUT_VARIABLE _GCC_CXX11_ABI)
+
+  set(GLIBCXX_USE_CXX11_ABI "UNKNOWN")
 
-    # Try and query the Houdini CXX11 ABI. Allow it to be provided by users to
-    # override this logic should Houdini's CMake ever change
-
-    if(NOT DEFINED HOUDINI_CXX11_ABI)
-      get_target_property(houdini_interface_compile_options
-        Houdini INTERFACE_COMPILE_OPTIONS)
-      set(HOUDINI_CXX11_ABI "UNKNOWN")
-      if("-D_GLIBCXX_USE_CXX11_ABI=0" IN_LIST houdini_interface_compile_options)
-        set(HOUDINI_CXX11_ABI 0)
-      elseif("-D_GLIBCXX_USE_CXX11_ABI=1" IN_LIST houdini_interface_compile_options)
-        set(HOUDINI_CXX11_ABI 1)
-      endif()
+  if(NOT QUERIED_GCC_CXX11_ABI_SUCCESS)
+    string(FIND ${_GCC_CXX11_ABI} "_GLIBCXX_USE_CXX11_ABI 0" GCC_OLD_CXX11_ABI)
+    string(FIND ${_GCC_CXX11_ABI} "_GLIBCXX_USE_CXX11_ABI 1" GCC_NEW_CXX11_ABI)
+    if(NOT (${GCC_OLD_CXX11_ABI} EQUAL -1))
+      set(GLIBCXX_USE_CXX11_ABI 0)
     endif()
+    if(NOT (${GCC_NEW_CXX11_ABI} EQUAL -1))
+      set(GLIBCXX_USE_CXX11_ABI 1)
+    endif()
+  endif()
 
-    message(STATUS "  GCC CXX11 ABI     : ${GLIBCXX_USE_CXX11_ABI}")
-    message(STATUS "  Houdini CXX11 ABI : ${HOUDINI_CXX11_ABI}")
+  # Try and query the Houdini CXX11 ABI. Allow it to be provided by users to
+  # override this logic should Houdini's CMake ever change
 
-    if(${HOUDINI_CXX11_ABI} STREQUAL "UNKNOWN")
-      message(WARNING "Unable to determine Houdini CXX11 ABI. Assuming newer ABI "
-        "has been used.")
+  if(NOT DEFINED HOUDINI_CXX11_ABI)
+    get_target_property(houdini_interface_compile_options
+      Houdini INTERFACE_COMPILE_OPTIONS)
+    set(HOUDINI_CXX11_ABI "UNKNOWN")
+    if("-D_GLIBCXX_USE_CXX11_ABI=0" IN_LIST houdini_interface_compile_options)
+      set(HOUDINI_CXX11_ABI 0)
+    elseif("-D_GLIBCXX_USE_CXX11_ABI=1" IN_LIST houdini_interface_compile_options)
       set(HOUDINI_CXX11_ABI 1)
     endif()
+  endif()
 
-    if(${GLIBCXX_USE_CXX11_ABI} EQUAL ${HOUDINI_CXX11_ABI})
-      message(STATUS "  Current CXX11 ABI matches Houdini configuration "
-        "(_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI}).")
-    else()
-      message(WARNING "A potential mismatch was detected between the CXX11 ABI "
-        "of GCC and Houdini. The following ABI configuration will be used: "
-        "-D_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI}. See: "
-        "https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html and "
-        "https://vfxplatform.com/#footnote-gcc6 for more information.")
-    endif()
+  message(STATUS "  GNU CXX11 ABI     : ${GLIBCXX_USE_CXX11_ABI}")
+  message(STATUS "  Houdini CXX11 ABI : ${HOUDINI_CXX11_ABI}")
+
+  if(${HOUDINI_CXX11_ABI} STREQUAL "UNKNOWN")
+    message(WARNING "Unable to determine Houdini CXX11 ABI. Assuming newer ABI "
+      "has been used.")
+    set(HOUDINI_CXX11_ABI 1)
+  endif()
 
-    add_definitions(-D_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI})
+  if(${GLIBCXX_USE_CXX11_ABI} EQUAL ${HOUDINI_CXX11_ABI})
+    message(STATUS "  Current CXX11 ABI matches Houdini configuration "
+      "(_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI}).")
+  else()
+    message(WARNING "A potential mismatch was detected between the CXX11 ABI "
+      "of libstdc++ and Houdini. The following ABI configuration will be used: "
+      "-D_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI}. See: "
+      "https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html and "
+      "https://vfxplatform.com/#footnote-gcc6 for more information.")
   endif()
+
+  add_definitions(-D_GLIBCXX_USE_CXX11_ABI=${HOUDINI_CXX11_ABI})
 endif()
+