fVDB: Fix missing fvdb.utils.data modules

Expose various 'ELU non-linearities Integer JaggedReduce Updated build TORCH_CUDA_ARCH_LIST for supported CUDA compute architectures, fixed Docker dev and CI images Updated REAME's env and build instructions Signed-off-by: Jonathan Swartz <[email protected]>
AcademySoftwareFoundation · Dec 3, 2024 · f7b18d9 · f7b18d9
1 parent f1d9e2b
commit f7b18d9
Show file tree

Hide file tree

Showing 33 changed files with 2,958 additions and 81 deletions.
diff --git a/fvdb/.clang-format b/fvdb/.clang-format
@@ -77,7 +77,7 @@ Cpp11BracedListStyle: false
 IncludeBlocks: Preserve
 IncludeIsMainRegex: "$"
 IncludeCategories:
-  - Regex: '^<.*\.h>'
+  - Regex: '^<.*\.(h|cuh)>'
     Priority: 1
   - Regex: ".*"
     Priority: 2

diff --git a/fvdb/.github/workflows/tests.yml b/fvdb/.github/workflows/tests.yml
@@ -10,6 +10,11 @@ on:
       - 'notebooks/**'
       - 'scripts/**'
 
+# Allow subsequent pushes to the same PR or REF to cancel any previous jobs.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: write
   deployments: write
@@ -21,53 +26,64 @@ jobs:
     runs-on:
       - self-hosted
     container:
-      image: aswf/ci-openvdb:2023-clang15.0
+      image: aswf/ci-openvdb:2024
+      env:
+        PYTHONPATH: ""
+      options: --rm
+    defaults:
+      run:
+        shell: bash -el {0}
     steps:
       - uses: actions/checkout@v4
 
       - name: Set up fvdb_build Conda env
         uses: conda-incubator/setup-miniconda@v3
         with:
-          miniconda-version: "latest"
-          channels: conda-forge
-          channel-priority: strict
+          miniforge-version: latest
+          conda-remove-defaults: "true"
           activate-environment: fvdb_build
           environment-file: env/build_environment.yml
 
 
       - name: Buid fvdb
         run: |
-          TORCH_CUDA_ARCH_LIST="7.0;7.5;8.0;8.6+PTX" MAX_JOBS=$(($(nproc) < $(free -g | awk '/^Mem:/{jobs=int($7/2.5); if(jobs<1) jobs=1; print jobs}') ? $(nproc) : $(free -g | awk '/^Mem:/{jobs=int($7/2.5); if(jobs<1) jobs=1; print jobs}'))) python setup.py bdist_wheel --dist-dir=dist
-        shell:
-          bash -el {0}
+          TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9+PTX" MAX_JOBS=$(($(nproc) < $(free -g | awk '/^Mem:/{jobs=int($7/2.5); if(jobs<1) jobs=1; print jobs}') ? $(nproc) : $(free -g | awk '/^Mem:/{jobs=int($7/2.5); if(jobs<1) jobs=1; print jobs}'))) conda run --no-capture-output -n fvdb_build python setup.py bdist_wheel --dist-dir=dist
 
       - name: Upload package
         uses: actions/upload-artifact@v4
         with:
             name: fvdb-test-package
             path: dist/*.whl
+            retention-days: 2
 
-      - name: Clean Conda
+      - name: Cleanup
+        if: always()
         run: |
-          conda clean -pty
-        shell:
-          bash -el {0}
+          echo "Cleaning up /__w/_temp directory"
+          sudo rm -rf /__w/_temp/*
+          echo "Cleanup completed"
+
 
   fvdb-unit-test:
     needs: [fvdb-build]
     name: fVDB Unit Tests
     runs-on:
       - self-hosted
     container:
-      image: aswf/ci-openvdb:2023-clang15.0
+      image: aswf/ci-openvdb:2024
+      env:
+        PYTHONPATH: ""
+      options: --rm
+    defaults:
+      run:
+        shell: bash -el {0}
     steps:
       - uses: actions/checkout@v4
       - name: Set up fvdb_test Conda env
         uses: conda-incubator/setup-miniconda@v3
         with:
-          miniconda-version: "latest"
-          channels: conda-forge
-          channel-priority: strict
+          miniforge-version: latest
+          conda-remove-defaults: "true"
           activate-environment: fvdb_test
           environment-file: env/test_environment.yml
 
@@ -81,38 +97,84 @@ jobs:
         run: |
             conda activate fvdb_test
             pip install ./dist/*.whl
-        shell:
-              bash -el {0}
 
       - name: Run tests
         run: |
             cd tests;
             pytest -v unit
-        shell:
-              bash -el {0}
 
-      - name: Clean Conda
+      - name: Cleanup
+        if: always()
         run: |
-            conda clean -pty
-        shell:
-            bash -el {0}
+          echo "Cleaning up /__w/_temp directory"
+          sudo rm -rf /__w/_temp/*
+          echo "Cleanup completed"
+
+  fvdb-docs-test:
+    needs: [fvdb-build]
+    name: fVDB Documentation Tests
+    runs-on:
+      - self-hosted
+    container:
+      image: aswf/ci-openvdb:2024
+      env:
+        PYTHONPATH: ""
+      options: --rm
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up fvdb_test Conda env
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          conda-remove-defaults: "true"
+          activate-environment: fvdb_test
+          environment-file: env/test_environment.yml
+
+      - name: Download package
+        uses: actions/download-artifact@v4
+        with:
+            name: fvdb-test-package
+            path: ./dist
+
+      - name: Install package
+        run: |
+            conda activate fvdb_test
+            pip install ./dist/*.whl
+
+      - name: Run tests
+        run: |
+            cd .. && pytest --markdown-docs fvdb/docs
+
+      - name: Cleanup
+        if: always()
+        run: |
+          echo "Cleaning up /__w/_temp directory"
+          sudo rm -rf /__w/_temp/*
+          echo "Cleanup completed"
 
   fvdb-benchmarks:
     needs: [fvdb-build]
     name: fVDB Continuous Benchmarking
     runs-on:
       - self-hosted
     container:
-      image: aswf/ci-openvdb:2023-clang15.0
-
+      image: aswf/ci-openvdb:2024
+      env:
+        PYTHONPATH: ""
+      options: --rm
+    defaults:
+      run:
+        shell: bash -el {0}
     steps:
       - uses: actions/checkout@v4
       - name: Set up fvdb_test Conda env
         uses: conda-incubator/setup-miniconda@v3
         with:
-          miniconda-version: "latest"
-          channels: conda-forge
-          channel-priority: strict
+          miniforge-version: latest
+          conda-remove-defaults: "true"
           activate-environment: fvdb_test
           environment-file: env/test_environment.yml
 
@@ -126,21 +188,15 @@ jobs:
         run: |
             conda activate fvdb_test
             pip install ./dist/*.whl
-        shell:
-          bash -el {0}
 
       - name: Disable git ownership verification
         run: |
           git config --global --add safe.directory "$(pwd)"
-        shell:
-          bash -el {0}
 
       - name: Run benchmarks
         run: |
           cd tests;
           pytest benchmark --benchmark-json benchmark/output.json
-        shell:
-          bash -el {0}
 
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1
@@ -157,8 +213,9 @@ jobs:
             fail-on-alert: true
             alert-comment-cc-users: '@swahtz'
 
-      - name: Clean Conda
+      - name: Cleanup
+        if: always()
         run: |
-            conda clean -pty
-        shell:
-            bash -el {0}
+          echo "Cleaning up /__w/_temp directory"
+          sudo rm -rf /__w/_temp/*
+          echo "Cleanup completed"
diff --git a/fvdb/.gitignore b/fvdb/.gitignore
@@ -31,23 +31,18 @@ apex/
 *.npy
 *.o
 *.h5
-data/lego/transforms_test.json
-data/lego/transforms_train.json
-data/fegr/*
-data/fegr2/*
 docs/_build
 fvdb/include/
 trash
 scratch
-data/output_hi.0.nvdb
-data/output_mid.0.nvdb
 _build
 docs/build
 *tmp*.py
 *.whl
 tests/benchmark/data
 releases/
-data/
+/data/
+/data/*
 !tests/data
 .vscode/launch.json
 lightning_logs

diff --git a/fvdb/.vscode/c_cpp_properties.json b/fvdb/.vscode/c_cpp_properties.json
@@ -7,6 +7,7 @@
                 "${env:CONDA_PREFIX}/envs/fvdb/include/cuda",
                 "${workspaceFolder}/src",
                 "${workspaceFolder}/external/openvdb/nanovdb",
+                "${workspaceFolder}/external/glm",
                 "${workspaceFolder}/external/cudnn_fe/include",
                 "${workspaceFolder}/external/cutlass/include",
                 "${env:CONDA_PREFIX}/envs/fvdb/include/python3.10",

diff --git a/fvdb/.vscode/settings.json b/fvdb/.vscode/settings.json
@@ -135,19 +135,6 @@
         },
         "editor.rulers": [ 120 ]
     },
-    "python.linting.pylintArgs": [
-        "--disable", "C0114",
-        "--disable", "C0115",
-        "--disable", "C0116",
-        "--disable", "C0103",
-        "--disable", "C0209",
-        "--disable", "C0301",
-        "--disable", "R0914",
-        "--disable", "R0913",
-        "--disable", "W0622",
-        "--disable", "E1102",
-        "--generated-members=numpy.* ,torch.*"
-    ],
     "python.analysis.typeCheckingMode": "basic",
     "python.testing.unittestArgs": [
         "-v", "-s", "."
@@ -158,9 +145,13 @@
         "--target-version=py311",
         "--line-length=120"
     ],
+    "python.analysis.diagnosticSeverityOverrides": {
+        "reportPrivateImportUsage": "none",
+    },
     "isort.args": [
         "--profile", "black"
     ],
     "json.format.keepLines": true,
     "C_Cpp.formatting": "clangFormat",
+    "C_Cpp.clang_format_path": "${env:CONDA_PREFIX}/envs/fvdb/bin/clang-format-18",
 }
diff --git a/fvdb/Dockerfile b/fvdb/Dockerfile
@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/pytorch:24.04-py3
+FROM nvcr.io/nvidia/pytorch:24.05-py3
 
 ARG MODE=production
 RUN echo "Building fVDB container in $MODE mode"
@@ -12,6 +12,6 @@ RUN  pip install --no-cache-dir -r env/build_requirements.txt
 
 RUN if [ "$MODE" = "production" ]; then \
      MAX_JOBS=$(free -g | awk '/^Mem:/{jobs=int($4/2.5); if(jobs<1) jobs=1; print jobs}')  \
-     TORCH_CUDA_ARCH_LIST="7.0;7.5;8.0;8.6+PTX" \
+     TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9+PTX" \
      python setup.py install; \
     fi