3D gaussians .splat reader (f3d-app#1248)

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
 skip = ./testing/**/*,./.git/**/*,./**/*.patch,./external/cxxopts/cxxopts.hpp,./external/nlohmann_json/nlohmann/json.hpp
-ignore-words-list=nnumber,unknwn,dota,modle
+ignore-words-list=nnumber,unknwn,dota,modle,inout

.cppcheck.supp

@@ -16,3 +16,4 @@ preprocessorErrorDirective:plugins/draco/module/vtkF3DDracoReader.cxx
 unknownMacro:library/VTKExtensions/Applicative/vtkF3DObjectFactory.cxx
 unusedVariable:*factory.cxx*
 constParameter:library/src/image.cxx
+invalidPointerCast:plugins/native/module/vtkF3DSplatReader.cxx

.github/actions/mesa-install-bin/action.yml

@@ -18,9 +18,10 @@ runs:
       run: |
         mkdir mesa
         cd mesa
-        curl.exe -L --output mesa.7z --url https://github.com/pal1000/mesa-dist-win/releases/download/22.0.1/mesa3d-22.0.1-release-msvc.7z
+        curl.exe -L --output mesa.7z --url https://github.com/pal1000/mesa-dist-win/releases/download/23.3.5/mesa3d-23.3.5-release-msvc.7z
         C:\'Program Files'\7-Zip\7z.exe x mesa.7z
         # A * is added next line to force Get-ChildItem to look for directory within the path
         Get-ChildItem -Directory ${{inputs.path}}* | ForEach-Object { Copy-Item -Path .\x64\opengl32.dll, .\x64\libglapi.dll, .\x64\libgallium_wgl.dll -Destination $_ }
+        echo "GALLIUM_DRIVER=llvmpipe"| Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
         cd ..
         rm .\mesa -r -force

.github/actions/vtk_commit_sha

@@ -1 +1 @@
-a4cb9c6c90c18a31e225aa4a18df6a591bf8cd3b
+be69fa3a50de292857e69702885576ac6f114805

.lsan.supp

@@ -19,6 +19,7 @@ leak:TKernel
 leak:libtbb
 
 # Potential mesa/VTK leak with incomplete callstack
-# forces us to hide all leaks from the libf3dSDKTests
+# forces us to hide all leaks from the tests using a render window
 # https://gitlab.kitware.com/vtk/vtk/-/issues/18504
 leak:libf3dSDKTests
+leak:VTKExtensionsRenderingTests

application/F3DOptionsParser.cxx

@@ -341,6 +341,7 @@ void ConfigurationOptions::GetOptions(F3DAppOptions& appOptions, f3d::options& o
 
     auto grp2 = cxxOptions.add_options("Material");
     this->DeclareOption(grp2, "point-sprites", "o", "Show sphere sprites instead of geometry", options.getAsBoolRef("model.point-sprites.enable"), HasDefault::YES, MayHaveConfig::YES);
+    this->DeclareOption(grp2, "point-type", "", "Point splat type when showing point sprites", options.getAsStringRef("render.splat-type"), HasDefault::YES, MayHaveConfig::YES, "<sphere|gaussian>");
     this->DeclareOption(grp2, "point-size", "", "Point size when showing vertices or point sprites", options.getAsDoubleRef("render.point-size"), HasDefault::YES, MayHaveConfig::YES, "<size>");
     this->DeclareOption(grp2, "line-width", "", "Line width when showing edges", options.getAsDoubleRef("render.line-width"), HasDefault::YES, MayHaveConfig::YES, "<width>");
     this->DeclareOption(grp2, "color", "", "Solid color", options.getAsDoubleVectorRef("model.color.rgb"), HasDefault::YES, MayHaveConfig::YES, "<R,G,B>");

application/testing/CMakeLists.txt

@@ -212,6 +212,13 @@ else()
   endif()
 endif()
 
+# Needs splat sorting with compute shaders
+if(VTK_VERSION VERSION_GREATER_EQUAL 9.3.20240203)
+  if(NOT APPLE) # MacOS does not support compute shaders
+    f3d_test(NAME Test3DGaussiansSplatting DATA small.splat ARGS -osy --up=-Y --point-size=1 --point-type=gaussian --camera-position=-3.6,0.5,-4.2)
+  endif()
+endif()
+
 if (NOT APPLE)
   # This test is broken on apple because of #792
   f3d_test(NAME TestScalarsCell DATA f3d.vtp ARGS --scalars --cells --comp=-2 --up=+Z DEFAULT_LIGHTS)

doc/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## Ongoing development
 
+For F3D users:
+- Added a new option `--point-type` used to specify how to display points
+- Add support for 3D Gaussians Splatting in binary .splat format
+
+For libf3d users:
+- Added a new option `render.splat-type` used to specify how to display points (only if `model.point-sprites.enable` is true)
+
 ## v2.3.0
 
 For F3D users:

doc/GALLERY.md

@@ -15,6 +15,10 @@ Images and videos displayed below use public datasets, you can download them [he
 
 *Animated scientific visualization rendering*: `f3d can.ex2 -xtgans --up=+Z --scalars=VEL`
 
+<video src='https://media.githubusercontent.com/media/f3d-app/f3d-media/main/media/counter.webm' autoplay="autoplay" loop="loop" width="700"></video>
+
+*3D Gaussians Splatting*: `f3d counter.splat --point-size=1 --point-type=gaussian -soynxz --up=-Y --camera-position=0,1,-5.2 --camera-focal-point=0,1,0`
+
 <img src="https://user-images.githubusercontent.com/3129530/194735272-5bcd3e7c-a333-41f5-8066-9b0bec9885e8.png" width="700">
 
 *Direct scalars rendering of a point cloud*: `f3d Carola_PointCloud.ply --point-size=0 --comp=-2 -so --up=+Z --hdri-skybox --hdri-ambient --hdri-file=venice_sunset_8k.hdr`

doc/libf3d/OPTIONS.md

@@ -66,6 +66,7 @@ render.effect.tone-mapping|bool<br>false<br>render|Enable generic filmic *Tone M
 render.line-width|double<br>1.0<br>render|Set the *width* of lines when showing edges.|\-\-line-width
 render.show-edges|bool<br>false<br>render|Show the *cell edges*|\-\-edges
 render.point-size|double<br>10.0<br>render|Set the *size* of points when showing vertices and point sprites.|\-\-point-size
+render.splat-type|string<br>sphere<br>render|Set the splat type when showing point sprites (can be `sphere` or `gaussian`).|\-\-point-type
 render.grid.enable|bool<br>false<br>render|Show *a grid* aligned with the horizontal (orthogonal to the Up direction) plane.|\-\-grid
 render.grid.absolute|bool<br>false<br>render|Position the grid at the *absolute origin* of the model's coordinate system instead of below the model.|\-\-grid
 render.grid.unit|double<br>0<br>render|Set the size of the *unit square* for the grid. If set to non-positive (the default) a suitable value will be automatically computed.|\-\-grid\-unit

doc/user/OPTIONS.md

@@ -45,6 +45,7 @@ Options|Default|Description
 ------|------|------
 -o, \-\-point-sprites||Show sphere *points sprites* instead of the geometry.
 \-\-point-size=\<size\>|10.0|Set the *size* of points when showing vertices and point sprites.
+\-\-point-type=\<sphere|gaussian\>|sphere|Set the splat type when showing point sprites.
 \-\-line-width=\<size\>|1.0|Set the *width* of lines when showing edges.
 \-\-color=\<R,G,B\>|1.0, 1.0, 1.0| Set a *color* on the geometry. Multiplied with the base color texture when present. <br>Requires a default scene.
 \-\-opacity=\<opacity\>|1.0|Set *opacity* on the geometry. Multiplied with the base color texture when present. <br>Requires a default scene. Usually used with Depth Peeling option.

library/VTKExtensions/Applicative/Testing/TestF3DObjectFactory.cxx

@@ -13,12 +13,20 @@
 #include <vtkTestUtilities.h>
 #include <vtkVersion.h>
 
+#if VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+#include "vtkF3DPointSplatMapper.h"
+#endif
+
 int TestF3DObjectFactory(int argc, char* argv[])
 {
   vtkNew<vtkF3DObjectFactory> factory;
   vtkObjectFactory::RegisterFactory(factory);
   vtkObjectFactory::SetAllEnableFlags(0, "vtkPolyDataMapper", "vtkOpenGLPolyDataMapper");
 
+#if VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+  vtkObjectFactory::SetAllEnableFlags(0, "vtkPointGaussianMapper", "vtkOpenGLPointGaussianMapper");
+#endif
+
   // Check factory utility methods
   if (strcmp(factory->GetVTKSourceVersion(), VTK_SOURCE_VERSION) != 0)
   {
@@ -40,6 +48,17 @@ int TestF3DObjectFactory(int argc, char* argv[])
     return EXIT_FAILURE;
   }
 
+#if VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+  vtkNew<vtkPointGaussianMapper> pointMapper;
+  pointMapper->Print(cout);
+  vtkF3DPointSplatMapper* pointMapperPtr = vtkF3DPointSplatMapper::SafeDownCast(pointMapper);
+  if (pointMapperPtr == nullptr)
+  {
+    std::cerr << "vtkF3DObjectFactory failed to create a vtkF3DPointSplatMapper" << std::endl;
+    return EXIT_FAILURE;
+  }
+#endif
+
   vtkNew<vtkOutputWindow> window;
 #if F3D_WINDOWS_GUI
   vtkF3DWin32OutputWindow* windowPtr = vtkF3DWin32OutputWindow::SafeDownCast(window);

library/VTKExtensions/Applicative/vtkF3DObjectFactory.cxx

@@ -1,28 +1,38 @@
 #include "vtkF3DObjectFactory.h"
 
+#include <vtkVersion.h>
+
 #include "vtkF3DConfigure.h"
 #include "vtkF3DPolyDataMapper.h"
 
+#if !defined(__ANDROID__) && !defined(__EMSCRIPTEN__) &&                                           \
+  VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+#include <vtkF3DPointSplatMapper.h>
+#endif
+
 #ifdef __ANDROID__
-#include "vtkF3DAndroidLogOutputWindow.h"
+#include <vtkF3DAndroidLogOutputWindow.h>
 #elif F3D_WINDOWS_GUI
-#include "vtkF3DWin32OutputWindow.h"
+#include <vtkF3DWin32OutputWindow.h>
 #else
-#include "vtkF3DConsoleOutputWindow.h"
+#include <vtkF3DConsoleOutputWindow.h>
 #endif
 
 #ifdef __EMSCRIPTEN__
-#include "vtkSDL2OpenGLRenderWindow.h"
-#include "vtkSDL2RenderWindowInteractor.h"
+#include <vtkSDL2OpenGLRenderWindow.h>
+#include <vtkSDL2RenderWindowInteractor.h>
 #endif
 
-#include <vtkVersion.h>
-
 vtkStandardNewMacro(vtkF3DObjectFactory);
 
 // Now create the functions to create overrides with.
 VTK_CREATE_CREATE_FUNCTION(vtkF3DPolyDataMapper)
 
+#if !defined(__ANDROID__) && !defined(__EMSCRIPTEN__) &&                                           \
+  VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+VTK_CREATE_CREATE_FUNCTION(vtkF3DPointSplatMapper)
+#endif
+
 #ifdef __ANDROID__
 VTK_CREATE_CREATE_FUNCTION(vtkF3DAndroidLogOutputWindow)
 #elif F3D_WINDOWS_GUI
@@ -42,6 +52,12 @@ vtkF3DObjectFactory::vtkF3DObjectFactory()
   this->RegisterOverride("vtkPolyDataMapper", "vtkF3DPolyDataMapper",
     "vtkPolyDataMapper override for F3D", 1, vtkObjectFactoryCreatevtkF3DPolyDataMapper);
 
+#if !defined(__ANDROID__) && !defined(__EMSCRIPTEN__) &&                                           \
+  VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+  this->RegisterOverride("vtkPointGaussianMapper", "vtkF3DPointSplatMapper",
+    "vtkPointGaussianMapper override for F3D", 1, vtkObjectFactoryCreatevtkF3DPointSplatMapper);
+#endif
+
 #ifdef __ANDROID__
   this->RegisterOverride("vtkOutputWindow", "vtkF3DAndroidLogOutputWindow",
     "vtkOutputWindow override for F3D", 1, vtkObjectFactoryCreatevtkF3DAndroidLogOutputWindow);

library/VTKExtensions/Readers/vtkF3DGenericImporter.cxx

@@ -34,17 +34,6 @@ struct ReaderPipeline
     this->GeometryActor->GetProperty()->SetInterpolationToPBR();
     this->VolumeMapper->SetRequestedRenderModeToGPU();
     this->PolyDataMapper->InterpolateScalarsBeforeMappingOn();
-    this->PointGaussianMapper->EmissiveOff();
-    this->PointGaussianMapper->SetSplatShaderCode(
-      "//VTK::Color::Impl\n"
-      "float dist = dot(offsetVCVSOutput.xy, offsetVCVSOutput.xy);\n"
-      "if (dist > 1.0) {\n"
-      "  discard;\n"
-      "} else {\n"
-      "  float scale = (1.0 - dist);\n"
-      "  ambientColor *= scale;\n"
-      "  diffuseColor *= scale;\n"
-      "}\n");
   }
 
   std::string Name;

library/VTKExtensions/Rendering/CMakeLists.txt

@@ -10,6 +10,25 @@ set(sources
 set(private_headers
   ${CMAKE_CURRENT_BINARY_DIR}/F3DDefaultHDRI.h)
 
+set(shader_files
+  glsl/vtkF3DBitonicSortGlobalDisperseCS.glsl
+  glsl/vtkF3DBitonicSortGlobalFlipCS.glsl
+  glsl/vtkF3DBitonicSortLocalDisperseCS.glsl
+  glsl/vtkF3DBitonicSortLocalSortCS.glsl
+  glsl/vtkF3DBitonicSortFunctions.glsl
+  glsl/vtkF3DComputeDepthCS.glsl)
+
+foreach(file IN LISTS shader_files)
+  vtk_encode_string(
+    INPUT         "${file}"
+    HEADER_OUTPUT header
+    SOURCE_OUTPUT source)
+  list(APPEND sources
+    "${source}")
+  list(APPEND private_headers
+    "${header}")
+endforeach()
+
 set(classes
   vtkF3DCachedLUTTexture
   vtkF3DCachedSpecularTexture
@@ -25,8 +44,13 @@ set(classes
   vtkF3DRendererWithColoring
   )
 
+# Needs https://gitlab.kitware.com/vtk/vtk/-/merge_requests/10675
+if(NOT ANDROID AND NOT EMSCRIPTEN AND VTK_VERSION VERSION_GREATER_EQUAL 9.3.20240203)
+  set(classes ${classes} vtkF3DBitonicSort vtkF3DPointSplatMapper)
+endif()
+
 if(NOT VTK_VERSION VERSION_GREATER_EQUAL 9.2.20220907)
-    set(classes ${classes} vtkF3DOrientationMarkerWidget)
+  set(classes ${classes} vtkF3DOrientationMarkerWidget)
 endif()
 
 vtk_module_add_module(f3d::VTKExtensionsRendering

library/VTKExtensions/Rendering/Testing/CMakeLists.txt

@@ -3,12 +3,22 @@ if(VTK_VERSION VERSION_LESS_EQUAL 9.1.0)
   cmake_policy(SET CMP0115 OLD)
 endif()
 
+list(APPEND VTKExtensionsRenderingTests_list
+     TestF3DInteractorEventRecorder.cxx
+     TestF3DOpenGLGridMapper.cxx 
+     TestF3DRenderPass.cxx
+     TestF3DRendererWithColoring.cxx
+     TestF3DCachedTexturesPrint.cxx
+  )
+
+# Also needs https://gitlab.kitware.com/vtk/vtk/-/merge_requests/10675
+if(NOT ANDROID AND NOT EMSCRIPTEN AND VTK_VERSION VERSION_GREATER_EQUAL 9.3.20240203)
+  list(APPEND VTKExtensionsRenderingTests_list 
+       TestF3DBitonicSort.cxx)
+endif()
+
 vtk_add_test_cxx(VTKExtensionsRenderingTests tests
   NO_DATA NO_VALID NO_OUTPUT
-  TestF3DInteractorEventRecorder.cxx
-  TestF3DOpenGLGridMapper.cxx 
-  TestF3DRenderPass.cxx
-  TestF3DRendererWithColoring.cxx
-  TestF3DCachedTexturesPrint.cxx
+  ${VTKExtensionsRenderingTests_list}
   ${F3D_SOURCE_DIR}/testing/ ${CMAKE_BINARY_DIR}/Testing/Temporary/)
 vtk_test_cxx_executable(VTKExtensionsRenderingTests tests)

library/VTKExtensions/Rendering/Testing/TestF3DBitonicSort.cxx

@@ -0,0 +1,107 @@
+#include <vtkOpenGLBufferObject.h>
+#include <vtkOpenGLRenderWindow.h>
+#include <vtkShader.h>
+
+#include "vtkF3DBitonicSort.h"
+#include "vtkF3DBitonicSortFunctions.h"
+#include "vtkF3DBitonicSortGlobalDisperseCS.h"
+#include "vtkF3DBitonicSortGlobalFlipCS.h"
+#include "vtkF3DBitonicSortLocalDisperseCS.h"
+#include "vtkF3DBitonicSortLocalSortCS.h"
+
+#include <algorithm>
+#include <random>
+
+int TestF3DBitonicSort(int argc, char* argv[])
+{
+  // Turn off VTK error reporting to avoid unwanted failure detection by ctest
+  vtkObject::GlobalWarningDisplayOff();
+
+  // we need an OpenGL context
+  vtkNew<vtkRenderWindow> renWin;
+  renWin->OffScreenRenderingOn();
+  renWin->Start();
+
+  if (!vtkShader::IsComputeShaderSupported())
+  {
+    std::cerr << "Compute shaders are not supported on this system, skipping the test.\n";
+    return EXIT_SUCCESS;
+  }
+
+  constexpr int nbElements = 10000;
+
+  // fill CPU keys and values buffers
+  std::vector<double> keys(nbElements);
+  std::vector<int> values(nbElements);
+
+  std::random_device dev;
+  std::mt19937 rng(dev());
+  std::uniform_real_distribution<double> dist(0.0, 1.0);
+
+  std::generate(std::begin(keys), std::end(keys), [&]() { return dist(rng); });
+  std::fill(std::begin(values), std::end(values), 0); // we do not care about the values
+
+  // upload these buffers to the GPU
+  vtkNew<vtkOpenGLBufferObject> bufferKeys;
+  vtkNew<vtkOpenGLBufferObject> bufferValues;
+
+  bufferKeys->Upload(keys, vtkOpenGLBufferObject::ArrayBuffer);
+  bufferValues->Upload(values, vtkOpenGLBufferObject::ArrayBuffer);
+
+  // sort
+  vtkNew<vtkF3DBitonicSort> sorter;
+
+  // check invalid workgroup size
+  if (sorter->Initialize(-1, VTK_FLOAT, VTK_FLOAT))
+  {
+    std::cerr << "The invalid workgroup size is not failing" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  // check invalid types
+  if (sorter->Initialize(128, VTK_CHAR, VTK_FLOAT))
+  {
+    std::cerr << "The invalid key type is not failing" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if (sorter->Initialize(128, VTK_FLOAT, VTK_CHAR))
+  {
+    std::cerr << "The invalid key type is not failing" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if (sorter->Run(
+        vtkOpenGLRenderWindow::SafeDownCast(renWin), nbElements, bufferKeys, bufferValues))
+  {
+    std::cerr << "Uninitialized run is not failing" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if (!sorter->Initialize(128, VTK_DOUBLE, VTK_INT))
+  {
+    std::cerr << "Valid Initialize call failed" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  if (!sorter->Run(
+        vtkOpenGLRenderWindow::SafeDownCast(renWin), nbElements, bufferKeys, bufferValues))
+  {
+    std::cerr << "Sorter Run call failed" << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  // download sorted key buffer to CPU
+  bufferKeys->Download(keys.data(), keys.size());
+
+  // check if correctly sorted
+  for (int i = 1; i < nbElements; i++)
+  {
+    if (keys[i - 1] > keys[i])
+    {
+      return EXIT_FAILURE;
+    }
+  }
+
+  return EXIT_SUCCESS;
+}