3D gaussians .splat reader (#1248)

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
 skip = ./testing/**/*,./.git/**/*,./**/*.patch,./external/cxxopts/cxxopts.hpp,./external/nlohmann_json/nlohmann/json.hpp
-ignore-words-list=nnumber,unknwn,dota,modle
+ignore-words-list=nnumber,unknwn,dota,modle,inout

.cppcheck.supp

@@ -16,3 +16,4 @@ preprocessorErrorDirective:plugins/draco/module/vtkF3DDracoReader.cxx
 unknownMacro:library/VTKExtensions/Applicative/vtkF3DObjectFactory.cxx
 unusedVariable:*factory.cxx*
 constParameter:library/src/image.cxx
+invalidPointerCast:plugins/native/module/vtkF3DSplatReader.cxx

.github/actions/mesa-install-bin/action.yml

@@ -18,9 +18,10 @@ runs:
  run: |
  mkdir mesa
  cd mesa
- curl.exe -L --output mesa.7z --url https://github.com/pal1000/mesa-dist-win/releases/download/22.0.1/mesa3d-22.0.1-release-msvc.7z
+ curl.exe -L --output mesa.7z --url https://github.com/pal1000/mesa-dist-win/releases/download/23.3.5/mesa3d-23.3.5-release-msvc.7z
  C:\'Program Files'\7-Zip\7z.exe x mesa.7z
  # A * is added next line to force Get-ChildItem to look for directory within the path
  Get-ChildItem -Directory ${{inputs.path}}* | ForEach-Object { Copy-Item -Path .\x64\opengl32.dll, .\x64\libglapi.dll, .\x64\libgallium_wgl.dll -Destination $_ }
+ echo "GALLIUM_DRIVER=llvmpipe"| Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
  cd ..
  rm .\mesa -r -force

.github/actions/vtk_commit_sha

@@ -1 +1 @@
-a4cb9c6c90c18a31e225aa4a18df6a591bf8cd3b
+be69fa3a50de292857e69702885576ac6f114805

.lsan.supp

@@ -19,6 +19,7 @@ leak:TKernel
 leak:libtbb
 
 # Potential mesa/VTK leak with incomplete callstack
-# forces us to hide all leaks from the libf3dSDKTests
+# forces us to hide all leaks from the tests using a render window
 # https://gitlab.kitware.com/vtk/vtk/-/issues/18504
 leak:libf3dSDKTests
+leak:VTKExtensionsRenderingTests

application/F3DOptionsParser.cxx

@@ -341,6 +341,7 @@ void ConfigurationOptions::GetOptions(F3DAppOptions& appOptions, f3d::options& o
 
  auto grp2 = cxxOptions.add_options("Material");
  this->DeclareOption(grp2, "point-sprites", "o", "Show sphere sprites instead of geometry", options.getAsBoolRef("model.point-sprites.enable"), HasDefault::YES, MayHaveConfig::YES);
+ this->DeclareOption(grp2, "point-type", "", "Point splat type when showing point sprites", options.getAsStringRef("render.splat-type"), HasDefault::YES, MayHaveConfig::YES, "<sphere|gaussian>");
  this->DeclareOption(grp2, "point-size", "", "Point size when showing vertices or point sprites", options.getAsDoubleRef("render.point-size"), HasDefault::YES, MayHaveConfig::YES, "<size>");
  this->DeclareOption(grp2, "line-width", "", "Line width when showing edges", options.getAsDoubleRef("render.line-width"), HasDefault::YES, MayHaveConfig::YES, "<width>");
  this->DeclareOption(grp2, "color", "", "Solid color", options.getAsDoubleVectorRef("model.color.rgb"), HasDefault::YES, MayHaveConfig::YES, "<R,G,B>");

application/testing/CMakeLists.txt

@@ -212,6 +212,13 @@ else()
  endif()
 endif()
 
+# Needs splat sorting with compute shaders
+if(VTK_VERSION VERSION_GREATER_EQUAL 9.3.20240203)
+ if(NOT APPLE) # MacOS does not support compute shaders
+ f3d_test(NAME Test3DGaussiansSplatting DATA small.splat ARGS -osy --up=-Y --point-size=1 --point-type=gaussian --camera-position=-3.6,0.5,-4.2)
+ endif()
+endif()
+
 if (NOT APPLE)
  # This test is broken on apple because of #792
  f3d_test(NAME TestScalarsCell DATA f3d.vtp ARGS --scalars --cells --comp=-2 --up=+Z DEFAULT_LIGHTS)

doc/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## Ongoing development
 
+For F3D users:
+- Added a new option `--point-type` used to specify how to display points
+- Add support for 3D Gaussians Splatting in binary .splat format
+
+For libf3d users:
+- Added a new option `render.splat-type` used to specify how to display points (only if `model.point-sprites.enable` is true)
+
 ## v2.3.0
 
 For F3D users:

doc/GALLERY.md

@@ -15,6 +15,10 @@ Images and videos displayed below use public datasets, you can download them [he
 
 *Animated scientific visualization rendering*: `f3d can.ex2 -xtgans --up=+Z --scalars=VEL`
 
+<video src='https://media.githubusercontent.com/media/f3d-app/f3d-media/main/media/counter.webm' autoplay="autoplay" loop="loop" width="700"></video>
+
+*3D Gaussians Splatting*: `f3d counter.splat --point-size=1 --point-type=gaussian -soynxz --up=-Y --camera-position=0,1,-5.2 --camera-focal-point=0,1,0`
+
 <img src="https://user-images.githubusercontent.com/3129530/194735272-5bcd3e7c-a333-41f5-8066-9b0bec9885e8.png" width="700">
 
 *Direct scalars rendering of a point cloud*: `f3d Carola_PointCloud.ply --point-size=0 --comp=-2 -so --up=+Z --hdri-skybox --hdri-ambient --hdri-file=venice_sunset_8k.hdr`

doc/libf3d/OPTIONS.md

@@ -66,6 +66,7 @@ render.effect.tone-mapping|bool<br>false<br>render|Enable generic filmic *Tone M
 render.line-width|double<br>1.0<br>render|Set the *width* of lines when showing edges.|\-\-line-width
 render.show-edges|bool<br>false<br>render|Show the *cell edges*|\-\-edges
 render.point-size|double<br>10.0<br>render|Set the *size* of points when showing vertices and point sprites.|\-\-point-size
+render.splat-type|string<br>sphere<br>render|Set the splat type when showing point sprites (can be `sphere` or `gaussian`).|\-\-point-type
 render.grid.enable|bool<br>false<br>render|Show *a grid* aligned with the horizontal (orthogonal to the Up direction) plane.|\-\-grid
 render.grid.absolute|bool<br>false<br>render|Position the grid at the *absolute origin* of the model's coordinate system instead of below the model.|\-\-grid
 render.grid.unit|double<br>0<br>render|Set the size of the *unit square* for the grid. If set to non-positive (the default) a suitable value will be automatically computed.|\-\-grid\-unit

doc/user/OPTIONS.md

@@ -45,6 +45,7 @@ Options|Default|Description
 ------|------|------
 -o, \-\-point-sprites||Show sphere *points sprites* instead of the geometry.
 \-\-point-size=\<size\>|10.0|Set the *size* of points when showing vertices and point sprites.
+\-\-point-type=\<sphere|gaussian\>|sphere|Set the splat type when showing point sprites.
 \-\-line-width=\<size\>|1.0|Set the *width* of lines when showing edges.
 \-\-color=\<R,G,B\>|1.0, 1.0, 1.0| Set a *color* on the geometry. Multiplied with the base color texture when present. <br>Requires a default scene.
 \-\-opacity=\<opacity\>|1.0|Set *opacity* on the geometry. Multiplied with the base color texture when present. <br>Requires a default scene. Usually used with Depth Peeling option.

library/VTKExtensions/Applicative/Testing/TestF3DObjectFactory.cxx

@@ -13,12 +13,20 @@
 #include <vtkTestUtilities.h>
 #include <vtkVersion.h>
 
+#if VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+#include "vtkF3DPointSplatMapper.h"
+#endif
+
 int TestF3DObjectFactory(int argc, char* argv[])
 {
  vtkNew<vtkF3DObjectFactory> factory;
  vtkObjectFactory::RegisterFactory(factory);
  vtkObjectFactory::SetAllEnableFlags(0, "vtkPolyDataMapper", "vtkOpenGLPolyDataMapper");
 
+#if VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+ vtkObjectFactory::SetAllEnableFlags(0, "vtkPointGaussianMapper", "vtkOpenGLPointGaussianMapper");
+#endif
+
  // Check factory utility methods
  if (strcmp(factory->GetVTKSourceVersion(), VTK_SOURCE_VERSION) != 0)
  {
@@ -40,6 +48,17 @@ int TestF3DObjectFactory(int argc, char* argv[])
  return EXIT_FAILURE;
  }
 
+#if VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+ vtkNew<vtkPointGaussianMapper> pointMapper;
+ pointMapper->Print(cout);
+ vtkF3DPointSplatMapper* pointMapperPtr = vtkF3DPointSplatMapper::SafeDownCast(pointMapper);
+ if (pointMapperPtr == nullptr)
+ {
+ std::cerr << "vtkF3DObjectFactory failed to create a vtkF3DPointSplatMapper" << std::endl;
+ return EXIT_FAILURE;
+ }
+#endif
+
  vtkNew<vtkOutputWindow> window;
 #if F3D_WINDOWS_GUI
  vtkF3DWin32OutputWindow* windowPtr = vtkF3DWin32OutputWindow::SafeDownCast(window);

library/VTKExtensions/Applicative/vtkF3DObjectFactory.cxx

@@ -1,28 +1,38 @@
 #include "vtkF3DObjectFactory.h"
 
+#include <vtkVersion.h>
+
 #include "vtkF3DConfigure.h"
 #include "vtkF3DPolyDataMapper.h"
 
+#if !defined(__ANDROID__) && !defined(__EMSCRIPTEN__) && \
+ VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+#include <vtkF3DPointSplatMapper.h>
+#endif
+
 #ifdef __ANDROID__
-#include "vtkF3DAndroidLogOutputWindow.h"
+#include <vtkF3DAndroidLogOutputWindow.h>
 #elif F3D_WINDOWS_GUI
-#include "vtkF3DWin32OutputWindow.h"
+#include <vtkF3DWin32OutputWindow.h>
 #else
-#include "vtkF3DConsoleOutputWindow.h"
+#include <vtkF3DConsoleOutputWindow.h>
 #endif
 
 #ifdef __EMSCRIPTEN__
-#include "vtkSDL2OpenGLRenderWindow.h"
-#include "vtkSDL2RenderWindowInteractor.h"
+#include <vtkSDL2OpenGLRenderWindow.h>
+#include <vtkSDL2RenderWindowInteractor.h>
 #endif
 
-#include <vtkVersion.h>
-
 vtkStandardNewMacro(vtkF3DObjectFactory);
 
 // Now create the functions to create overrides with.
 VTK_CREATE_CREATE_FUNCTION(vtkF3DPolyDataMapper)
 
+#if !defined(__ANDROID__) && !defined(__EMSCRIPTEN__) && \
+ VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+VTK_CREATE_CREATE_FUNCTION(vtkF3DPointSplatMapper)
+#endif
+
 #ifdef __ANDROID__
 VTK_CREATE_CREATE_FUNCTION(vtkF3DAndroidLogOutputWindow)
 #elif F3D_WINDOWS_GUI
@@ -42,6 +52,12 @@ vtkF3DObjectFactory::vtkF3DObjectFactory()
  this->RegisterOverride("vtkPolyDataMapper", "vtkF3DPolyDataMapper",
  "vtkPolyDataMapper override for F3D", 1, vtkObjectFactoryCreatevtkF3DPolyDataMapper);
 
+#if !defined(__ANDROID__) && !defined(__EMSCRIPTEN__) && \
+ VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 3, 20240203)
+ this->RegisterOverride("vtkPointGaussianMapper", "vtkF3DPointSplatMapper",
+ "vtkPointGaussianMapper override for F3D", 1, vtkObjectFactoryCreatevtkF3DPointSplatMapper);
+#endif
+
 #ifdef __ANDROID__
  this->RegisterOverride("vtkOutputWindow", "vtkF3DAndroidLogOutputWindow",
  "vtkOutputWindow override for F3D", 1, vtkObjectFactoryCreatevtkF3DAndroidLogOutputWindow);

library/VTKExtensions/Readers/vtkF3DGenericImporter.cxx

@@ -34,17 +34,6 @@ struct ReaderPipeline
  this->GeometryActor->GetProperty()->SetInterpolationToPBR();
  this->VolumeMapper->SetRequestedRenderModeToGPU();
  this->PolyDataMapper->InterpolateScalarsBeforeMappingOn();
- this->PointGaussianMapper->EmissiveOff();
- this->PointGaussianMapper->SetSplatShaderCode(
- "//VTK::Color::Impl\n"
- "float dist = dot(offsetVCVSOutput.xy, offsetVCVSOutput.xy);\n"
- "if (dist > 1.0) {\n"
- " discard;\n"
- "} else {\n"
- " float scale = (1.0 - dist);\n"
- " ambientColor *= scale;\n"
- " diffuseColor *= scale;\n"
- "}\n");
  }
 
  std::string Name;

library/VTKExtensions/Rendering/CMakeLists.txt

@@ -10,6 +10,25 @@ set(sources
 set(private_headers
  ${CMAKE_CURRENT_BINARY_DIR}/F3DDefaultHDRI.h)
 
+set(shader_files
+ glsl/vtkF3DBitonicSortGlobalDisperseCS.glsl
+ glsl/vtkF3DBitonicSortGlobalFlipCS.glsl
+ glsl/vtkF3DBitonicSortLocalDisperseCS.glsl
+ glsl/vtkF3DBitonicSortLocalSortCS.glsl
+ glsl/vtkF3DBitonicSortFunctions.glsl
+ glsl/vtkF3DComputeDepthCS.glsl)
+
+foreach(file IN LISTS shader_files)
+ vtk_encode_string(
+ INPUT "${file}"
+ HEADER_OUTPUT header
+ SOURCE_OUTPUT source)
+ list(APPEND sources
+ "${source}")
+ list(APPEND private_headers
+ "${header}")
+endforeach()
+
 set(classes
  vtkF3DCachedLUTTexture
  vtkF3DCachedSpecularTexture
@@ -25,8 +44,13 @@ set(classes
  vtkF3DRendererWithColoring
  )
 
+# Needs https://gitlab.kitware.com/vtk/vtk/-/merge_requests/10675
+if(NOT ANDROID AND NOT EMSCRIPTEN AND VTK_VERSION VERSION_GREATER_EQUAL 9.3.20240203)
+ set(classes ${classes} vtkF3DBitonicSort vtkF3DPointSplatMapper)
+endif()
+
 if(NOT VTK_VERSION VERSION_GREATER_EQUAL 9.2.20220907)
-  set(classes ${classes} vtkF3DOrientationMarkerWidget)
+ set(classes ${classes} vtkF3DOrientationMarkerWidget)
 endif()
 
 vtk_module_add_module(f3d::VTKExtensionsRendering

library/VTKExtensions/Rendering/Testing/CMakeLists.txt

@@ -3,12 +3,22 @@ if(VTK_VERSION VERSION_LESS_EQUAL 9.1.0)
  cmake_policy(SET CMP0115 OLD)
 endif()
 
+list(APPEND VTKExtensionsRenderingTests_list
+ TestF3DInteractorEventRecorder.cxx
+ TestF3DOpenGLGridMapper.cxx 
+ TestF3DRenderPass.cxx
+ TestF3DRendererWithColoring.cxx
+ TestF3DCachedTexturesPrint.cxx
+ )
+
+# Also needs https://gitlab.kitware.com/vtk/vtk/-/merge_requests/10675
+if(NOT ANDROID AND NOT EMSCRIPTEN AND VTK_VERSION VERSION_GREATER_EQUAL 9.3.20240203)
+ list(APPEND VTKExtensionsRenderingTests_list 
+ TestF3DBitonicSort.cxx)
+endif()
+
 vtk_add_test_cxx(VTKExtensionsRenderingTests tests
  NO_DATA NO_VALID NO_OUTPUT
- TestF3DInteractorEventRecorder.cxx
- TestF3DOpenGLGridMapper.cxx 
- TestF3DRenderPass.cxx
- TestF3DRendererWithColoring.cxx
- TestF3DCachedTexturesPrint.cxx
+ ${VTKExtensionsRenderingTests_list}
  ${F3D_SOURCE_DIR}/testing/ ${CMAKE_BINARY_DIR}/Testing/Temporary/)
 vtk_test_cxx_executable(VTKExtensionsRenderingTests tests)

library/VTKExtensions/Rendering/Testing/TestF3DBitonicSort.cxx

@@ -0,0 +1,107 @@
+#include <vtkOpenGLBufferObject.h>
+#include <vtkOpenGLRenderWindow.h>
+#include <vtkShader.h>
+
+#include "vtkF3DBitonicSort.h"
+#include "vtkF3DBitonicSortFunctions.h"
+#include "vtkF3DBitonicSortGlobalDisperseCS.h"
+#include "vtkF3DBitonicSortGlobalFlipCS.h"
+#include "vtkF3DBitonicSortLocalDisperseCS.h"
+#include "vtkF3DBitonicSortLocalSortCS.h"
+
+#include <algorithm>
+#include <random>
+
+int TestF3DBitonicSort(int argc, char* argv[])
+{
+ // Turn off VTK error reporting to avoid unwanted failure detection by ctest
+ vtkObject::GlobalWarningDisplayOff();
+
+ // we need an OpenGL context
+ vtkNew<vtkRenderWindow> renWin;
+ renWin->OffScreenRenderingOn();
+ renWin->Start();
+
+ if (!vtkShader::IsComputeShaderSupported())
+ {
+ std::cerr << "Compute shaders are not supported on this system, skipping the test.\n";
+ return EXIT_SUCCESS;
+ }
+
+ constexpr int nbElements = 10000;
+
+ // fill CPU keys and values buffers
+ std::vector<double> keys(nbElements);
+ std::vector<int> values(nbElements);
+
+ std::random_device dev;
+ std::mt19937 rng(dev());
+ std::uniform_real_distribution<double> dist(0.0, 1.0);
+
+ std::generate(std::begin(keys), std::end(keys), [&]() { return dist(rng); });
+ std::fill(std::begin(values), std::end(values), 0); // we do not care about the values
+
+ // upload these buffers to the GPU
+ vtkNew<vtkOpenGLBufferObject> bufferKeys;
+ vtkNew<vtkOpenGLBufferObject> bufferValues;
+
+ bufferKeys->Upload(keys, vtkOpenGLBufferObject::ArrayBuffer);
+ bufferValues->Upload(values, vtkOpenGLBufferObject::ArrayBuffer);
+
+ // sort
+ vtkNew<vtkF3DBitonicSort> sorter;
+
+ // check invalid workgroup size
+ if (sorter->Initialize(-1, VTK_FLOAT, VTK_FLOAT))
+ {
+ std::cerr << "The invalid workgroup size is not failing" << std::endl;
+ return EXIT_FAILURE;
+ }
+
+ // check invalid types
+ if (sorter->Initialize(128, VTK_CHAR, VTK_FLOAT))
+ {
+ std::cerr << "The invalid key type is not failing" << std::endl;
+ return EXIT_FAILURE;
+ }
+
+ if (sorter->Initialize(128, VTK_FLOAT, VTK_CHAR))
+ {
+ std::cerr << "The invalid key type is not failing" << std::endl;
+ return EXIT_FAILURE;
+ }
+
+ if (sorter->Run(
+ vtkOpenGLRenderWindow::SafeDownCast(renWin), nbElements, bufferKeys, bufferValues))
+ {
+ std::cerr << "Uninitialized run is not failing" << std::endl;
+ return EXIT_FAILURE;
+ }
+
+ if (!sorter->Initialize(128, VTK_DOUBLE, VTK_INT))
+ {
+ std::cerr << "Valid Initialize call failed" << std::endl;
+ return EXIT_FAILURE;
+ }
+
+ if (!sorter->Run(
+ vtkOpenGLRenderWindow::SafeDownCast(renWin), nbElements, bufferKeys, bufferValues))
+ {
+ std::cerr << "Sorter Run call failed" << std::endl;
+ return EXIT_FAILURE;
+ }
+
+ // download sorted key buffer to CPU
+ bufferKeys->Download(keys.data(), keys.size());
+
+ // check if correctly sorted
+ for (int i = 1; i < nbElements; i++)
+ {
+ if (keys[i - 1] > keys[i])
+ {
+ return EXIT_FAILURE;
+ }
+ }
+
+ return EXIT_SUCCESS;
+}