Add depth unprojection support to batch renderer

src/esp/gfx/CMakeLists.txt

@@ -118,11 +118,6 @@ add_library(
 )
 
 if(BUILD_WITH_CUDA)
-  target_include_directories(
-    gfx PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
-                ${CMAKE_CURRENT_LIST_DIR}/../gfx_batch/cuda_helpers
-  )
-
   target_link_libraries(gfx PUBLIC ${CUDART_LIBRARY})
 endif()
 

src/esp/gfx_batch/CMakeLists.txt

@@ -35,8 +35,8 @@ add_library(
 target_include_directories(gfx_batch PUBLIC ${PROJECT_BINARY_DIR})
 if(BUILD_WITH_CUDA)
   target_include_directories(
-    gfx_batch PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
-                      ${CMAKE_CURRENT_LIST_DIR}/cuda_helpers
+    gfx_batch PUBLIC ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
+                     ${CMAKE_CURRENT_LIST_DIR}/cuda_helpers
   )
 
   target_link_libraries(gfx_batch PUBLIC ${CUDART_LIBRARY})

src/esp/gfx_batch/DepthUnprojection.cpp

@@ -6,6 +6,7 @@
 
 #include <Corrade/Containers/ArrayView.h>
 #include <Corrade/Containers/Reference.h>
+#include <Corrade/Containers/StridedArrayView.h>
 #include <Corrade/Utility/Resource.h>
 #include <Magnum/GL/Shader.h>
 #include <Magnum/GL/Texture.h>
@@ -126,5 +127,29 @@ void unprojectDepth(const Mn::Vector2& unprojection,
   }
 }
 
+void unprojectDepth(
+    const Mn::Vector2& unprojection,
+    const Cr::Containers::StridedArrayView2D<Mn::Float>& depth) {
+  for (Cr::Containers::StridedArrayView1D<Mn::Float> row : depth) {
+    for (Mn::Float& d : row) {
+      d = unprojection[1] / (d + unprojection[0]);
+    }
+  }
+
+  /* Change pixels on the far plane to be 0. Done in a separate loop to allow
+     the optimizer to vectorize the above better.  */
+  const Mn::Float farDepth = unprojection[1] / (1.0f + unprojection[0]);
+  for (Cr::Containers::StridedArrayView1D<Mn::Float> row : depth) {
+    for (Mn::Float& d : row) {
+      /* We can afford using == for comparison as 1.0f has an exact
+         representation, the depth was cleared to exactly this value and the
+         calculation is done exactly the same way in both cases -- thus the
+         result should be bit-exact. */
+      if (d == farDepth)
+        d = 0.0f;
+    }
+  }
+}
+
 }  // namespace gfx_batch
 }  // namespace esp

src/esp/gfx_batch/DepthUnprojection.h

@@ -151,6 +151,21 @@ consumers expect zeros for things that are too far.
 void unprojectDepth(const Magnum::Vector2& unprojection,
                     Corrade::Containers::ArrayView<Magnum::Float> depth);
 
+/**
+@brief Unproject depth values
+@param[in] unprojection Unprojection coefficients from
+    @ref calculateDepthUnprojection()
+@param[in,out] depth    Depth values in range @f$ [ 0 ; 1 ] @f$
+
+See @ref calculateDepthUnprojection() for the full algorithm explanation.
+Additionally to applying that calculation, if the input depth is at the far
+plane (of value @cpp 1.0f @ce), it's set to @cpp 0.0f @ce on output as
+consumers expect zeros for things that are too far.
+*/
+void unprojectDepth(
+    const Magnum::Vector2& unprojection,
+    const Corrade::Containers::StridedArrayView2D<Magnum::Float>& depth);
+
 }  // namespace gfx_batch
 }  // namespace esp
 

src/esp/gfx_batch/Renderer.cpp

@@ -1062,6 +1062,11 @@ const Magnum::Matrix4& Renderer::camera(Magnum::UnsignedInt sceneId) const {
   return state_->cameraMatrices[sceneId].projectionMatrix;
 }
 
+const Magnum::Vector2& Renderer::cameraDepthUnprojection(
+    Magnum::UnsignedInt sceneId) const {
+  return state_->cameraUnprojections[sceneId];
+}
+
 void Renderer::updateCamera(Magnum::UnsignedInt sceneId,
                             const Magnum::Matrix4& projection,
                             const Magnum::Matrix4& view) {
@@ -1335,12 +1340,6 @@ void Renderer::draw(Mn::GL::AbstractFramebuffer& framebuffer) {
   framebuffer.setViewport(previousViewport);
 }
 
-void Renderer::unprojectDepth(int sceneId, const Mn::MutableImageView2D& view) {
-  // TODO: Add GPU depth unprojection support.
-  gfx_batch::unprojectDepth(state_->cameraUnprojections[sceneId],
-                            Cr::Containers::arrayCast<Mn::Float>(view.data()));
-}
-
 SceneStats Renderer::sceneStats(Mn::UnsignedInt sceneId) const {
   CORRADE_ASSERT(sceneId < state_->scenes.size(),
                  "Renderer::sceneStats(): index"

src/esp/gfx_batch/Renderer.h

@@ -239,7 +239,7 @@ implemented in @ref shaders-usage-multidraw "Magnum shaders".
   added to a *draw list*. The draw list is
   @ref gfx_batch-Renderer-workflow-draw-list "further detailed below".
 - For each @ref draw() and each non-empty scene, the following is done:
-  - The @ref cameraMatrices transformations are uploaded to a uniform buffer.
+  - The transformation passed to @ref updateCamera() is uploaded to a uniform buffer.
   - The renderer calculates hierarchical transformations for all nodes based on
     the matrices supplied via @ref transformations(). Each item in the draw
     list is then assigned a corresponding calculated absolute transformation,
@@ -646,6 +646,13 @@ class Renderer {
    */
   const Magnum::Matrix4& camera(Magnum::UnsignedInt sceneId) const;
 
+  /**
+   * @brief Get the depth unprojection parameters of a camera (read-only)
+   * @param sceneId Scene ID, expected to be less than @ref sceneCount()
+   */
+  const Magnum::Vector2& cameraDepthUnprojection(
+      Magnum::UnsignedInt sceneId) const;
+
   /**
    * @brief Set the camera projection and view matrices
    * @param sceneId     Scene ID, expected to be less than @ref sceneCount()
@@ -715,16 +722,6 @@ class Renderer {
    */
   void draw(Magnum::GL::AbstractFramebuffer& framebuffer);
 
-  /**
-   * @brief Unprojects depth onto the provided image.
-   * Produces an image that contains the distance of each pixels, in meters.
-   *
-   * @param sceneId Scene ID, expected to be less than @ref sceneCount()
-   * @param view View on raw depth of scene corresponding to the specified
-   * sceneId.
-   */
-  void unprojectDepth(int sceneId, const Magnum::MutableImageView2D& view);
-
   /**
    * @brief Scene stats
    *

src/esp/sim/BatchReplayRenderer.cpp

@@ -4,6 +4,7 @@
 
 #include "BatchReplayRenderer.h"
 
+#include <esp/gfx_batch/DepthUnprojection.h>
 #include "esp/sensor/CameraSensor.h"
 
 #include <Corrade/Containers/GrowableArray.h>
@@ -279,7 +280,11 @@ void BatchReplayRenderer::doRender(
           standalone.depthFramebufferFormat(), depthImageViews[envIndex].size(),
           depthImageViews[envIndex].data()};
       standalone.depthImageInto(rectangle, depthBufferView);
-      renderer_->unprojectDepth(envIndex, depthBufferView);
+
+      // TODO: Add GPU depth unprojection support.
+      gfx_batch::unprojectDepth(
+          renderer_->cameraDepthUnprojection(envIndex),
+          Cr::Containers::arrayCast<Mn::Float>(depthBufferView.data()));
     }
   }
 }

src/shaders/gfx/Shaders.conf

@@ -3,12 +3,6 @@ group = gfx-shaders
 # To avoid a needless copy when the sources get added to GL::Shader
 nullTerminated=true
 
-[file]
-filename = ../gfx_batch/depth.vert
-
-[file]
-filename = ../gfx_batch/depth.frag
-
 [file]
 filename = ptex-default-gl410.vert
 

src/tests/GfxBatchRendererTest.cpp

@@ -2,9 +2,7 @@
 // This source code is licensed under the MIT license found in the
 // LICENSE file in the root directory of this source tree.
 
-#include <Corrade/Containers/StridedArrayView.h>
 #include <Corrade/Containers/Triple.h>
-#include <Corrade/PluginManager/Manager.h>
 #include <Corrade/PluginManager/PluginMetadata.h>
 #include <Corrade/TestSuite/Compare/File.h>
 #include <Corrade/TestSuite/Tester.h>
@@ -15,8 +13,6 @@
 #include <Magnum/Image.h>
 #include <Magnum/ImageView.h>
 #include <Magnum/Math/Color.h>
-#include <Magnum/Math/Matrix3.h>
-#include <Magnum/Math/Matrix4.h>
 #include <Magnum/Math/Range.h>
 #include <Magnum/MeshTools/Combine.h>
 #include <Magnum/MeshTools/Concatenate.h>
@@ -33,7 +29,7 @@
 #include <Magnum/Trade/SceneData.h>
 #include <Magnum/Trade/TextureData.h>
 
-#include "Corrade/TestSuite/Comparator.h"
+#include <esp/gfx_batch/DepthUnprojection.h>
 #include "Corrade/TestSuite/Compare/Numeric.h"
 #include "esp/gfx_batch/RendererStandalone.h"
 
@@ -2177,17 +2173,15 @@ void GfxBatchRendererTest::imageInto() {
 }
 
 void GfxBatchRendererTest::depthUnprojection() {
-  constexpr int tileCountX = 2;
-  constexpr int tileCountY = 2;
-  constexpr int envCount = tileCountX * tileCountY;
+  constexpr Mn::Vector2i tileCount{2, 2};
   constexpr float near = 0.001f;
   constexpr float far = 10.0f;
   constexpr Mn::Vector2i tileSize(64, 64);
 
   // clang-format off
   esp::gfx_batch::RendererStandalone renderer{
       esp::gfx_batch::RendererConfiguration{}
-          .setTileSizeCount(tileSize, {tileCountX, tileCountY}),
+          .setTileSizeCount(tileSize, tileCount),
       esp::gfx_batch::RendererStandaloneConfiguration{}
           .setFlags(esp::gfx_batch::RendererStandaloneFlag::QuietLog)
   };
@@ -2214,7 +2208,7 @@ void GfxBatchRendererTest::depthUnprojection() {
 
   // Spawn a plane in each environment, at origin, facing the camera.
   CORRADE_VERIFY(renderer.hasNodeHierarchy("square"));
-  for (int i = 0; i < envCount; ++i) {
+  for (int i = 0; i < tileCount.product(); ++i) {
     CORRADE_COMPARE(renderer.addNodeHierarchy(i, "square"), 0);
   }
 
@@ -2224,26 +2218,29 @@ void GfxBatchRendererTest::depthUnprojection() {
   MAGNUM_VERIFY_NO_GL_ERROR();
 
   // Unproject each scene.
-  const size_t subBufferDataSize = depth.data().size() / envCount;
-  const auto subBufferSize = Mn::Vector2i{depth.size().x() / tileCountX,
-                                          depth.size().y() / tileCountY};
-  for (int i = 0; i < envCount; ++i) {
-    Mn::MutableImageView2D depthSubBufferView{
-        Mn::PixelFormat::Depth32F, subBufferSize,
-        depth.data().sliceSize(i * subBufferDataSize, subBufferDataSize)};
-    renderer.unprojectDepth(i, depthSubBufferView);
+  for (int y = 0; y != tileCount.y(); ++y) {
+    for (int x = 0; x != tileCount.x(); ++x) {
+      const std::size_t sceneId = y * tileCount.x() + x;
+      const Mn::Containers::Size2D offset(x * tileSize.x(), y * tileSize.y());
+      const Mn::Containers::Size2D size(tileSize.x(), tileSize.y());
+      esp::gfx_batch::unprojectDepth(
+          renderer.cameraDepthUnprojection(sceneId),
+          depth.pixels<Mn::Float>().sliceSize(offset, size));
+    }
   }
 
   // Unprojected depth should read the distance between the cameras and the
   // plane, in meters. For each environment, the center pixel is compared with
   // the camera distance from origin.
-  auto errorMargin = Corrade::TestSuite::Compare::around(0.01f);
   // Target 0 is at 2.5 meters from the camera
-  CORRADE_COMPARE_WITH(depth.pixels<Mn::Float>()[32][32], 2.5f, errorMargin);
+  CORRADE_COMPARE_WITH(depth.pixels<Mn::Float>()[32][32], 2.5f,
+                       Corrade::TestSuite::Compare::around(0.01f));
   // Target 1 is at 5.0 meters from the camera
-  CORRADE_COMPARE_WITH(depth.pixels<Mn::Float>()[32][96], 5.0f, errorMargin);
+  CORRADE_COMPARE_WITH(depth.pixels<Mn::Float>()[32][96], 5.0f,
+                       Corrade::TestSuite::Compare::around(0.01f));
   // Target 2 is at 7.5 meters from the camera
-  CORRADE_COMPARE_WITH(depth.pixels<Mn::Float>()[96][32], 7.5f, errorMargin);
+  CORRADE_COMPARE_WITH(depth.pixels<Mn::Float>()[96][32], 7.5f,
+                       Corrade::TestSuite::Compare::around(0.01f));
   // Target 3 is beyond the far plane. Here, unprojected depth is set to 0.
   CORRADE_COMPARE(depth.pixels<Mn::Float>()[96][96], 0.0f);
 }