Add depth unprojection support to batch renderer

CONTRIBUTING.md

@@ -55,7 +55,7 @@ We also use pre-commit hooks to ensure linting and style enforcement. Install th
 
 ## Documentation
 - Our documentation style is based on Magnum / Corrade and uses [a similar build system](https://mcss.mosra.cz/documentation/doxygen/).
-- A good example of the documentation style is in esp::gfx::DepthUnprojection (DepthUnprojection.h).
+- The gfx_batch library is a good example of the documentation style.
 - Documentation of PRs is highly encouraged!
 
 ## Development Tips

src/cmake/FindMagnumPlugins.cmake

@@ -446,7 +446,7 @@ foreach(_component ${MagnumPlugins_FIND_COMPONENTS})
             # config if appropriate
             find_package(OpenEXR REQUIRED MODULE)
             set_property(TARGET MagnumPlugins::${_component} APPEND PROPERTY
-                INTERFACE_LINK_LIBRARIES OpenEXR::IlmImf)
+                INTERFACE_LINK_LIBRARIES OpenEXR::OpenEXR)
 
         # No special setup for the OpenDdl library
         # OpenGexImporter has no dependencies

src/esp/gfx/CMakeLists.txt

@@ -6,8 +6,6 @@ set(
   gfx_SOURCES
   CubeMap.cpp
   CubeMap.h
-  DepthUnprojection.cpp
-  DepthUnprojection.h
   Drawable.cpp
   Drawable.h
   DrawableGroup.cpp
@@ -108,10 +106,8 @@ find_package(MagnumPlugins REQUIRED GltfImporter StbImageImporter StbImageConver
 find_package(MagnumIntegration REQUIRED Eigen)
 
 find_package(Corrade REQUIRED Utility)
-# TODO: enable the following flag and fix the compilation warnings
-# set_directory_properties(PROPERTIES CORRADE_USE_PEDANTIC_FLAGS ON)
-corrade_add_resource(ShaderResources ../../shaders/Shaders.conf)
-list(APPEND gfx_SOURCES ${ShaderResources})
+corrade_add_resource(GfxShaderResources ../../shaders/gfx/Shaders.conf)
+list(APPEND gfx_SOURCES ${GfxShaderResources})
 
 corrade_add_resource(PbrImageResources ../../../data/pbr/PbrImages.conf)
 list(APPEND gfx_SOURCES ${PbrImageResources})
@@ -122,14 +118,6 @@ add_library(
 )
 
 if(BUILD_WITH_CUDA)
-  # We currently don't directly depend on gfx_batch for anything, just taking
-  # the header-only CUDA helpers from there, so gfx_batch isn't in
-  # target_link_libraries().
-  target_include_directories(
-    gfx PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}
-                ${CMAKE_CURRENT_LIST_DIR}/../gfx_batch/cuda_helpers
-  )
-
   target_link_libraries(gfx PUBLIC ${CUDART_LIBRARY})
 endif()
 
@@ -138,6 +126,7 @@ target_link_libraries(
   PUBLIC assets
          core
          io
+         gfx_batch
          physics
          Magnum::AnyImageImporter
          Magnum::AnySceneImporter

src/esp/gfx/DoubleSphereCameraShader.cpp

@@ -17,7 +17,7 @@
 // compiled into static library, it must be explicitly initialized via this
 // macro, and should be called *outside* of any namespace.
 static void importShaderResources() {
-  CORRADE_RESOURCE_INITIALIZE(ShaderResources)
+  CORRADE_RESOURCE_INITIALIZE(GfxShaderResources)
 }
 
 namespace Mn = Magnum;
@@ -28,7 +28,7 @@ namespace gfx {
 DoubleSphereCameraShader::DoubleSphereCameraShader(
     CubeMapShaderBase::Flags flags)
     : CubeMapShaderBase(flags) {
-  if (!Cr::Utility::Resource::hasGroup("default-shaders")) {
+  if (!Cr::Utility::Resource::hasGroup("gfx-shaders")) {
     importShaderResources();
   }
 
@@ -40,7 +40,7 @@ DoubleSphereCameraShader::DoubleSphereCameraShader(
 
   // this is not the file name, but the group name in the config file
   // see Shaders.conf in the shaders folder
-  const Cr::Utility::Resource rs{"default-shaders"};
+  const Cr::Utility::Resource rs{"gfx-shaders"};
 
   Mn::GL::Shader vert{glVersion, Mn::GL::Shader::Type::Vertex};
   Mn::GL::Shader frag{glVersion, Mn::GL::Shader::Type::Fragment};

src/esp/gfx/EquirectangularShader.cpp

@@ -18,7 +18,7 @@
 // When the resource is compiled into static library, it must be explicitly
 // initialized via this macro, and should be called *outside* of any namespace.
 static void importShaderResources() {
-  CORRADE_RESOURCE_INITIALIZE(ShaderResources)
+  CORRADE_RESOURCE_INITIALIZE(GfxShaderResources)
 }
 
 namespace Mn = Magnum;
@@ -33,7 +33,7 @@ EquirectangularShader::EquirectangularShader(Flags flags)
                  "EquirectangularShader::EquirectangularShader(): shader "
                  "flags cannot be empty.", );
 
-  if (!Cr::Utility::Resource::hasGroup("default-shaders")) {
+  if (!Cr::Utility::Resource::hasGroup("gfx-shaders")) {
     importShaderResources();
   }
 
@@ -45,7 +45,7 @@ EquirectangularShader::EquirectangularShader(Flags flags)
 
   // this is not the file name, but the group name in the config file
   // see Shaders.conf in the shaders folder
-  const Cr::Utility::Resource rs{"default-shaders"};
+  const Cr::Utility::Resource rs{"gfx-shaders"};
 
   Mn::GL::Shader vert{glVersion, Mn::GL::Shader::Type::Vertex};
   Mn::GL::Shader frag{glVersion, Mn::GL::Shader::Type::Fragment};

src/esp/gfx/GaussianFilterShader.cpp

@@ -18,7 +18,7 @@ namespace Cr = Corrade;
 namespace Mn = Magnum;
 
 static void importShaderResources() {
-  CORRADE_RESOURCE_INITIALIZE(ShaderResources)
+  CORRADE_RESOURCE_INITIALIZE(GfxShaderResources)
 }
 
 namespace esp {
@@ -29,11 +29,11 @@ enum {
 };
 
 GaussianFilterShader::GaussianFilterShader() {
-  if (!Corrade::Utility::Resource::hasGroup("default-shaders")) {
+  if (!Corrade::Utility::Resource::hasGroup("gfx-shaders")) {
     importShaderResources();
   }
 
-  const Corrade::Utility::Resource rs{"default-shaders"};
+  const Corrade::Utility::Resource rs{"gfx-shaders"};
 
 #ifdef MAGNUM_TARGET_WEBGL
   Mn::GL::Version glVersion = Mn::GL::Version::GLES300;

src/esp/gfx/PTexMeshShader.cpp

@@ -20,7 +20,7 @@
 // compiled into static library, it must be explicitly initialized via this
 // macro, and should be called // *outside* of any namespace.
 static void importShaderResources() {
-  CORRADE_RESOURCE_INITIALIZE(ShaderResources)
+  CORRADE_RESOURCE_INITIALIZE(GfxShaderResources)
 }
 
 namespace Mn = Magnum;
@@ -40,12 +40,12 @@ enum TextureBindingPointIndex : uint8_t {
 PTexMeshShader::PTexMeshShader() {
   MAGNUM_ASSERT_GL_VERSION_SUPPORTED(Mn::GL::Version::GL410);
 
-  if (!Corrade::Utility::Resource::hasGroup("default-shaders")) {
+  if (!Corrade::Utility::Resource::hasGroup("gfx-shaders")) {
     importShaderResources();
   }
 
   // this is not the file name, but the group name in the config file
-  const Corrade::Utility::Resource rs{"default-shaders"};
+  const Corrade::Utility::Resource rs{"gfx-shaders"};
 
   Mn::GL::Shader vert{Mn::GL::Version::GL410, Mn::GL::Shader::Type::Vertex};
   Mn::GL::Shader geom{Mn::GL::Version::GL410, Mn::GL::Shader::Type::Geometry};

src/esp/gfx/PbrEquiRectangularToCubeMapShader.cpp

@@ -19,7 +19,7 @@
 // compiled into static library, it must be explicitly initialized via this
 // macro, and should be called *outside* of any namespace.
 static void importShaderResources() {
-  CORRADE_RESOURCE_INITIALIZE(ShaderResources)
+  CORRADE_RESOURCE_INITIALIZE(GfxShaderResources)
 }
 
 namespace Mn = Magnum;
@@ -33,7 +33,7 @@ enum {
 };
 
 PbrEquiRectangularToCubeMapShader::PbrEquiRectangularToCubeMapShader() {
-  if (!Corrade::Utility::Resource::hasGroup("default-shaders")) {
+  if (!Corrade::Utility::Resource::hasGroup("gfx-shaders")) {
     importShaderResources();
   }
 
@@ -45,7 +45,7 @@ PbrEquiRectangularToCubeMapShader::PbrEquiRectangularToCubeMapShader() {
 
   // this is not the file name, but the group name in the config file
   // see Shaders.conf in the shaders folder
-  const Cr::Utility::Resource rs{"default-shaders"};
+  const Cr::Utility::Resource rs{"gfx-shaders"};
 
   Mn::GL::Shader vert{glVersion, Mn::GL::Shader::Type::Vertex};
   Mn::GL::Shader frag{glVersion, Mn::GL::Shader::Type::Fragment};

src/esp/gfx/PbrPrecomputedMapShader.cpp

@@ -25,7 +25,7 @@
 // compiled into static library, it must be explicitly initialized via this
 // macro, and should be called *outside* of any namespace.
 static void importShaderResources() {
-  CORRADE_RESOURCE_INITIALIZE(ShaderResources)
+  CORRADE_RESOURCE_INITIALIZE(GfxShaderResources)
 }
 
 namespace Mn = Magnum;
@@ -41,7 +41,7 @@ PbrPrecomputedMapShader::PbrPrecomputedMapShader(Flags flags) : flags_(flags) {
       "Flag::IrradianceMap and "
       "Flag::PrefilteredMap are mutually exclusive.", );
 
-  if (!Corrade::Utility::Resource::hasGroup("default-shaders")) {
+  if (!Corrade::Utility::Resource::hasGroup("gfx-shaders")) {
     importShaderResources();
   }
 
@@ -53,7 +53,7 @@ PbrPrecomputedMapShader::PbrPrecomputedMapShader(Flags flags) : flags_(flags) {
 
   // this is not the file name, but the group name in the config file
   // see Shaders.conf in the shaders folder
-  const Cr::Utility::Resource rs{"default-shaders"};
+  const Cr::Utility::Resource rs{"gfx-shaders"};
 
   Mn::GL::Shader vert{glVersion, Mn::GL::Shader::Type::Vertex};
   Mn::GL::Shader frag{glVersion, Mn::GL::Shader::Type::Fragment};

src/esp/gfx/PbrShader.cpp

@@ -30,7 +30,7 @@
 // compiled into static library, it must be explicitly initialized via this
 // macro, and should be called *outside* of any namespace.
 static void importShaderResources() {
-  CORRADE_RESOURCE_INITIALIZE(ShaderResources)
+  CORRADE_RESOURCE_INITIALIZE(GfxShaderResources)
 }
 
 namespace Mn = Magnum;
@@ -41,7 +41,7 @@ namespace gfx {
 
 PbrShader::PbrShader(Flags originalFlags, unsigned int lightCount)
     : flags_(originalFlags), lightCount_(lightCount) {
-  if (!Cr::Utility::Resource::hasGroup("default-shaders")) {
+  if (!Cr::Utility::Resource::hasGroup("gfx-shaders")) {
     importShaderResources();
   }
 
@@ -74,7 +74,7 @@ PbrShader::PbrShader(Flags originalFlags, unsigned int lightCount)
 
   // this is not the file name, but the group name in the config file
   // see Shaders.conf in the shaders folder
-  const Cr::Utility::Resource rs{"default-shaders"};
+  const Cr::Utility::Resource rs{"gfx-shaders"};
 
   Mn::GL::Shader vert{glVersion, Mn::GL::Shader::Type::Vertex};
   Mn::GL::Shader frag{glVersion, Mn::GL::Shader::Type::Fragment};

src/esp/gfx/RenderTarget.cpp

@@ -22,7 +22,7 @@
 #include "RenderTarget.h"
 #include "esp/sensor/VisualSensor.h"
 
-#include "esp/gfx/DepthUnprojection.h"
+#include "esp/gfx_batch/DepthUnprojection.h"
 
 #ifdef ESP_BUILD_WITH_CUDA
 #include <cuda_gl_interop.h>
@@ -46,7 +46,7 @@ const Mn::GL::Framebuffer::ColorAttachment UnprojectedDepthBufferAttachment =
 struct RenderTarget::Impl {
   Impl(const Mn::Vector2i& size,
        const Mn::Vector2& depthUnprojection,
-       DepthShader* depthShader,
+       gfx_batch::DepthShader* depthShader,
        Flags flags,
        const sensor::VisualSensor* visualSensor)
       : colorBuffer_{},
@@ -61,8 +61,9 @@ struct RenderTarget::Impl {
         flags_{flags},
         visualSensor_{visualSensor} {
     if (depthShader_) {
-      CORRADE_INTERNAL_ASSERT(depthShader_->flags() &
-                              DepthShader::Flag::UnprojectExistingDepth);
+      CORRADE_INTERNAL_ASSERT(
+          depthShader_->flags() &
+          gfx_batch::DepthShader::Flag::UnprojectExistingDepth);
     }
 
     if (flags_ & Flag::RgbaAttachment) {
@@ -148,7 +149,7 @@ struct RenderTarget::Impl {
     CORRADE_INTERNAL_ASSERT(depthShader_ != nullptr);
     CORRADE_ASSERT(
         flags_ & Flag::DepthTextureAttachment,
-        "RenderTarget::Impl::unporojectDepthGPU(): this render target "
+        "RenderTarget::Impl::unprojectDepthGPU(): this render target "
         "was not created with depth texture enabled.", );
     initDepthUnprojector();
 
@@ -220,8 +221,7 @@ struct RenderTarget::Impl {
           Mn::GL::PixelFormat::DepthComponent, Mn::GL::PixelType::Float,
           view.size(), view.data()};
       framebuffer_.read(framebuffer_.viewport(), depthBufferView);
-      unprojectDepth(depthUnprojection_,
-                     Cr::Containers::arrayCast<Mn::Float>(view.data()));
+      gfx_batch::unprojectDepth(depthUnprojection_, view.pixels<Mn::Float>());
     }
   }
 
@@ -353,7 +353,7 @@ struct RenderTarget::Impl {
   Mn::GL::Framebuffer framebuffer_;
 
   Mn::Vector2 depthUnprojection_;
-  DepthShader* depthShader_;
+  gfx_batch::DepthShader* depthShader_;
   Mn::GL::Renderbuffer unprojectedDepth_;
   Mn::GL::Mesh depthUnprojectionMesh_;
   Mn::GL::Framebuffer depthUnprojectionFrameBuffer_;
@@ -371,7 +371,7 @@ struct RenderTarget::Impl {
 
 RenderTarget::RenderTarget(const Mn::Vector2i& size,
                            const Mn::Vector2& depthUnprojection,
-                           DepthShader* depthShader,
+                           gfx_batch::DepthShader* depthShader,
                            Flags flags,
                            const sensor::VisualSensor* visualSensor)
     : pimpl_(spimpl::make_unique_impl<Impl>(size,

src/esp/gfx/RenderTarget.h

@@ -10,14 +10,16 @@
 
 #include "esp/core/Esp.h"
 
-#include "esp/gfx/DepthUnprojection.h"
 #include "esp/gfx/Renderer.h"
 
 namespace esp {
 
 namespace sensor {
 class VisualSensor;
 }
+namespace gfx_batch {
+class DepthShader;
+}
 
 namespace gfx {
 
@@ -70,15 +72,15 @@ class RenderTarget {
    */
   RenderTarget(const Magnum::Vector2i& size,
                const Magnum::Vector2& depthUnprojection,
-               DepthShader* depthShader,
+               gfx_batch::DepthShader* depthShader,
                Flags flags = {Flag::RgbaAttachment | Flag::ObjectIdAttachment |
                               Flag::DepthTextureAttachment},
                const sensor::VisualSensor* visualSensor = nullptr);
 
   /**
    * @brief Constructor
    * @param size               The size of the underlying framebuffers in WxH
-   * @param depthUnprojection  Depth unrpojection parameters.  See @ref
+   * @param depthUnprojection  Depth unprojection parameters.  See @ref
    *                           calculateDepthUnprojection()
    * @param visualSensor       (optional) The visual sensor for this render
    * target

src/esp/gfx/Renderer.cpp

@@ -22,10 +22,10 @@
 #include <Magnum/ResourceManager.h>
 
 #include "esp/core/Check.h"
-#include "esp/gfx/DepthUnprojection.h"
 #include "esp/gfx/GaussianFilterShader.h"
 #include "esp/gfx/RenderTarget.h"
 #include "esp/gfx/TextureVisualizerShader.h"
+#include "esp/gfx_batch/DepthUnprojection.h"
 #include "esp/sensor/VisualSensor.h"
 #include "esp/sim/Simulator.h"
 
@@ -336,8 +336,8 @@ struct Renderer::Impl {
                    "depthUnprojection matrix", );
 
     if (!depthShader_) {
-      depthShader_ = std::make_unique<DepthShader>(
-          DepthShader::Flag::UnprojectExistingDepth);
+      depthShader_ = std::make_unique<gfx_batch::DepthShader>(
+          gfx_batch::DepthShader::Flag::UnprojectExistingDepth);
     }
 
     RenderTarget::Flags renderTargetFlags = {};
@@ -381,7 +381,7 @@ struct Renderer::Impl {
   WindowlessContext* context_;
   bool contextIsOwned_ = true;
   // TODO: shall we use shader resource manager from now?
-  std::unique_ptr<DepthShader> depthShader_;
+  std::unique_ptr<gfx_batch::DepthShader> depthShader_;
   const Flags flags_;
 #ifdef ESP_BUILD_WITH_BACKGROUND_RENDERER
   std::unique_ptr<BackgroundRenderer> backgroundRenderer_ = nullptr;
@@ -431,7 +431,8 @@ struct Renderer::Impl {
       if (type == RendererShaderType::DepthShader) {
         shaderManager_.set<Mn::GL::AbstractShaderProgram>(
             shader.key(),
-            new DepthShader{DepthShader::Flag::UnprojectExistingDepth},
+            new gfx_batch::DepthShader{
+                gfx_batch::DepthShader::Flag::UnprojectExistingDepth},
             Mn::ResourceDataState::Final, Mn::ResourcePolicy::Resident);
       } else if (type == RendererShaderType::DepthTextureVisualizer) {
         shaderManager_.set<Mn::GL::AbstractShaderProgram>(