From a2a79fefa4057cac3a2bb9d4e65fa8cf88652783 Mon Sep 17 00:00:00 2001 From: Mikael Dallaire Cote <110583667+0mdc@users.noreply.github.com> Date: Wed, 22 Mar 2023 17:14:24 -0400 Subject: [PATCH 01/10] Add API to render any sensor type (GPU-to-CPU). --- src/esp/bindings/SimBindings.cpp | 7 ++++--- src/esp/sim/AbstractReplayRenderer.cpp | 23 ++++++++++++++------ src/esp/sim/AbstractReplayRenderer.h | 10 ++++++--- src/esp/sim/BatchReplayRenderer.cpp | 17 ++++++++++++--- src/esp/sim/BatchReplayRenderer.h | 4 +++- src/esp/sim/ClassicReplayRenderer.cpp | 29 ++++++++++++++++++++++---- src/esp/sim/ClassicReplayRenderer.h | 4 +++- src/tests/BatchReplayRendererTest.cpp | 4 ++-- 8 files changed, 75 insertions(+), 23 deletions(-) diff --git a/src/esp/bindings/SimBindings.cpp b/src/esp/bindings/SimBindings.cpp index 80bb9e20fb..82580b557b 100644 --- a/src/esp/bindings/SimBindings.cpp +++ b/src/esp/bindings/SimBindings.cpp @@ -425,10 +425,11 @@ void initSimBindings(py::module& m) { .def( "render", [](AbstractReplayRenderer& self, - std::vector images) { - self.render(images); + std::vector colorImageViews, + std::vector depthImageViews) { + self.render(colorImageViews, depthImageViews); }, - R"(Render color sensors into the specified image vector (one per environment). + R"(TODO Render color sensors into the specified image vector (one per environment). The images are required to be pre-allocated. Blocks the thread during the GPU-to-CPU memory transfer operation.)") .def( diff --git a/src/esp/sim/AbstractReplayRenderer.cpp b/src/esp/sim/AbstractReplayRenderer.cpp index a09c05c01a..00320c3b3e 100644 --- a/src/esp/sim/AbstractReplayRenderer.cpp +++ b/src/esp/sim/AbstractReplayRenderer.cpp @@ -91,12 +91,23 @@ void AbstractReplayRenderer::setSensorTransformsFromKeyframe( } void AbstractReplayRenderer::render( - Cr::Containers::ArrayView imageViews) { - CORRADE_ASSERT(imageViews.size() == doEnvironmentCount(), - "ReplayRenderer::render(): expected" << doEnvironmentCount() - << "image views but got" - << imageViews.size(), ); - return doRender(imageViews); + Corrade::Containers::ArrayView + colorImageViews, + Corrade::Containers::ArrayView + depthImageViews) { + if (colorImageViews != nullptr) { + CORRADE_ASSERT(colorImageViews.size() == doEnvironmentCount(), + "ReplayRenderer::render(): expected" + << doEnvironmentCount() << "color image views but got" + << colorImageViews.size(), ); + } + if (depthImageViews != nullptr) { + CORRADE_ASSERT(depthImageViews.size() == doEnvironmentCount(), + "ReplayRenderer::render(): expected" + << doEnvironmentCount() << "depth image views but got" + << depthImageViews.size(), ); + } + return doRender(colorImageViews, depthImageViews); } void AbstractReplayRenderer::render( diff --git a/src/esp/sim/AbstractReplayRenderer.h b/src/esp/sim/AbstractReplayRenderer.h index ef24a8f27f..27659ae02f 100644 --- a/src/esp/sim/AbstractReplayRenderer.h +++ b/src/esp/sim/AbstractReplayRenderer.h @@ -95,9 +95,11 @@ class AbstractReplayRenderer { void setSensorTransformsFromKeyframe(unsigned envIndex, const std::string& prefix); - // Renders and waits for the render to finish + // TODO Renders and waits for the render to finish void render(Corrade::Containers::ArrayView - imageViews); + colorImageViews, + Corrade::Containers::ArrayView + depthImageViews); // Assumes the framebuffer color & depth is cleared void render(Magnum::GL::AbstractFramebuffer& framebuffer); @@ -140,7 +142,9 @@ class AbstractReplayRenderer { /* imageViews.size() is guaranteed to be same as doEnvironmentCount() */ virtual void doRender( Corrade::Containers::ArrayView - imageViews) = 0; + colorImageViews, + Corrade::Containers::ArrayView + depthImageViews) = 0; virtual void doRender(Magnum::GL::AbstractFramebuffer& framebuffer) = 0; diff --git a/src/esp/sim/BatchReplayRenderer.cpp b/src/esp/sim/BatchReplayRenderer.cpp index 871e4ee70b..22cabd8ce7 100644 --- a/src/esp/sim/BatchReplayRenderer.cpp +++ b/src/esp/sim/BatchReplayRenderer.cpp @@ -4,6 +4,7 @@ #include "BatchReplayRenderer.h" +#include "esp/core/Logging.h" #include "esp/sensor/CameraSensor.h" #include @@ -251,7 +252,10 @@ void BatchReplayRenderer::doSetSensorTransformsFromKeyframe( } void BatchReplayRenderer::doRender( - Cr::Containers::ArrayView imageViews) { + Corrade::Containers::ArrayView + colorImageViews, + Corrade::Containers::ArrayView + depthImageViews) { CORRADE_ASSERT(standalone_, "BatchReplayRenderer::render(): can use this function only " "with a standalone renderer", ); @@ -263,8 +267,15 @@ void BatchReplayRenderer::doRender( Mn::Vector2i{envIndex % renderer_->tileCount().x(), envIndex / renderer_->tileCount().x()}, renderer_->tileSize()); - static_cast(*renderer_) - .colorImageInto(rectangle, imageViews[envIndex]); + + if (colorImageViews) { + static_cast(*renderer_) + .colorImageInto(rectangle, colorImageViews[envIndex]); + } + if (depthImageViews) { + static_cast(*renderer_) + .depthImageInto(rectangle, depthImageViews[envIndex]); + } } } diff --git a/src/esp/sim/BatchReplayRenderer.h b/src/esp/sim/BatchReplayRenderer.h index 7f113b591b..7627aec421 100644 --- a/src/esp/sim/BatchReplayRenderer.h +++ b/src/esp/sim/BatchReplayRenderer.h @@ -45,7 +45,9 @@ class BatchReplayRenderer : public AbstractReplayRenderer { const std::string& prefix) override; void doRender(Corrade::Containers::ArrayView - imageViews) override; + colorImageViews, + Corrade::Containers::ArrayView + depthImageViews) override; void doRender(Magnum::GL::AbstractFramebuffer& framebuffer) override; diff --git a/src/esp/sim/ClassicReplayRenderer.cpp b/src/esp/sim/ClassicReplayRenderer.cpp index 67fa483e5b..997be28fd2 100644 --- a/src/esp/sim/ClassicReplayRenderer.cpp +++ b/src/esp/sim/ClassicReplayRenderer.cpp @@ -8,6 +8,7 @@ #include "esp/gfx/RenderTarget.h" #include "esp/gfx/Renderer.h" #include "esp/metadata/MetadataMediator.h" +#include "esp/sensor/Sensor.h" #include "esp/sensor/SensorFactory.h" #include "esp/sim/SimulatorConfiguration.h" @@ -197,7 +198,10 @@ void ClassicReplayRenderer::doSetSensorTransformsFromKeyframe( } void ClassicReplayRenderer::doRender( - Cr::Containers::ArrayView imageViews) { + Corrade::Containers::ArrayView + colorImageViews, + Corrade::Containers::ArrayView + depthImageViews) { for (int envIndex = 0; envIndex < config_.numEnvironments; envIndex++) { auto& sensorMap = getEnvironmentSensors(envIndex); CORRADE_INTERNAL_ASSERT(sensorMap.size() == 1); @@ -208,11 +212,28 @@ void ClassicReplayRenderer::doRender( auto& sceneGraph = getSceneGraph(envIndex); + auto& sensorType = visualSensor.specification()->sensorType; + Corrade::Containers::ArrayView + imageViews; + switch (sensorType) { + case esp::sensor::SensorType::Color: + imageViews = colorImageViews; + break; + case esp::sensor::SensorType::Depth: + imageViews = depthImageViews; + break; + default: + imageViews = nullptr; + break; + } + #ifdef ESP_BUILD_WITH_BACKGROUND_RENDERER // todo: investigate flags (frustum culling?) - renderer_->enqueueAsyncDrawJob(visualSensor, sceneGraph, - imageViews[envIndex], - esp::gfx::RenderCamera::Flags{}); + if (imageViews != nullptr) { + renderer_->enqueueAsyncDrawJob(visualSensor, sceneGraph, + imageViews[envIndex], + esp::gfx::RenderCamera::Flags{}); + } #else // TODO what am I supposed to do here? CORRADE_ASSERT_UNREACHABLE("Not implemented yet, sorry.", ); diff --git a/src/esp/sim/ClassicReplayRenderer.h b/src/esp/sim/ClassicReplayRenderer.h index c2955337e5..780b329187 100644 --- a/src/esp/sim/ClassicReplayRenderer.h +++ b/src/esp/sim/ClassicReplayRenderer.h @@ -71,7 +71,9 @@ class ClassicReplayRenderer : public AbstractReplayRenderer { const std::string& prefix) override; void doRender(Corrade::Containers::ArrayView - imageViews) override; + colorImageViews, + Corrade::Containers::ArrayView + depthImageViews) override; void doRender(Magnum::GL::AbstractFramebuffer& framebuffer) override; diff --git a/src/tests/BatchReplayRendererTest.cpp b/src/tests/BatchReplayRendererTest.cpp index 6f5cd4095c..a4635dd854 100644 --- a/src/tests/BatchReplayRendererTest.cpp +++ b/src/tests/BatchReplayRendererTest.cpp @@ -181,8 +181,8 @@ void BatchReplayRendererTest::testIntegration() { renderer->setEnvironmentKeyframe(envIndex, serKeyframes[envIndex]); renderer->setSensorTransformsFromKeyframe(envIndex, userPrefix); } - - renderer->render(imageViews); + + renderer->render(imageViews, nullptr); for (int envIndex = 0; envIndex < numEnvs; envIndex++) { CORRADE_ITERATION(envIndex); From 329dc3c9093f8d022c6f3144291b7d0339adf8b9 Mon Sep 17 00:00:00 2001 From: Mikael Dallaire Cote <110583667+0mdc@users.noreply.github.com> Date: Wed, 22 Mar 2023 18:22:22 -0400 Subject: [PATCH 02/10] Format fix. --- src/tests/BatchReplayRendererTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/BatchReplayRendererTest.cpp b/src/tests/BatchReplayRendererTest.cpp index a4635dd854..75fe669fe8 100644 --- a/src/tests/BatchReplayRendererTest.cpp +++ b/src/tests/BatchReplayRendererTest.cpp @@ -181,7 +181,7 @@ void BatchReplayRendererTest::testIntegration() { renderer->setEnvironmentKeyframe(envIndex, serKeyframes[envIndex]); renderer->setSensorTransformsFromKeyframe(envIndex, userPrefix); } - + renderer->render(imageViews, nullptr); for (int envIndex = 0; envIndex < numEnvs; envIndex++) { From e06d3eff853906af10b7f410e1ac17826b6833cd Mon Sep 17 00:00:00 2001 From: Mikael Dallaire Cote <110583667+0mdc@users.noreply.github.com> Date: Wed, 22 Mar 2023 18:35:47 -0400 Subject: [PATCH 03/10] Change function docs. --- src/esp/bindings/SimBindings.cpp | 7 ++++--- src/esp/sim/AbstractReplayRenderer.h | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/esp/bindings/SimBindings.cpp b/src/esp/bindings/SimBindings.cpp index 82580b557b..f81bc32435 100644 --- a/src/esp/bindings/SimBindings.cpp +++ b/src/esp/bindings/SimBindings.cpp @@ -429,9 +429,10 @@ void initSimBindings(py::module& m) { std::vector depthImageViews) { self.render(colorImageViews, depthImageViews); }, - R"(TODO Render color sensors into the specified image vector (one per environment). - The images are required to be pre-allocated. - Blocks the thread during the GPU-to-CPU memory transfer operation.)") + R"(Render sensors into the specified image vectors (one per environment). + Blocks the thread during the GPU-to-CPU memory transfer operation. + Empty lists can be supplied to skip the copying render targets. + The images are required to be pre-allocated.)") .def( "set_sensor_transforms_from_keyframe", &AbstractReplayRenderer::setSensorTransformsFromKeyframe, diff --git a/src/esp/sim/AbstractReplayRenderer.h b/src/esp/sim/AbstractReplayRenderer.h index 27659ae02f..7c3ed3e179 100644 --- a/src/esp/sim/AbstractReplayRenderer.h +++ b/src/esp/sim/AbstractReplayRenderer.h @@ -95,7 +95,8 @@ class AbstractReplayRenderer { void setSensorTransformsFromKeyframe(unsigned envIndex, const std::string& prefix); - // TODO Renders and waits for the render to finish + // Renders into the specified CPU-resident image view arrays (one image per + // environment). Waits for the render to finish. void render(Corrade::Containers::ArrayView colorImageViews, Corrade::Containers::ArrayView From 3919c75d8e94d35465caec084b9be9eb79af6277 Mon Sep 17 00:00:00 2001 From: Mikael Dallaire Cote <110583667+0mdc@users.noreply.github.com> Date: Wed, 22 Mar 2023 19:10:14 -0400 Subject: [PATCH 04/10] Change CORRADE_ASSERT by ESP_CHECK. --- src/esp/sim/AbstractReplayRenderer.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/esp/sim/AbstractReplayRenderer.cpp b/src/esp/sim/AbstractReplayRenderer.cpp index 00320c3b3e..a40add48f3 100644 --- a/src/esp/sim/AbstractReplayRenderer.cpp +++ b/src/esp/sim/AbstractReplayRenderer.cpp @@ -96,16 +96,16 @@ void AbstractReplayRenderer::render( Corrade::Containers::ArrayView depthImageViews) { if (colorImageViews != nullptr) { - CORRADE_ASSERT(colorImageViews.size() == doEnvironmentCount(), - "ReplayRenderer::render(): expected" - << doEnvironmentCount() << "color image views but got" - << colorImageViews.size(), ); + ESP_CHECK(colorImageViews.size() == doEnvironmentCount(), + "ReplayRenderer::render(): expected" + << doEnvironmentCount() << "color image views but got" + << colorImageViews.size()); } if (depthImageViews != nullptr) { - CORRADE_ASSERT(depthImageViews.size() == doEnvironmentCount(), - "ReplayRenderer::render(): expected" - << doEnvironmentCount() << "depth image views but got" - << depthImageViews.size(), ); + ESP_CHECK(depthImageViews.size() == doEnvironmentCount(), + "ReplayRenderer::render(): expected" + << doEnvironmentCount() << "depth image views but got" + << depthImageViews.size()); } return doRender(colorImageViews, depthImageViews); } From dfc6e05a5ea59cb8ad852d96c9f08fbf9bde0b92 Mon Sep 17 00:00:00 2001 From: Mikael Dallaire Cote <110583667+0mdc@users.noreply.github.com> Date: Thu, 23 Mar 2023 14:58:06 -0400 Subject: [PATCH 05/10] Remove unused include. --- src/esp/sim/BatchReplayRenderer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/esp/sim/BatchReplayRenderer.cpp b/src/esp/sim/BatchReplayRenderer.cpp index 22cabd8ce7..74f853403d 100644 --- a/src/esp/sim/BatchReplayRenderer.cpp +++ b/src/esp/sim/BatchReplayRenderer.cpp @@ -4,7 +4,6 @@ #include "BatchReplayRenderer.h" -#include "esp/core/Logging.h" #include "esp/sensor/CameraSensor.h" #include From bc980bf6590424c1508b733f58e4c1476b5f2dab Mon Sep 17 00:00:00 2001 From: Mikael Dallaire Cote <110583667+0mdc@users.noreply.github.com> Date: Mon, 5 Jun 2023 21:31:59 -0400 Subject: [PATCH 06/10] Formatting change. --- src/esp/sim/ClassicReplayRenderer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/esp/sim/ClassicReplayRenderer.cpp b/src/esp/sim/ClassicReplayRenderer.cpp index 2f2c568316..b14fde89d5 100644 --- a/src/esp/sim/ClassicReplayRenderer.cpp +++ b/src/esp/sim/ClassicReplayRenderer.cpp @@ -234,10 +234,10 @@ void ClassicReplayRenderer::doRender( #ifdef ESP_BUILD_WITH_BACKGROUND_RENDERER // todo: investigate flags (frustum culling?) if (imageViews != nullptr) { - renderer_->enqueueAsyncDrawJob(visualSensor, sceneGraph, - imageViews[envIndex], - esp::gfx::RenderCamera::Flags{ - gfx::RenderCamera::Flag::FrustumCulling}); + renderer_->enqueueAsyncDrawJob( + visualSensor, sceneGraph, imageViews[envIndex], + esp::gfx::RenderCamera::Flags{ + gfx::RenderCamera::Flag::FrustumCulling}); } #else // TODO what am I supposed to do here? From 003b5c67f5c83625ed397a3760d9f65a8e0c5d3e Mon Sep 17 00:00:00 2001 From: Mikael Dallaire Cote <110583667+0mdc@users.noreply.github.com> Date: Mon, 5 Jun 2023 21:37:42 -0400 Subject: [PATCH 07/10] Formatting changes. --- src/esp/sim/AbstractReplayRenderer.cpp | 6 ++---- src/esp/sim/BatchReplayRenderer.cpp | 6 ++---- src/esp/sim/ClassicReplayRenderer.cpp | 9 +++------ 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/esp/sim/AbstractReplayRenderer.cpp b/src/esp/sim/AbstractReplayRenderer.cpp index 09be15fdd4..f0c57a923a 100644 --- a/src/esp/sim/AbstractReplayRenderer.cpp +++ b/src/esp/sim/AbstractReplayRenderer.cpp @@ -91,10 +91,8 @@ void AbstractReplayRenderer::setSensorTransformsFromKeyframe( } void AbstractReplayRenderer::render( - Corrade::Containers::ArrayView - colorImageViews, - Corrade::Containers::ArrayView - depthImageViews) { + Cr::Containers::ArrayView colorImageViews, + Cr::Containers::ArrayView depthImageViews) { if (colorImageViews != nullptr) { ESP_CHECK(colorImageViews.size() == doEnvironmentCount(), "ReplayRenderer::render(): expected" diff --git a/src/esp/sim/BatchReplayRenderer.cpp b/src/esp/sim/BatchReplayRenderer.cpp index aeda83a5bf..05053f9c2c 100644 --- a/src/esp/sim/BatchReplayRenderer.cpp +++ b/src/esp/sim/BatchReplayRenderer.cpp @@ -252,10 +252,8 @@ void BatchReplayRenderer::doSetSensorTransformsFromKeyframe( } void BatchReplayRenderer::doRender( - Corrade::Containers::ArrayView - colorImageViews, - Corrade::Containers::ArrayView - depthImageViews) { + Cr::Containers::ArrayView colorImageViews, + Cr::Containers::ArrayView depthImageViews) { CORRADE_ASSERT(standalone_, "BatchReplayRenderer::render(): can use this function only " "with a standalone renderer", ); diff --git a/src/esp/sim/ClassicReplayRenderer.cpp b/src/esp/sim/ClassicReplayRenderer.cpp index b14fde89d5..91e42c1747 100644 --- a/src/esp/sim/ClassicReplayRenderer.cpp +++ b/src/esp/sim/ClassicReplayRenderer.cpp @@ -202,10 +202,8 @@ void ClassicReplayRenderer::doSetSensorTransformsFromKeyframe( } void ClassicReplayRenderer::doRender( - Corrade::Containers::ArrayView - colorImageViews, - Corrade::Containers::ArrayView - depthImageViews) { + Cr::Containers::ArrayView colorImageViews, + Cr::Containers::ArrayView depthImageViews) { for (int envIndex = 0; envIndex < config_.numEnvironments; envIndex++) { auto& sensorMap = getEnvironmentSensors(envIndex); CORRADE_INTERNAL_ASSERT(sensorMap.size() == 1); @@ -217,8 +215,7 @@ void ClassicReplayRenderer::doRender( auto& sceneGraph = getSceneGraph(envIndex); auto& sensorType = visualSensor.specification()->sensorType; - Corrade::Containers::ArrayView - imageViews; + Cr::Containers::ArrayView imageViews; switch (sensorType) { case esp::sensor::SensorType::Color: imageViews = colorImageViews; From 9b5319a0e0161dabc2bdae7aff90a3724cd29a8f Mon Sep 17 00:00:00 2001 From: Mikael Dallaire Cote <110583667+0mdc@users.noreply.github.com> Date: Thu, 8 Jun 2023 18:32:41 -0400 Subject: [PATCH 08/10] Change replay renderer render api arguments to keywords. --- src/esp/bindings/SimBindings.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/esp/bindings/SimBindings.cpp b/src/esp/bindings/SimBindings.cpp index f444e9ee64..df21e32b79 100644 --- a/src/esp/bindings/SimBindings.cpp +++ b/src/esp/bindings/SimBindings.cpp @@ -445,7 +445,9 @@ void initSimBindings(py::module& m) { R"(Render sensors into the specified image vectors (one per environment). Blocks the thread during the GPU-to-CPU memory transfer operation. Empty lists can be supplied to skip the copying render targets. - The images are required to be pre-allocated.)") + The images are required to be pre-allocated.)", + py::arg("color_images") = std::vector{}, + py::arg("depth_images") = std::vector{}) .def( "set_sensor_transforms_from_keyframe", &AbstractReplayRenderer::setSensorTransformsFromKeyframe, From cc28e0724b88e1752896603a268ac25fe0a4cdd7 Mon Sep 17 00:00:00 2001 From: Mikael Dallaire Cote <110583667+0mdc@users.noreply.github.com> Date: Thu, 8 Jun 2023 19:11:19 -0400 Subject: [PATCH 09/10] Add test for depth sensors. --- src/tests/BatchReplayRendererTest.cpp | 156 ++++++++++++++++++++------ 1 file changed, 122 insertions(+), 34 deletions(-) diff --git a/src/tests/BatchReplayRendererTest.cpp b/src/tests/BatchReplayRendererTest.cpp index da92f4fedd..d9a6f01bbf 100644 --- a/src/tests/BatchReplayRendererTest.cpp +++ b/src/tests/BatchReplayRendererTest.cpp @@ -2,6 +2,7 @@ // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. +#include "Corrade/Containers/EnumSet.h" #include "Corrade/Utility/Assert.h" #include "Magnum/DebugTools/Screenshot.h" #include "Magnum/GL/Context.h" @@ -40,6 +41,9 @@ namespace { const std::string screenshotDir = Cr::Utility::Path::join(TEST_ASSETS, "screenshots/"); +enum class TestFlag : Magnum::UnsignedInt { Color = 1 << 0, Depth = 1 << 1 }; +typedef Corrade::Containers::EnumSet TestFlags; + struct BatchReplayRendererTest : Cr::TestSuite::Tester { explicit BatchReplayRendererTest(); @@ -66,16 +70,47 @@ Mn::MutableImageView2D getRGBView(int width, return view; } -std::vector getDefaultSensorSpecs( - const std::string& sensorName = "my_rgb") { +Mn::MutableImageView2D getDepthView(int width, + int height, + std::vector& buffer, + bool classic) { + Mn::Vector2i size(width, height); + constexpr int pixelSize = 4; + + buffer.resize(std::size_t(width * height * pixelSize)); + + // BEWARE: Classic renderer requires R32F because the depth is unprojected. + // Batch renderer directly returns the depth buffer at the moment. + auto pixelFormat = + classic ? Mn::PixelFormat::R32F : Mn::PixelFormat::Depth32F; + auto view = Mn::MutableImageView2D(pixelFormat, size, buffer); + + return view; +} + +esp::sensor::SensorSpec::ptr getDefaultSensorSpecs( + const std::string& sensorName, + const esp::sensor::SensorType sensorType) { auto pinholeCameraSpec = esp::sensor::CameraSensorSpec::create(); pinholeCameraSpec->sensorSubType = esp::sensor::SensorSubType::Pinhole; - pinholeCameraSpec->sensorType = esp::sensor::SensorType::Color; + pinholeCameraSpec->sensorType = sensorType; pinholeCameraSpec->position = {0.0f, 0.f, 0.0f}; pinholeCameraSpec->resolution = {512, 384}; pinholeCameraSpec->uuid = sensorName; - std::vector sensorSpecifications = { - pinholeCameraSpec}; + return pinholeCameraSpec; +} + +std::vector getDefaultSensorSpecs( + TestFlags flags) { + std::vector sensorSpecifications{}; + if (flags & TestFlag::Color) { + sensorSpecifications.push_back( + getDefaultSensorSpecs("rgb", esp::sensor::SensorType::Color)); + } + if (flags & TestFlag::Depth) { + sensorSpecifications.push_back( + getDefaultSensorSpecs("depth", esp::sensor::SensorType::Depth)); + } return sensorSpecifications; } @@ -83,25 +118,54 @@ const struct { const char* name; Cr::Containers::Pointer (*create)( const ReplayRendererConfiguration& configuration); +} TestUnprojectData[]{ + {"classic", + [](const ReplayRendererConfiguration& configuration) { + return Cr::Containers::Pointer{ + new esp::sim::ClassicReplayRenderer{configuration}}; + }}, + // temp only enable testUnproject for classic + //{"batch", [](const ReplayRendererConfiguration& configuration) { + // return Cr::Containers::Pointer{ + // new esp::sim::BatchReplayRenderer{configuration}}; + // } + //}, +}; + +const struct { + const char* name; + TestFlags testFlags; + Cr::Containers::Pointer (*create)( + const ReplayRendererConfiguration& configuration); } TestIntegrationData[]{ - {"classic renderer", + {"rgb - classic", TestFlag::Color, + [](const ReplayRendererConfiguration& configuration) { + return Cr::Containers::Pointer{ + new esp::sim::ClassicReplayRenderer{configuration}}; + }}, + {"rgb - batch", TestFlag::Color, + [](const ReplayRendererConfiguration& configuration) { + return Cr::Containers::Pointer{ + new esp::sim::BatchReplayRenderer{configuration}}; + }}, + {"depth - classic", TestFlag::Depth, [](const ReplayRendererConfiguration& configuration) { return Cr::Containers::Pointer{ new esp::sim::ClassicReplayRenderer{configuration}}; }}, - {"batch renderer", [](const ReplayRendererConfiguration& configuration) { + {"depth - batch", TestFlag::Depth, + [](const ReplayRendererConfiguration& configuration) { return Cr::Containers::Pointer{ new esp::sim::BatchReplayRenderer{configuration}}; - }}}; + }}, +}; BatchReplayRendererTest::BatchReplayRendererTest() { + addInstancedTests({&BatchReplayRendererTest::testUnproject}, + Cr::Containers::arraySize(TestUnprojectData)); + addInstancedTests({&BatchReplayRendererTest::testIntegration}, Cr::Containers::arraySize(TestIntegrationData)); - - // temp only enable testUnproject for classic - addInstancedTests({&BatchReplayRendererTest::testUnproject}, 1); - // addInstancedTests({&BatchReplayRendererTest::testUnproject}, - // Cr::Containers::arraySize(TestIntegrationData)); } // ctor // test recording and playback through the simulator interface @@ -159,10 +223,11 @@ void BatchReplayRendererTest::testIntegration() { auto&& data = TestIntegrationData[testCaseInstanceId()]; setTestCaseDescription(data.name); + const auto sensorSpecs = getDefaultSensorSpecs(data.testFlags); + const std::string vangogh = Cr::Utility::Path::join( SCENE_DATASETS, "habitat-test-scenes/van-gogh-room.glb"); constexpr int numEnvs = 4; - const std::string sensorName = "my_rgb"; const std::string userPrefix = "sensor_"; const std::string screenshotPrefix = "ReplayBatchRendererTest_env"; const std::string screenshotExtension = ".png"; @@ -203,10 +268,13 @@ void BatchReplayRendererTest::testIntegration() { } auto& recorder = *sim->getGfxReplayManager()->getRecorder(); - recorder.addUserTransformToKeyframe( - userPrefix + sensorName, Mn::Vector3(3.3f, 1.3f + envIndex * 0.1f, 0.f), - Mn::Quaternion::rotation(Mn::Deg(80.f + envIndex * 5.f), - Mn::Vector3(0.f, 1.f, 0.f))); + for (const auto& sensor : sensorSpecs) { + recorder.addUserTransformToKeyframe( + userPrefix + sensor->uuid, + Mn::Vector3(3.3f, 1.3f + envIndex * 0.1f, 0.f), + Mn::Quaternion::rotation(Mn::Deg(80.f + envIndex * 5.f), + Mn::Vector3(0.f, 1.f, 0.f))); + } std::string serKeyframe = esp::gfx::replay::Recorder::keyframeToString( recorder.extractKeyframe()); @@ -214,25 +282,34 @@ void BatchReplayRendererTest::testIntegration() { } ReplayRendererConfiguration batchRendererConfig; - batchRendererConfig.sensorSpecifications = getDefaultSensorSpecs(sensorName); + batchRendererConfig.sensorSpecifications = sensorSpecs; batchRendererConfig.numEnvironments = numEnvs; { Cr::Containers::Pointer renderer = data.create(batchRendererConfig); + bool isClassicRenderer = dynamic_cast( + renderer.get()) != nullptr; // Check that the context is properly created CORRADE_VERIFY(Mn::GL::Context::hasCurrent()); - std::vector> buffers(numEnvs); - std::vector imageViews; + std::vector> colorBuffers(numEnvs); + std::vector> depthBuffers(numEnvs); + std::vector colorImageViews; + std::vector depthImageViews; for (int envIndex = 0; envIndex < numEnvs; envIndex++) { - // TODO pass size as a Vector2i; use an Image instead of a std::vector - // once there's Iterable that can be implicitly - // converted from a list of Image2D. - imageViews.emplace_back(getRGBView(renderer->sensorSize(envIndex).x(), - renderer->sensorSize(envIndex).y(), - buffers[envIndex])); + if (data.testFlags & TestFlag::Color) { + colorImageViews.emplace_back(getRGBView( + renderer->sensorSize(envIndex).x(), + renderer->sensorSize(envIndex).y(), colorBuffers[envIndex])); + } + if (data.testFlags & TestFlag::Depth) { + depthImageViews.emplace_back( + getDepthView(renderer->sensorSize(envIndex).x(), + renderer->sensorSize(envIndex).y(), + depthBuffers[envIndex], isClassicRenderer)); + } } for (int envIndex = 0; envIndex < numEnvs; envIndex++) { @@ -240,16 +317,27 @@ void BatchReplayRendererTest::testIntegration() { renderer->setSensorTransformsFromKeyframe(envIndex, userPrefix); } - renderer->render(imageViews, nullptr); + renderer->render(colorImageViews, depthImageViews); for (int envIndex = 0; envIndex < numEnvs; envIndex++) { CORRADE_ITERATION(envIndex); - std::string groundTruthImageFile = - screenshotPrefix + std::to_string(envIndex) + screenshotExtension; - CORRADE_COMPARE_WITH( - Mn::ImageView2D{imageViews[envIndex]}, - Cr::Utility::Path::join(screenshotDir, groundTruthImageFile), - (Mn::DebugTools::CompareImageToFile{maxThreshold, meanThreshold})); + // Test color output + if (data.testFlags & TestFlag::Color) { + std::string groundTruthImageFile = + screenshotPrefix + std::to_string(envIndex) + screenshotExtension; + CORRADE_COMPARE_WITH( + Mn::ImageView2D{colorImageViews[envIndex]}, + Cr::Utility::Path::join(screenshotDir, groundTruthImageFile), + (Mn::DebugTools::CompareImageToFile{maxThreshold, meanThreshold})); + } + // Test depth output + if (data.testFlags & TestFlag::Depth) { + const auto depth = depthImageViews[envIndex]; + float pixelA = depth.pixels()[32][32]; + float pixelB = depth.pixels()[64][64]; + CORRADE_VERIFY(pixelA > 0.0f); + CORRADE_VERIFY(pixelA != pixelB); + } } const auto colorPtr = renderer->getCudaColorBufferDevicePointer(); From afb3a908a84336817accb9a6c2af928fc90e9f39 Mon Sep 17 00:00:00 2001 From: Mikael Dallaire Cote <110583667+0mdc@users.noreply.github.com> Date: Mon, 12 Jun 2023 17:34:27 -0400 Subject: [PATCH 10/10] Review pass. --- src/esp/sim/AbstractReplayRenderer.cpp | 4 ++-- src/esp/sim/BatchReplayRenderer.cpp | 4 ++-- src/esp/sim/ClassicReplayRenderer.cpp | 2 -- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/esp/sim/AbstractReplayRenderer.cpp b/src/esp/sim/AbstractReplayRenderer.cpp index f0c57a923a..73c88ff927 100644 --- a/src/esp/sim/AbstractReplayRenderer.cpp +++ b/src/esp/sim/AbstractReplayRenderer.cpp @@ -93,13 +93,13 @@ void AbstractReplayRenderer::setSensorTransformsFromKeyframe( void AbstractReplayRenderer::render( Cr::Containers::ArrayView colorImageViews, Cr::Containers::ArrayView depthImageViews) { - if (colorImageViews != nullptr) { + if (colorImageViews.size() > 0) { ESP_CHECK(colorImageViews.size() == doEnvironmentCount(), "ReplayRenderer::render(): expected" << doEnvironmentCount() << "color image views but got" << colorImageViews.size()); } - if (depthImageViews != nullptr) { + if (depthImageViews.size() > 0) { ESP_CHECK(depthImageViews.size() == doEnvironmentCount(), "ReplayRenderer::render(): expected" << doEnvironmentCount() << "depth image views but got" diff --git a/src/esp/sim/BatchReplayRenderer.cpp b/src/esp/sim/BatchReplayRenderer.cpp index 05053f9c2c..121a84b21b 100644 --- a/src/esp/sim/BatchReplayRenderer.cpp +++ b/src/esp/sim/BatchReplayRenderer.cpp @@ -269,11 +269,11 @@ void BatchReplayRenderer::doRender( envIndex / renderer_->tileCount().x()}, renderer_->tileSize()); - if (colorImageViews) { + if (colorImageViews.size() > 0) { static_cast(*renderer_) .colorImageInto(rectangle, colorImageViews[envIndex]); } - if (depthImageViews) { + if (depthImageViews.size() > 0) { static_cast(*renderer_) .depthImageInto(rectangle, depthImageViews[envIndex]); } diff --git a/src/esp/sim/ClassicReplayRenderer.cpp b/src/esp/sim/ClassicReplayRenderer.cpp index ad94c89ad9..a72e7b1c6e 100644 --- a/src/esp/sim/ClassicReplayRenderer.cpp +++ b/src/esp/sim/ClassicReplayRenderer.cpp @@ -223,12 +223,10 @@ void ClassicReplayRenderer::doRender( imageViews = depthImageViews; break; default: - imageViews = nullptr; break; } #ifdef ESP_BUILD_WITH_BACKGROUND_RENDERER - // todo: investigate flags (frustum culling?) if (imageViews != nullptr) { renderer_->enqueueAsyncDrawJob( visualSensor, sceneGraph, imageViews[envIndex],