Gfx-replay polish: workaround for material-overrides; new keyframe ge…

…tter (#2035)

* gfx-replay polish: workaround for material-overrides; new keyframe getter API needed for a python application

src/esp/bindings/GfxReplayBindings.cpp

@@ -110,6 +110,19 @@ void initGfxReplayBindings(py::module& m) {
           },
           R"(Write all saved keyframes to a string, then discard the keyframes.)")
 
+      .def(
+          "write_incremental_saved_keyframes_to_string_array",
+          [](ReplayManager& self) {
+            if (!self.getRecorder()) {
+              throw std::runtime_error(
+                  "replay save not enabled. See "
+                  "SimulatorConfiguration.enable_gfx_replay_save.");
+            }
+            return self.getRecorder()
+                ->writeIncrementalSavedKeyframesToStringArray();
+          },
+          R"(Write all saved keyframes to individual strings. See Recorder.h for details.)")
+
       .def("read_keyframes_from_file", &ReplayManager::readKeyframesFromFile,
            R"(Create a Player object from a replay file.)");
 }

src/esp/gfx/replay/Player.cpp

@@ -18,6 +18,27 @@ namespace esp {
 namespace gfx {
 namespace replay {
 
+namespace {
+
+// At recording time, material overrides gets stringified and appended to the
+// filepath. See ResourceManager::createModifiedAssetName.
+// AbstractSceneGraphPlayerImplementation doesn't support parsing this material
+// info. More info at
+// https://docs.google.com/document/d/1ngA73cXl3YRaPfFyICSUHONZN44C-XvieS7kwyQDbkI/edit#bookmark=id.aoe7xgsro2r7
+std::string removeMaterialOverrideFromFilepathAndWarn(const std::string& src) {
+  auto pos = src.find('?');
+  if (pos != std::string::npos) {
+    ESP_WARNING(Mn::Debug::Flag::NoSpace)
+        << "Ignoring material-override for [" << src << "]";
+
+    return src.substr(0, pos);
+  } else {
+    return src;
+  }
+}
+
+}  // namespace
+
 static_assert(std::is_nothrow_move_constructible<Player>::value, "");
 
 void AbstractPlayerImplementation::setNodeSemanticId(NodeHandle, unsigned) {}
@@ -173,22 +194,28 @@ void Player::applyKeyframe(const Keyframe& keyframe) {
 
   for (const auto& pair : keyframe.creations) {
     const auto& creation = pair.second;
-    if (assetInfos_.count(creation.filepath) == 0u) {
-      if (failedFilepaths_.count(creation.filepath) == 0u) {
+
+    auto adjustedFilepath =
+        removeMaterialOverrideFromFilepathAndWarn(creation.filepath);
+
+    if (assetInfos_.count(adjustedFilepath) == 0u) {
+      if (failedFilepaths_.count(adjustedFilepath) == 0u) {
         ESP_WARNING(Mn::Debug::Flag::NoSpace)
-            << "Missing asset info for [" << creation.filepath << "]";
-        failedFilepaths_.insert(creation.filepath);
+            << "Missing asset info for [" << adjustedFilepath << "]";
+        failedFilepaths_.insert(adjustedFilepath);
       }
       continue;
     }
-    CORRADE_INTERNAL_ASSERT(assetInfos_.count(creation.filepath));
+    CORRADE_INTERNAL_ASSERT(assetInfos_.count(adjustedFilepath));
+    auto adjustedCreation = creation;
+    adjustedCreation.filepath = adjustedFilepath;
     auto* node = implementation_->loadAndCreateRenderAssetInstance(
-        assetInfos_[creation.filepath], creation);
+        assetInfos_[adjustedFilepath], adjustedCreation);
     if (!node) {
-      if (failedFilepaths_.count(creation.filepath) == 0u) {
+      if (failedFilepaths_.count(adjustedFilepath) == 0u) {
         ESP_WARNING(Mn::Debug::Flag::NoSpace)
-            << "Load failed for asset [" << creation.filepath << "]";
-        failedFilepaths_.insert(creation.filepath);
+            << "Load failed for asset [" << adjustedFilepath << "]";
+        failedFilepaths_.insert(adjustedFilepath);
       }
       continue;
     }

src/esp/gfx/replay/Recorder.cpp

@@ -232,6 +232,25 @@ std::string Recorder::writeSavedKeyframesToString() {
   return esp::io::jsonToString(document);
 }
 
+std::vector<std::string>
+Recorder::writeIncrementalSavedKeyframesToStringArray() {
+  std::vector<std::string> results;
+  results.reserve(savedKeyframes_.size());
+
+  for (const auto& keyframe : savedKeyframes_) {
+    results.emplace_back(keyframeToString(keyframe));
+  }
+
+  // note we don't call consolidateSavedKeyframes. Use this function if you are
+  // using keyframes incrementally, e.g. repeated calls to this function and
+  // feeding them to a renderer. Contrast with writeSavedKeyframesToFile, which
+  // "consolidates" before discarding old keyframes to avoid losing state
+  // information.
+  savedKeyframes_.clear();
+
+  return results;
+}
+
 std::string Recorder::keyframeToString(const Keyframe& keyframe) {
   rapidjson::Document d(rapidjson::kObjectType);
   rapidjson::Document::AllocatorType& allocator = d.GetAllocator();

src/esp/gfx/replay/Recorder.h

@@ -126,10 +126,21 @@ class Recorder {
                                  bool usePrettyWriter = false);
 
   /**
-   * @brief write saved keyframes to string.
+   * @brief write saved keyframes to string. '{"keyframes": [{...},{...},...]}'
    */
   std::string writeSavedKeyframesToString();
 
+  /**
+   * @brief write saved keyframes as individual strings ['{"keyframe": ...}',
+   * '{"keyframe": ...}', ...]
+   *
+   * Use this function if you are using keyframes incrementally, e.g.
+   * repeated calls to this function and feeding them to a renderer. Contrast
+   * with writeSavedKeyframesToFile, which "consolidates" before discarding old
+   * keyframes to avoid losing state information.
+   */
+  std::vector<std::string> writeIncrementalSavedKeyframesToStringArray();
+
   /**
    * @brief returns JSONized version of given keyframe.
    */

src/esp/sim/BatchReplayRenderer.cpp

@@ -99,11 +99,12 @@ BatchReplayRenderer::BatchReplayRenderer(
         ESP_WARNING()
             << creation.filepath
             << "not found in any composite file, loading from the filesystem";
-        // TODO asserts might be TOO BRUTAL?
-        CORRADE_INTERNAL_ASSERT_OUTPUT(
+
+        ESP_CHECK(
             renderer_.addFile(creation.filepath,
                               gfx_batch::RendererFileFlag::Whole |
-                                  gfx_batch::RendererFileFlag::GenerateMipmap));
+                                  gfx_batch::RendererFileFlag::GenerateMipmap),
+            "addFile failed for " << creation.filepath);
         CORRADE_INTERNAL_ASSERT(renderer_.hasNodeHierarchy(creation.filepath));
       }