Support instance sensor (it can be used in viewer.cpp directly)

src/esp/gfx/BackgroundRenderer.cpp

@@ -171,7 +171,7 @@ int BackgroundRenderer::threadRender() {
     if (sensorType == sensor::SensorType::Depth)
       sensor.renderTarget().readFrameDepth(view);
 
-    if (sensorType == sensor::SensorType::Semantic)
+    if (sensorType == sensor::SensorType::Semantic || sensorType == sensor::SensorType::Instance)
       sensor.renderTarget().readFrameObjectId(view);
   }
 

src/esp/gfx/PbrDrawable.cpp

@@ -195,6 +195,15 @@ void PbrDrawable::draw(const Mn::Matrix4& transformationMatrix,
     Mn::GL::Renderer::setFrontFace(Mn::GL::Renderer::FrontFace::ClockWise);
   }
 
+  // Pick the semantic id or the instance id based on the camera type
+  int specificID = 0;
+  if(static_cast<RenderCamera&>(camera).getIsInstanceId()) {
+    specificID = node_.getInstanceId();
+  }
+  else {
+    specificID = node_.getSemanticId();
+  }
+
   (*shader_)
       // e.g., semantic mesh has its own per vertex annotation, which has
       // been uploaded to GPU so simply pass 0 to the uniform "objectId" in
@@ -204,7 +213,7 @@ void PbrDrawable::draw(const Mn::Matrix4& transformationMatrix,
                        : ((flags_ & PbrShader::Flag::InstancedObjectId) ==
                                   PbrShader::Flag::InstancedObjectId
                               ? 0
-                              : node_.getSemanticId()))
+                              : specificID))
       .setProjectionMatrix(camera.projectionMatrix())
       .setViewMatrix(camera.cameraMatrix())
       .setModelMatrix(modelMatrix)  // NOT modelview matrix!

src/esp/gfx/RenderCamera.h

@@ -215,6 +215,20 @@ class RenderCamera : public MagnumCamera {
   size_t filterTransforms(DrawableTransforms& drawableTransforms,
                           Flags flags = {});
 
+  /**
+   * @brief Set the flag to infer if the camera should pick semantic id or instance id for the object.
+   *
+   * @param isInstanceId true if picking instance id, false if picking semantic id.
+   */
+  void setIsInstanceId(bool isInstanceId) { isInstanceId_ = isInstanceId; }
+
+  /**
+   * @brief Get the flag of whether the camera should pick semantic id or instance id for the object.
+   *
+   * @return true if picking instance id, false if picking semantic id.
+   */
+  bool getIsInstanceId() const { return isInstanceId_; }
+
   /**
    * @brief if the "immediate" following rendering pass is to use drawable ids
    * as the object ids.
@@ -254,6 +268,7 @@ class RenderCamera : public MagnumCamera {
   Mn::Matrix4 invertedProjectionMatrix;
   size_t previousNumVisibleDrawables_ = 0;
   bool useDrawableIds_ = false;
+  bool isInstanceId_ = false;
   ESP_SMART_POINTERS(RenderCamera)
 };
 

src/esp/gfx/Renderer.cpp

@@ -87,7 +87,8 @@ struct Renderer::Impl {
   void draw(sensor::VisualSensor& visualSensor, sim::Simulator& sim) {
     acquireGlContext();
     if (visualSensor.specification()->sensorType ==
-        sensor::SensorType::Semantic) {
+        sensor::SensorType::Semantic || visualSensor.specification()->sensorType ==
+        sensor::SensorType::Instance) {
       ESP_CHECK(sim.semanticSceneGraphExists(),
                 "Renderer::Impl::draw(): SemanticSensor observation requested "
                 "but no SemanticSceneGraph is loaded");
@@ -101,7 +102,7 @@ struct Renderer::Impl {
     acquireGlContext();
     sensor::SensorType& type = visualSensor.specification()->sensorType;
     if (type == sensor::SensorType::Depth ||
-        type == sensor::SensorType::Semantic) {
+        type == sensor::SensorType::Semantic || type == sensor::SensorType::Instance) {
       Mn::GL::Renderer::disable(Mn::GL::Renderer::Feature::DepthTest);
       gfx::RenderTarget& tgt = visualSensor.renderTarget();
       if (!mesh_) {
@@ -112,7 +113,7 @@ struct Renderer::Impl {
       esp::gfx::Renderer::Impl::RendererShaderType rendererShaderType =
           esp::gfx::Renderer::Impl::RendererShaderType::DepthTextureVisualizer;
 
-      if (type == sensor::SensorType::Semantic) {
+      if (type == sensor::SensorType::Semantic || type == sensor::SensorType::Instance) {
         rendererShaderType =
             gfx::Renderer::Impl::RendererShaderType::ObjectIdTextureVisualizer;
       }
@@ -160,7 +161,7 @@ struct Renderer::Impl {
         shader->bindDepthTexture(tgt.getDepthTexture());
 #endif
         shader->setDepthUnprojection(*visualSensor.depthUnprojection());
-      } else if (type == sensor::SensorType::Semantic) {
+      } else if (type == sensor::SensorType::Semantic || type == sensor::SensorType::Instance) {
         shader->bindObjectIdTexture(tgt.getObjectIdTexture());
       }
       if ((colorMapOffset >= 0) && (colorMapScale >= 0)) {
@@ -365,6 +366,13 @@ struct Renderer::Impl {
         renderTargetFlags |= RenderTarget::Flag::ObjectIdAttachment;
         break;
 
+      case sensor::SensorType::Instance:
+        if (bindingFlags & Flag::VisualizeTexture) {
+          renderTargetFlags |= RenderTarget::Flag::RgbaAttachment;
+        }
+        renderTargetFlags |= RenderTarget::Flag::ObjectIdAttachment;
+        break;
+
       default:
         // I need this default, since sensor type list is long, and without
         // default clang-tidy will complain

src/esp/metadata/attributes/ObjectAttributes.h

@@ -367,6 +367,20 @@ class ObjectAttributes : public AbstractObjectAttributes {
 
   uint32_t getSemanticId() const { return get<int>("semantic_id"); }
 
+  /**
+   * @brief Set the instance id for the object. The instance id should be unique in each scene for the object.
+   *
+   * @param instance_id the id for the object in the scene (can be the implicit index of the object in the scene config)
+   */
+  void setInstanceId(int instance_id) { set("instance_id", instance_id); }
+
+  /**
+   * @brief Get the instance id for the object.
+   *
+   * @return instance_id the id for the object in the scene
+   */
+  uint32_t getInstanceId() const { return get<int>("instance_id"); }
+
  protected:
   /**
    * @brief Write object-specific values to json object

src/esp/physics/PhysicsManager.cpp

@@ -129,6 +129,10 @@ int PhysicsManager::addObjectInstance(
                 << "as specified in object instance attributes.";
     return 0;
   }
+
+  // Set the instance id of the object
+  objAttributes->setInstanceId(objInstAttributes->getID());
+
   // check if an object is being set to be not visible for a particular
   // instance.
   int visSet = objInstAttributes->getIsInstanceVisible();

src/esp/physics/RigidBase.h

@@ -386,6 +386,22 @@ class RigidBase : public esp::physics::PhysicsObjectBase {
     }
   }
 
+  /**
+   * @brief Get the Instance ID for this object.
+   */
+  int getInstanceId() const { return visualNode_->getInstanceId(); }
+
+  /**
+   * @brief Set the @ref esp::scene::SceneNode::InstanceId_ for all visual nodes
+   * belonging to the object.
+   * @param InstanceId The desired Instance id for the object.
+   */
+  void setInstanceId(uint32_t instanceId) {
+    for (auto* node : visualNodes_) {
+      node->setInstanceId(instanceId);
+    }
+  }
+
   /**
    * @brief Get pointers to this rigid's visual SceneNodes.
    * @return vector of pointers to the rigid's visual scene nodes.

src/esp/physics/RigidObject.cpp

@@ -49,6 +49,8 @@ bool RigidObject::finalizeObject() {
 
   // set the visualization semantic id
   setSemanticId(ObjectAttributes->getSemanticId());
+  // Set the visualization instance id
+  setInstanceId(ObjectAttributes->getInstanceId());
 
   // finish object by instancing any dynamics library-specific code required
   return finalizeObject_LibSpecific();

src/esp/scene/SceneNode.h

@@ -127,6 +127,12 @@ class SceneNode : public MagnumObject,
   //! Sets node semanticId
   virtual void setSemanticId(int semanticId) { semanticId_ = semanticId; }
 
+  //! Returns node instanceId
+  virtual int getInstanceId() const { return instanceId_; }
+
+  //! Sets node instanceId
+  virtual void setInstanceId(int instanceId) { instanceId_ = instanceId; }
+
   Magnum::Vector3 absoluteTranslation() const;
 
   Magnum::Vector3 absoluteTranslation();
@@ -234,6 +240,7 @@ class SceneNode : public MagnumObject,
   //! The semantic category of this node. Used to render attached Drawables with
   //! Semantic sensor when no perVertexObjectIds are present.
   uint32_t semanticId_ = 0;
+  uint32_t instanceId_ = 0;
 
   //! the local bounding box for meshes stored at this node
   Magnum::Range3D meshBB_;

src/esp/sensor/CameraSensor.cpp

@@ -134,7 +134,13 @@ bool CameraSensor::drawObservation(sim::Simulator& sim) {
     flags |= gfx::RenderCamera::Flag::FrustumCulling;
   }
 
-  if (cameraSensorSpec_->sensorType == SensorType::Semantic) {
+  if (cameraSensorSpec_->sensorType == SensorType::Semantic || cameraSensorSpec_->sensorType == SensorType::Instance) {
+    // Set the camera instance Flag to infer if the camera should pick semantic id or instance id based on the camera type
+    if (cameraSensorSpec_->sensorType == SensorType::Semantic) {
+      renderCamera_->setIsInstanceId(false);
+    } else {
+      renderCamera_->setIsInstanceId(true);
+    }
     // TODO: check sim has semantic scene graph
     bool twoSceneGraphs =
         (&sim.getActiveSemanticSceneGraph() != &sim.getActiveSceneGraph());

src/esp/sensor/CubeMapSensorBase.cpp

@@ -57,6 +57,9 @@ CubeMapSensorBase::CubeMapSensorBase(scene::SceneNode& cameraNode,
     case SensorType::Semantic:
       cubeMapFlags |= gfx::CubeMap::Flag::ObjectIdTexture;
       break;
+    case SensorType::Instance:
+      cubeMapFlags |= gfx::CubeMap::Flag::ObjectIdTexture;
+      break;
     default:
       CORRADE_INTERNAL_ASSERT_UNREACHABLE();
       break;
@@ -80,6 +83,9 @@ CubeMapSensorBase::CubeMapSensorBase(scene::SceneNode& cameraNode,
     case SensorType::Semantic:
       cubeMapShaderBaseFlags_ |= gfx::CubeMapShaderBase::Flag::ObjectIdTexture;
       break;
+    case SensorType::Instance:
+      cubeMapShaderBaseFlags_ |= gfx::CubeMapShaderBase::Flag::ObjectIdTexture;
+      break;
     // sensor type list is too long, have to use default
     default:
       CORRADE_INTERNAL_ASSERT_UNREACHABLE();
@@ -121,7 +127,7 @@ bool CubeMapSensorBase::renderToCubemapTexture(sim::Simulator& sim) {
 
   // generate the cubemap texture
   const char* defaultDrawableGroupName = "";
-  if (cubeMapSensorBaseSpec_->sensorType == SensorType::Semantic) {
+  if (cubeMapSensorBaseSpec_->sensorType == SensorType::Semantic || cubeMapSensorBaseSpec_->sensorType == SensorType::Instance) {
     bool twoSceneGraphs =
         (&sim.getActiveSemanticSceneGraph() != &sim.getActiveSceneGraph());
 
@@ -164,7 +170,8 @@ void CubeMapSensorBase::drawWith(gfx::CubeMapShaderBase& shader) {
     shader.bindDepthTexture(
         cubeMap_->getTexture(gfx::CubeMap::TextureType::Depth));
   }
-  if (cubeMapSensorBaseSpec_->sensorType == SensorType::Semantic) {
+  if (cubeMapSensorBaseSpec_->sensorType == SensorType::Semantic ||
+      cubeMapSensorBaseSpec_->sensorType == SensorType::Instance) {
     shader.bindObjectIdTexture(
         cubeMap_->getTexture(gfx::CubeMap::TextureType::ObjectId));
   }

src/esp/sensor/Sensor.h

@@ -33,6 +33,7 @@ enum class SensorType : int32_t {
   Tensor,
   Text,
   Audio,
+  Instance,
   SensorTypeCount,  // add new type above this term!!
 };
 

src/esp/sensor/VisualSensor.cpp

@@ -24,7 +24,7 @@ void VisualSensorSpec::sanityCheck() const {
   SensorSpec::sanityCheck();
   bool isVisualSensor =
       (sensorType == SensorType::Color || sensorType == SensorType::Depth ||
-       sensorType == SensorType::Normal || sensorType == SensorType::Semantic);
+       sensorType == SensorType::Normal || sensorType == SensorType::Semantic || sensorType == SensorType::Instance);
   CORRADE_ASSERT(
       isVisualSensor,
       "VisualSensorSpec::sanityCheck(): sensorType must be Color, Depth, "
@@ -90,7 +90,7 @@ bool VisualSensor::getObservationSpace(ObservationSpace& space) {
                  static_cast<size_t>(visualSensorSpec_->resolution[1]),
                  static_cast<size_t>(visualSensorSpec_->channels)};
   space.dataType = core::DataType::DT_UINT8;
-  if (visualSensorSpec_->sensorType == SensorType::Semantic) {
+  if (visualSensorSpec_->sensorType == SensorType::Semantic || visualSensorSpec_->sensorType == SensorType::Instance) {
     space.dataType = core::DataType::DT_UINT32;
   } else if (visualSensorSpec_->sensorType == SensorType::Depth) {
     space.dataType = core::DataType::DT_FLOAT;
@@ -110,7 +110,7 @@ void VisualSensor::readObservation(Observation& obs) {
 
   // TODO: have different classes for the different types of sensors
   // TODO: do we need to flip axis?
-  if (visualSensorSpec_->sensorType == SensorType::Semantic) {
+  if (visualSensorSpec_->sensorType == SensorType::Semantic || visualSensorSpec_->sensorType == SensorType::Instance) {
     renderTarget().readFrameObjectId(Magnum::MutableImageView2D{
         Magnum::PixelFormat::R32UI, renderTarget().framebufferSize(),
         obs.buffer->data});
@@ -153,7 +153,7 @@ VisualSensor::MoveSemanticSensorNodeHelper::MoveSemanticSensorNodeHelper(
     sim::Simulator& sim)
     : visualSensor_(visualSensor), sim_(sim) {
   CORRADE_INTERNAL_ASSERT(visualSensor_.specification()->sensorType ==
-                          SensorType::Semantic);
+                          SensorType::Semantic || visualSensor_.specification()->sensorType == SensorType::Instance);
   scene::SceneNode& node = visualSensor_.node();
   CORRADE_ASSERT(
       !scene::SceneGraph::isRootNode(node),
@@ -189,7 +189,7 @@ VisualSensor::MoveSemanticSensorNodeHelper::MoveSemanticSensorNodeHelper(
 
 VisualSensor::MoveSemanticSensorNodeHelper::~MoveSemanticSensorNodeHelper() {
   CORRADE_INTERNAL_ASSERT(visualSensor_.specification()->sensorType ==
-                          SensorType::Semantic);
+                          SensorType::Semantic || visualSensor_.specification()->sensorType == SensorType::Instance);
 
   scene::SceneNode& node = visualSensor_.node();
   CORRADE_ASSERT(