Thermal Camera: Visualize objects with a heat signature

ogre2/include/ignition/rendering/ogre2/Ogre2RenderEngine.hh

@@ -90,8 +90,8 @@ namespace ignition
       // Documentation Inherited.
       public: virtual std::string Name() const override;
 
-      /// \brief Add path to resourcea in ogre2's resource manager
-      /// \param[in] _uri Reousrce path in the form of an uri
+      /// \brief Add path to resource in ogre2's resource manager
+      /// \param[in] _uri Resource path in the form of an uri
       public: void AddResourcePath(const std::string &_uri) override;
 
       /// \brief Get the ogre2 root object

ogre2/src/Ogre2ThermalCamera.cc

@@ -24,14 +24,30 @@
 #endif
 
 #include <math.h>
+
+#include <algorithm>
 #include <limits>
+#include <string>
+#include <unordered_map>
+#include <variant>
+
+#ifdef _MSC_VER
+#pragma warning(push, 0)
+#endif
+#include <Hlms/Pbs/OgreHlmsPbsDatablock.h>
+#include <Hlms/Unlit/OgreHlmsUnlitDatablock.h>
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
 
 #include <ignition/common/Console.hh>
+#include <ignition/common/Filesystem.hh>
 #include <ignition/math/Helpers.hh>
 
 #include "ignition/rendering/RenderTypes.hh"
 #include "ignition/rendering/ogre2/Ogre2Conversions.hh"
 #include "ignition/rendering/ogre2/Ogre2Includes.hh"
+#include "ignition/rendering/ogre2/Ogre2Material.hh"
 #include "ignition/rendering/ogre2/Ogre2RenderEngine.hh"
 #include "ignition/rendering/ogre2/Ogre2RenderTarget.hh"
 #include "ignition/rendering/ogre2/Ogre2RenderTypes.hh"
@@ -52,7 +68,9 @@ class Ogre2ThermalCameraMaterialSwitcher : public Ogre::RenderTargetListener
 {
   /// \brief constructor
   /// \param[in] _scene the scene manager responsible for rendering
-  public: explicit Ogre2ThermalCameraMaterialSwitcher(Ogre2ScenePtr _scene);
+  /// \param[in] _name the name of the thermal camera
+  public: explicit Ogre2ThermalCameraMaterialSwitcher(Ogre2ScenePtr _scene,
+              const std::string & _name);
 
   /// \brief destructor
   public: ~Ogre2ThermalCameraMaterialSwitcher() = default;
@@ -75,13 +93,32 @@ class Ogre2ThermalCameraMaterialSwitcher : public Ogre::RenderTargetListener
   /// \brief Pointer to the heat source material
   private: Ogre::MaterialPtr heatSourceMaterial;
 
+  /// \brief Pointer to the "base" heat signature material.
+  /// All renderable items with a heat signature texture use their own
+  /// copy of this base material, with the item's specific heat
+  /// signature texture applied to it
+  private: Ogre::MaterialPtr baseHeatSigMaterial;
+
+  /// \brief A map of all items that have a heat signature material.
+  /// The key is the item's ID, and the value is the heat signature
+  /// for that item.
+  private: std::unordered_map<Ogre::IdType, Ogre::MaterialPtr>
+            heatSignatureMaterials;
+
+  /// \brief The name of the thermal camera sensor
+  private: const std::string name;
+
+  /// \brief The thermal camera
+  private: const Ogre::Camera* ogreCamera{nullptr};
+
   /// \brief Custom parameter index of temperature data in an ogre subitem.
   /// This has to match the custom index specifed in ThermalHeatSource material
   /// script in media/materials/scripts/thermal_camera.material
   private: const unsigned int customParamIdx = 10u;
 
   /// \brief A map of ogre sub item pointer to their original hlms material
-  private: std::map<Ogre::SubItem *, Ogre::HlmsDatablock *> datablockMap;
+  private: std::unordered_map<Ogre::SubItem *, Ogre::HlmsDatablock *>
+      datablockMap;
 };
 }
 }
@@ -130,14 +167,19 @@ class ignition::rendering::Ogre2ThermalCameraPrivate
   /// \brief Pointer to material switcher
   public: std::unique_ptr<Ogre2ThermalCameraMaterialSwitcher>
       thermalMaterialSwitcher = nullptr;
+
+  /// \brief Add variation to temperature values based on object rgb values
+  /// This only affects objects that are not heat sources
+  /// TODO(anyone) add API for setting this value?
+  public: bool rgbToTemp = true;
 };
 
 using namespace ignition;
 using namespace rendering;
 
 //////////////////////////////////////////////////
 Ogre2ThermalCameraMaterialSwitcher::Ogre2ThermalCameraMaterialSwitcher(
-    Ogre2ScenePtr _scene)
+    Ogre2ScenePtr _scene, const std::string & _name) : name(_name)
 {
   this->scene = _scene;
   // plain opaque material
@@ -147,6 +189,11 @@ Ogre2ThermalCameraMaterialSwitcher::Ogre2ThermalCameraMaterialSwitcher(
 
   this->heatSourceMaterial = res.staticCast<Ogre::Material>();
   this->heatSourceMaterial->load();
+
+  this->baseHeatSigMaterial = Ogre::MaterialManager::getSingleton().
+    getByName("ThermalHeatSignature");
+
+  this->ogreCamera = this->scene->OgreSceneManager()->findCamera(this->name);
 }
 
 //////////////////////////////////////////////////
@@ -164,7 +211,7 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
     Ogre::MovableObject *object = itor.peekNext();
     Ogre::Item *item = static_cast<Ogre::Item *>(object);
 
-    std::string tempKey = "temperature";
+    const std::string tempKey = "temperature";
     // get visual
     Ogre::Any userAny = item->getUserObjectBindings().getUserAny();
     if (!userAny.isEmpty() && userAny.getType() == typeid(unsigned int))
@@ -183,7 +230,7 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
 
       // get temperature
       Variant tempAny = ogreVisual->UserData(tempKey);
-      if (tempAny.index() != 0)
+      if (tempAny.index() != 0 && !std::holds_alternative<std::string>(tempAny))
       {
         float temp = -1.0;
         try
@@ -213,8 +260,6 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
         // only accept positive temperature (in kelvin)
         if (temp >= 0.0)
         {
-          // set visibility flag so thermal camera can see it
-          item->addVisibilityFlags(0x10000000);
           for (unsigned int i = 0; i < item->getNumSubItems(); ++i)
           {
             Ogre::SubItem *subItem = item->getSubItem(i);
@@ -223,8 +268,12 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
               // normalize temperature value
               float color = temp * 100.0 /
                   static_cast<float>(std::numeric_limits<uint16_t>::max());
+
+              // set g, b, a to 0. This will be used by shaders to determine
+              // if particular fragment is a heat source or not
+              // see media/materials/programs/thermal_camera_fs.glsl
               subItem->setCustomParameter(this->customParamIdx,
-                  Ogre::Vector4(color, color, color, 1.0));
+                  Ogre::Vector4(color, 0, 0, 0.0));
             }
             Ogre::HlmsDatablock *datablock = subItem->getDatablock();
             this->datablockMap[subItem] = datablock;
@@ -233,6 +282,96 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
           }
         }
       }
+      // get heat signature and the corresponding min/max temperature values
+      else if (auto heatSignature = std::get_if<std::string>(&tempAny))
+      {
+        // if this is the first time rendering the heat signature,
+        // we need to make sure that the texture is loaded and applied to
+        // the heat signature material before loading the material
+        if (this->heatSignatureMaterials.find(item->getId()) ==
+            this->heatSignatureMaterials.end())
+        {
+          // make sure the texture is in ogre's resource path
+          const auto &texture = *heatSignature;
+          auto engine = Ogre2RenderEngine::Instance();
+          engine->AddResourcePath(texture);
+
+          // create a material for this item, now that the texture has been
+          // searched for. We must clone the base heat signature material since
+          // different items may use different textures. We also append the
+          // item's ID to the end of the new material name to ensure new
+          // material uniqueness in case two items use the same heat signature
+          // texture, but have different temperature ranges
+          std::string baseName = common::basename(texture);
+          auto heatSignatureMaterial = this->baseHeatSigMaterial->clone(
+              this->name + "_" + baseName + "_" +
+              Ogre::StringConverter::toString(item->getId()));
+          auto textureUnitStatePtr = heatSignatureMaterial->
+            getTechnique(0)->getPass(0)->getTextureUnitState(0);
+          Ogre::String textureName = baseName;
+          textureUnitStatePtr->setTextureName(textureName);
+
+          // set temperature range for the heat signature
+          auto minTempVariant = ogreVisual->UserData("minTemp");
+          auto maxTempVariant = ogreVisual->UserData("maxTemp");
+          auto minTemperature = std::get_if<float>(&minTempVariant);
+          auto maxTemperature = std::get_if<float>(&maxTempVariant);
+          if (minTemperature && maxTemperature)
+          {
+            // make sure the temperature range is between [0, 655.35] kelvin
+            Ogre::GpuProgramParametersSharedPtr params =
+              heatSignatureMaterial->getTechnique(0)->getPass(0)->
+              getFragmentProgramParameters();
+            params->setNamedConstant("minTemp",
+                std::max(static_cast<float>(*minTemperature), 0.0f));
+            params->setNamedConstant("maxTemp",
+                std::min(static_cast<float>(*maxTemperature), 655.35f));
+          }
+          heatSignatureMaterial->load();
+          this->heatSignatureMaterials[item->getId()] = heatSignatureMaterial;
+        }
+
+        for (unsigned int i = 0; i < item->getNumSubItems(); ++i)
+        {
+          Ogre::SubItem *subItem = item->getSubItem(i);
+
+          Ogre::HlmsDatablock *datablock = subItem->getDatablock();
+          this->datablockMap[subItem] = datablock;
+
+          subItem->setMaterial(this->heatSignatureMaterials[item->getId()]);
+        }
+      }
+      // background objects
+      else
+      {
+        Ogre::Aabb aabb = item->getWorldAabbUpdated();
+        Ogre::AxisAlignedBox box = Ogre::AxisAlignedBox(aabb.getMinimum(),
+            aabb.getMaximum());
+
+        // we will be converting rgb values to tempearture values in shaders
+        // but we want to make sure the object rgb values are not affected by
+        // lighting, so disable lighting
+        // Also check if objects are within camera view
+        if (ogreVisual->GeometryCount() > 0u &&
+            this->ogreCamera->isVisible(box))
+        {
+          auto geom = ogreVisual->GeometryByIndex(0);
+          if (geom)
+          {
+            MaterialPtr mat = geom->Material();
+            Ogre2MaterialPtr ogreMat =
+                std::dynamic_pointer_cast<Ogre2Material>(mat);
+            Ogre::HlmsUnlitDatablock *unlit = ogreMat->UnlitDatablock();
+            for (unsigned int i = 0; i < item->getNumSubItems(); ++i)
+            {
+              Ogre::SubItem *subItem = item->getSubItem(i);
+              Ogre::HlmsDatablock *datablock = subItem->getDatablock();
+              this->datablockMap[subItem] = datablock;
+              subItem->setDatablock(unlit);
+            }
+          }
+        }
+      }
     }
     itor.moveNext();
   }
@@ -242,7 +381,7 @@ void Ogre2ThermalCameraMaterialSwitcher::preRenderTargetUpdate(
 void Ogre2ThermalCameraMaterialSwitcher::postRenderTargetUpdate(
     const Ogre::RenderTargetEvent & /*_evt*/)
 {
-  // restore item to use hlms material
+  // restore item to use pbs hlms material
   for (auto it : this->datablockMap)
   {
     Ogre::SubItem *subItem = it.first;
@@ -441,6 +580,8 @@ void Ogre2ThermalCamera::CreateThermalTexture()
       static_cast<float>(this->ambientRange));
   psParams->setNamedConstant("heatSourceTempRange",
       static_cast<float>(this->heatSourceTempRange));
+  psParams->setNamedConstant("rgbToTemp",
+      static_cast<int>(this->dataPtr->rgbToTemp));
 
   // Create thermal camera compositor
   auto engine = Ogre2RenderEngine::Instance();
@@ -455,17 +596,7 @@ void Ogre2ThermalCamera::CreateThermalTexture()
   // {
   //   in 0 rt_input
   //   texture depthTexture target_width target_height PF_D32_FLOAT
-  //   texture colorTexture target_width target_height PF_R8G8B8
-  //   target depthTexture
-  //   {
-  //     pass clear
-  //     {
-  //       colour_value 0.0 0.0 0.0 1.0
-  //     }
-  //     pass render_scene
-  //     {
-  //     }
-  //   }
+  //   texture colorTexture target_width target_height PF_R8G8B8A8
   //   target colorTexture
   //   {
   //     pass clear
@@ -517,7 +648,9 @@ void Ogre2ThermalCamera::CreateThermalTexture()
     thermalTexDef->uav = false;
     thermalTexDef->automipmaps = false;
     thermalTexDef->hwGammaWrite = Ogre::TextureDefinitionBase::BoolFalse;
-    thermalTexDef->depthBufferId = Ogre::DepthBuffer::POOL_NON_SHAREABLE;
+    // set to default pool so that when the colorTexture pass is rendered, its
+    // depth data get populated to depthTexture
+    thermalTexDef->depthBufferId = Ogre::DepthBuffer::POOL_DEFAULT;
     thermalTexDef->depthBufferFormat = Ogre::PF_UNKNOWN;
     thermalTexDef->fsaaExplicitResolve = false;
 
@@ -530,7 +663,7 @@ void Ogre2ThermalCamera::CreateThermalTexture()
     colorTexDef->numMipmaps = 0;
     colorTexDef->widthFactor = 1;
     colorTexDef->heightFactor = 1;
-    colorTexDef->formatList = {Ogre::PF_R8G8B8};
+    colorTexDef->formatList = {Ogre::PF_R8G8B8A8};
     colorTexDef->fsaa = 0;
     colorTexDef->uav = false;
     colorTexDef->automipmaps = false;
@@ -540,24 +673,7 @@ void Ogre2ThermalCamera::CreateThermalTexture()
     colorTexDef->preferDepthTexture = true;
     colorTexDef->fsaaExplicitResolve = false;
 
-    nodeDef->setNumTargetPass(3);
-    Ogre::CompositorTargetDef *depthTargetDef =
-        nodeDef->addTargetPass("depthTexture");
-    depthTargetDef->setNumPasses(2);
-    {
-      // clear pass
-      Ogre::CompositorPassClearDef *passClear =
-          static_cast<Ogre::CompositorPassClearDef *>(
-          depthTargetDef->addPass(Ogre::PASS_CLEAR));
-      passClear->mColourValue = Ogre::ColourValue(this->FarClipPlane(), 0, 0);
-      // scene pass
-      Ogre::CompositorPassSceneDef *passScene =
-          static_cast<Ogre::CompositorPassSceneDef *>(
-          depthTargetDef->addPass(Ogre::PASS_SCENE));
-      passScene->mVisibilityMask = IGN_VISIBILITY_ALL
-          & ~(IGN_VISIBILITY_GUI | IGN_VISIBILITY_SELECTABLE);
-    }
-
+    nodeDef->setNumTargetPass(2);
     Ogre::CompositorTargetDef *colorTargetDef =
         nodeDef->addTargetPass("colorTexture");
     colorTargetDef->setNumPasses(2);
@@ -568,14 +684,10 @@ void Ogre2ThermalCamera::CreateThermalTexture()
           colorTargetDef->addPass(Ogre::PASS_CLEAR));
       passClear->mColourValue = Ogre::ColourValue(0, 0, 0);
       // scene pass
-      Ogre::CompositorPassSceneDef *passScene =
-          static_cast<Ogre::CompositorPassSceneDef *>(
-          colorTargetDef->addPass(Ogre::PASS_SCENE));
-      // set thermal camera custom visibility mask when rendering heat sources
-      passScene->mVisibilityMask = 0x10000000;
+      colorTargetDef->addPass(Ogre::PASS_SCENE);
     }
 
-    // rt_input target - converts depth to thermal
+    // rt_input target - converts to thermal
     Ogre::CompositorTargetDef *inputTargetDef =
         nodeDef->addTargetPass("rt_input");
     inputTargetDef->setNumPasses(2);
@@ -632,10 +744,10 @@ void Ogre2ThermalCamera::CreateThermalTexture()
   auto channels = node->getLocalTextures();
   for (auto c : channels)
   {
-    if (c.textures[0]->getSrcFormat() == Ogre::PF_R8G8B8)
+    if (c.textures[0]->getSrcFormat() == Ogre::PF_R8G8B8A8)
     {
       this->dataPtr->thermalMaterialSwitcher.reset(
-          new Ogre2ThermalCameraMaterialSwitcher(this->scene));
+          new Ogre2ThermalCameraMaterialSwitcher(this->scene, this->Name()));
       c.target->addListener(this->dataPtr->thermalMaterialSwitcher.get());
       break;
     }
@@ -699,14 +811,14 @@ void Ogre2ThermalCamera::PostRender()
         this->dataPtr->thermalImage, width, height, 1, "L16");
 
   // Uncomment to debug thermal output
-  // igndbg << "wxh: " << width << " x " << height << std::endl;
+  // std::cout << "wxh: " << width << " x " << height << std::endl;
   // for (unsigned int i = 0; i < height; ++i)
   // {
   //   for (unsigned int j = 0; j < width; ++j)
   //   {
-  //     igndbg << "[" << this->dataPtr->thermalImage[i*width + j] << "]";
+  //     std::cout << "[" << this->dataPtr->thermalImage[i*width + j] << "]";
   //   }
-  //   igndbg << std::endl;
+  //   std::cout << std::endl;
   // }
 }