--Region query tools (#2336)

* --add region poly loop area calc; add weighted point containment check
To support nested regions, this check will provide weighted results, with the highest weighted region being the one with the smallest area.
* --cleanup/organize semantic functions
* --add weighted region checking for a point, to account for nested regions
The weighting is derived from the region area, where the smallest region a point is found in has the highest weighting.  If a point only resides in a single region, its weighting is 1.
* --give every containing region an equal vote (in the case of nested regions)
For nested regions, every containing region will get an equal vote for the multiple points. This means that a set of points that are mostly in the inner region of a nested region will still appear to be 'more' in the outer region that contains all the points.

src/esp/bindings/SceneBindings.cpp

@@ -228,24 +228,38 @@ void initSceneBindings(py::module& m) {
           "file"_a, "scene"_a, "rotation"_a)
       .def_property_readonly("aabb", &SemanticScene::aabb)
       .def_property_readonly("categories", &SemanticScene::categories,
-                             "All semantic categories in the house")
+                             "All semantic categories in the scene")
       .def_property_readonly("levels", &SemanticScene::levels,
-                             "All levels in the house")
+                             "All levels in the scene")
       .def_property_readonly("regions", &SemanticScene::regions,
-                             "All regions in the house")
+                             "All regions in the scene")
       .def_property_readonly("objects", &SemanticScene::objects,
-                             "All object in the house")
+                             "All object in the scene")
       .def_property_readonly("semantic_index_map",
                              &SemanticScene::getSemanticIndexMap)
       .def("semantic_index_to_object_index",
            &SemanticScene::semanticIndexToObjectIndex)
       .def("get_regions_for_point", &SemanticScene::getRegionsForPoint,
-           "Compute all SemanticRegions which contain the point and return a "
-           "list of indices for the regions in this SemanticScene.")
+           R"(Compute all SemanticRegions which contain the point and return a
+             list of indices for the regions in this SemanticScene.)",
+           "point"_a)
+      .def("get_weighted_regions_for_point",
+           &SemanticScene::getWeightedRegionsForPoint,
+           R"("Find all SemanticRegions which contain the point and return a
+              sorted list of tuple pairs of the region index and a score of that
+              region, derived as
+                    1 - (region_area/ttl_region_area)
+              where ttl_region_area is the area of all the regions containing
+              the point, so that smaller regions are weighted higher. If only
+              one region contains the passed point, its weight will be 1.)",
+           "point"_a)
       .def("get_regions_for_points", &SemanticScene::getRegionsForPoints,
-           "Compute SemanticRegion containment for a set of points. Return a "
-           "sorted list of tuple pairs with each containing region index and "
-           "the percentage of points contained by that region.");
+           R"("Compute SemanticRegion containment for a set of points. Return a
+              sorted list of tuple pairs with each containing region index and
+              the percentage of points contained by that region. In the case of nested
+              regions, points are considered belonging to every region the point is
+              found in.)",
+           "points"_a);
 
   // ==== ObjectControls ====
   py::class_<ObjectControls, ObjectControls::ptr>(m, "ObjectControls")

src/esp/scene/SemanticScene.cpp

@@ -179,6 +179,8 @@ bool SemanticScene::
         int eIdx = 0;
         float yExtrusion = static_cast<float>(regionPtr->extrusionHeight_ +
                                               regionPtr->floorHeight_);
+
+        float polyArea = 0.0f;
         for (std::size_t i = 0; i < numPts; ++i) {
           Mn::Vector3 currPoint = loopPoints[i];
           regionPtr->polyLoopPoints_[i] = {currPoint.x(), currPoint.z()};
@@ -188,6 +190,10 @@ bool SemanticScene::
           std::size_t nextIdx = ((i + 1) % numPts);
           Mn::Vector3 nextPoint = loopPoints[nextIdx];
           Mn::Vector3 nextExtPt = {nextPoint.x(), yExtrusion, nextPoint.z()};
+          // Find region projection area based on Green's Theorem (.5 * abs(sum
+          // (xprod of sequential edges)))
+          polyArea +=
+              currPoint.x() * nextPoint.z() - nextPoint.x() * currPoint.z();
           // Horizontal edge
           regionPtr->visEdges_[eIdx++] = {currPoint, nextPoint};
           // Vertical edge
@@ -197,7 +203,8 @@ bool SemanticScene::
           // Diagonal edge
           regionPtr->visEdges_[eIdx++] = {currPoint, nextExtPt};
         }
-
+        // Set the polyloop area
+        regionPtr->area_ = .5 * abs(polyArea);
         scene.regions_.emplace_back(std::move(regionPtr));
       }
     } else {  // if semantic attributes specifes region annotations
@@ -600,29 +607,85 @@ std::vector<int> SemanticScene::getRegionsForPoint(
     }
   }
   return containingRegions;
-}
+}  // SemanticScene::getRegionsForPoint
+
+std::vector<std::pair<int, double>> SemanticScene::getWeightedRegionsForPoint(
+    const Mn::Vector3& point) const {
+  std::vector<int> containingRegions = getRegionsForPoint(point);
+  if (containingRegions.size() == 0) {
+    return {};
+  }
+
+  std::vector<std::pair<int, double>> containingRegionWeights;
+  containingRegionWeights.reserve(containingRegions.size());
+  // Only 1 containing region, so return region idx and weight of 1
+  if (containingRegions.size() == 1) {
+    containingRegionWeights.emplace_back(
+        std::pair<int, double>(containingRegions[0], 1.0f));
+    return containingRegionWeights;
+  }
+
+  // Sum up all areas containing point
+  double ttlArea = 0.0f;
+  for (int rix : containingRegions) {
+    ttlArea += regions_[rix]->getArea();
+  }
+
+  for (int rix : containingRegions) {
+    containingRegionWeights.emplace_back(std::pair<int, double>(
+        rix, 1.0f - (regions_[rix]->getArea() / ttlArea)));
+  }
 
-std::vector<std::pair<int, float>> SemanticScene::getRegionsForPoints(
+  std::sort(containingRegionWeights.begin(), containingRegionWeights.end(),
+            [](const std::pair<int, double>& a, std::pair<int, double>& b) {
+              return a.second > b.second;
+            });
+  return containingRegionWeights;
+
+}  // SemanticScene::getWeightedRegionsForPoint
+
+std::vector<std::pair<int, double>> SemanticScene::getRegionsForPoints(
     const std::vector<Mn::Vector3>& points) const {
-  std::vector<std::pair<int, float>> containingRegionWeights;
+  // Weights for every point for every region
+  std::vector<std::vector<double>> regAreaWeightsForPoints;
+  regAreaWeightsForPoints.reserve(points.size());
+  for (int i = 0; i < points.size(); ++i) {
+    // Get this point's weighted regions
+    auto regWeightsForPoint = getWeightedRegionsForPoint(points[i]);
+    // Initialize all region weights to be 0
+    std::vector<double> allRegionWeights(regions_.size(), 0);
+    // Set the weight for the containing region with the smallest area (if
+    // nested regions) for this particular point.
+    // Set the vote for each region to be equal.
+    for (const std::pair<int, double>& regionWeight : regWeightsForPoint) {
+      allRegionWeights[regionWeight.first] = 1.0;
+    }
+    // Save this points region weight vector
+    regAreaWeightsForPoints.emplace_back(allRegionWeights);
+  }
+
+  std::vector<std::pair<int, double>> containingRegionWeights;
+  // Will only have at max the number of regions in the scene
+  containingRegionWeights.reserve(regions_.size());
   for (int rix = 0; rix < regions_.size(); ++rix) {
-    float containmentCount = 0;
-    for (const auto& point : points) {
-      if (regions_[rix]->contains(point)) {
-        containmentCount += 1;
-      }
+    double containmentWeight = 0;
+    for (int i = 0; i < points.size(); ++i) {
+      std::vector<double> regWtsForPoint = regAreaWeightsForPoints[i];
+      containmentWeight += regWtsForPoint[rix];
     }
-    if (containmentCount > 0) {
+    if (containmentWeight > 0) {
       containingRegionWeights.emplace_back(
-          std::pair<int, float>(rix, containmentCount / points.size()));
+          std::pair<int, double>(rix, containmentWeight / points.size()));
     }
   }
+  // Free up unused capacity - every region probably does not contain a tested
+  // point
+  containingRegionWeights.shrink_to_fit();
   std::sort(containingRegionWeights.begin(), containingRegionWeights.end(),
-            [](const std::pair<int, float>& a, std::pair<int, float>& b) {
+            [](const std::pair<int, double>& a, std::pair<int, double>& b) {
               return a.second > b.second;
             });
   return containingRegionWeights;
-}
-
+}  // SemanticScene::getRegionsForPoints
 }  // namespace scene
 }  // namespace esp

src/esp/scene/SemanticScene.h

@@ -293,13 +293,26 @@ class SemanticScene {
   std::vector<int> getRegionsForPoint(const Mn::Vector3& point) const;
 
   /**
-   * @brief Compute SemanticRegion containment for a set of points.
+   * @brief Compute all the SemanticRegions that contain the passed point, and
+   * return a vector of indices and weights for each region, where the weights
+   * are inverted area of the region (smaller regions weighted higher)
+   * @param point The query point.
+   * @return std::vector<std::pair<int, float>> A sorted list of tuples
+   * containing region index and inverse area of that region
+   */
+  std::vector<std::pair<int, double>> getWeightedRegionsForPoint(
+      const Mn::Vector3& point) const;
+
+  /**
+   * @brief Compute SemanticRegion containment for a set of points. It is
+   * assumed the set of points belong to the same construct (i.e. points from an
+   * individual object's mesh)
    * @param points A set of points to test for semantic containment.
    * @return std::vector<std::pair<int, float>> A sorted list of tuples
    * containing region index and percentage of input points contained in that
    * region.
    */
-  std::vector<std::pair<int, float>> getRegionsForPoints(
+  std::vector<std::pair<int, double>> getRegionsForPoints(
       const std::vector<Mn::Vector3>& points) const;
 
  protected:
@@ -521,6 +534,17 @@ class SemanticRegion {
 
   SemanticCategory::ptr category() const { return category_; }
 
+  /**
+   * @brief Returns the area of the polyloop forming the base of the region
+   * extrusion
+   */
+  double getArea() const { return area_; }
+  /**
+   * @brief Returns the volume of the polyloop-based extrusion defining the
+   * bounds of this region.
+   */
+  double getVolume() const { return area_ * extrusionHeight_; }
+
  protected:
   int index_{};
   int parentIndex_{};
@@ -530,6 +554,9 @@ class SemanticRegion {
 
   std::string name_;
 
+  // The area of the surface enclosed by the region
+  double area_{};
+
   // Height of extrusion for Extruded poly-loop-based volumes
   double extrusionHeight_{};
   // Floor height
@@ -545,7 +572,7 @@ class SemanticRegion {
   std::shared_ptr<SemanticLevel> level_;
   friend SemanticScene;
   ESP_SMART_POINTERS(SemanticRegion)
-};
+};  // class SemanticRegion
 
 //! Represents a distinct semantically annotated object
 class SemanticObject {

src/esp/sim/Simulator.h

@@ -73,16 +73,6 @@ class Simulator {
   void seed(uint32_t newSeed);
 
   std::shared_ptr<gfx::Renderer> getRenderer() { return renderer_; }
-  std::shared_ptr<scene::SemanticScene> getSemanticScene() {
-    return resourceManager_->getSemanticScene();
-  }
-
-  /**
-   * @brief Return a view of the currently set Semantic scene colormap.
-   */
-  const std::vector<Mn::Vector3ub>& getSemanticSceneColormap() const {
-    return resourceManager_->getSemanticSceneColormap();
-  }
 
   inline void getRenderGLContext() {
     // acquire GL context from background thread, if background rendering
@@ -92,29 +82,6 @@ class Simulator {
     }
   }
 
-  /** @brief check if the semantic scene exists.*/
-  bool semanticSceneExists() const {
-    return resourceManager_->semanticSceneExists();
-  }
-
-  /**
-   * @brief get the current active scene graph
-   */
-  scene::SceneGraph& getActiveSceneGraph() {
-    CORRADE_INTERNAL_ASSERT(std::size_t(activeSceneID_) < sceneID_.size());
-    return sceneManager_->getSceneGraph(activeSceneID_);
-  }
-
-  /** @brief Check to see if there is a SemanticSceneGraph for rendering */
-  bool semanticSceneGraphExists() const {
-    return std::size_t(activeSemanticSceneID_) < sceneID_.size();
-  }
-
-  /** @brief get the semantic scene's SceneGraph for rendering */
-  scene::SceneGraph& getActiveSemanticSceneGraph() {
-    CORRADE_INTERNAL_ASSERT(semanticSceneGraphExists());
-    return sceneManager_->getSceneGraph(activeSemanticSceneID_);
-  }
   std::shared_ptr<gfx::replay::ReplayManager> getGfxReplayManager() {
     return gfxReplayMgr_;
   }
@@ -218,7 +185,44 @@ class Simulator {
   }
 
   /**
-   * @brief Build a map keyed by semantic color/id referencing a vector
+   * @brief get the current active scene graph
+   */
+  scene::SceneGraph& getActiveSceneGraph() {
+    CORRADE_INTERNAL_ASSERT(std::size_t(activeSceneID_) < sceneID_.size());
+    return sceneManager_->getSceneGraph(activeSceneID_);
+  }
+
+  ///////////////////////////
+  // Semantic Scene and Data
+  std::shared_ptr<scene::SemanticScene> getSemanticScene() {
+    return resourceManager_->getSemanticScene();
+  }
+
+  /**
+   * @brief Return a view of the currently set Semantic scene colormap.
+   */
+  const std::vector<Mn::Vector3ub>& getSemanticSceneColormap() const {
+    return resourceManager_->getSemanticSceneColormap();
+  }
+
+  /** @brief check if the semantic scene exists.*/
+  bool semanticSceneExists() const {
+    return resourceManager_->semanticSceneExists();
+  }
+
+  /** @brief Check to see if there is a SemanticSceneGraph for rendering */
+  bool semanticSceneGraphExists() const {
+    return std::size_t(activeSemanticSceneID_) < sceneID_.size();
+  }
+
+  /** @brief get the semantic scene's SceneGraph for rendering */
+  scene::SceneGraph& getActiveSemanticSceneGraph() {
+    CORRADE_INTERNAL_ASSERT(semanticSceneGraphExists());
+    return sceneManager_->getSceneGraph(activeSemanticSceneID_);
+  }
+
+  /**
+   * @brief Build a map keyed by semantic color/id referencing a list of
    * connected component-based Semantic objects.
    */
   std::unordered_map<uint32_t, std::vector<scene::CCSemanticObject::ptr>>
@@ -248,6 +252,9 @@ class Simulator {
     return {};
   }
 
+  ///////////////////////////
+  // End Semantic Scene and Data
+
   /**
    * @brief Builds a @ref esp::metadata::attributes::SceneInstanceAttributes describing the
    * current scene configuration, and saves it to a JSON file, using @p

tests/test_semantic_scene.py

@@ -135,7 +135,6 @@ def test_semantic_regions():
             (-1 * p) for p in hit_test_points
         ]  # add one less to create imbalance
         regions_weights = semantic_scene.get_regions_for_points(mixed_points)
-        print(f"regions_weights = {regions_weights}")
         assert regions_weights[0][0] == 1  # bathroom with more points comes first
         assert (
             regions_weights[0][1] >= 0.51