From f964e5143a5d477d666ebedf322e472499d0c799 Mon Sep 17 00:00:00 2001 From: John Turner <7strbass@gmail.com> Date: Wed, 31 Jan 2024 17:53:44 -0500 Subject: [PATCH] --[Part 2 of 2] Semantic region building (#2307) * --add support for non-wildcard relative filenames. This enables relative path/filename values set in the default attributes section of the scene dataset config to be resolved to real files on attributes load. * --pass semantic attributes to loadSemanticSceneDescriptor instead of just filename. * --move all semantic scene creation code to SemanticScene * --add semantic region creation and bindings. * --add regions to the scene's region vector * viewer.py region rendering demo * --make extrusions * --rename sub-configs to SemanticVolumes Will eventually be used for describing non-region semantic constructs. * --add construction of volume edges for debug/visualizations. * --add semantic visualization in a more formal manner. * --test containment; fix height calc and containment check. * --update test to read from test JSON file instead of using a JSON string. * --ssd specified through scene dataset config default value * --add semantic config files with names matching scene dataset tags * --require ssd handle to be found explicitly in dataset Since this handle is only a string tag to tie to a semantic attributes, it should -always- be matched explicitly. * --adjust test to cover expanded test scene dataset. * --minor simplification of attributes managers tests. * --add bindings specifically for loop-based semantic category * --python test for semantic regions * --fix attempting to map ssd file handle if scene instance is NONE scene. --------- Co-authored-by: aclegg3 --- .../dataset_test_scene.scene_instance.json | 3 +- .../dataset_test_scene.semantic_config.json | 36 ++ ...ntic_descriptor_path1.semantic_config.json | 36 ++ ...ntic_descriptor_path2.semantic_config.json | 36 ++ .../test_dataset_0.scene_dataset_config.json | 88 ++--- .../test_dataset_1.scene_dataset_config.json | 15 +- .../test_regions.semantic_config.json | 288 ++-------------- examples/viewer.py | 48 ++- src/esp/assets/ResourceManager.cpp | 85 ++--- src/esp/assets/ResourceManager.h | 15 +- src/esp/bindings/SceneBindings.cpp | 27 +- .../attributes/SceneDatasetAttributes.cpp | 20 +- .../attributes/SceneDatasetAttributes.h | 5 +- .../attributes/SemanticAttributes.cpp | 24 +- .../metadata/attributes/SemanticAttributes.h | 34 +- .../metadata/managers/AttributesManagerBase.h | 41 ++- .../managers/SemanticAttributesManager.cpp | 14 +- .../managers/SemanticAttributesManager.h | 20 +- src/esp/scene/SemanticScene.cpp | 308 +++++++++++++----- src/esp/scene/SemanticScene.h | 79 ++++- src/esp/sim/Simulator.cpp | 69 ++-- src/esp/sim/Simulator.h | 6 +- src/tests/CMakeLists.txt | 12 +- src/tests/MetadataMediatorTest.cpp | 12 +- src/tests/Mp3dTest.cpp | 9 +- src/tests/SemanticTest.cpp | 132 ++++++++ tests/test_attributes_managers.py | 37 ++- tests/test_semantic_scene.py | 93 +++++- 28 files changed, 1007 insertions(+), 585 deletions(-) create mode 100644 data/test_assets/dataset_tests/dataset_0/semantics/dataset_test_scene.semantic_config.json create mode 100644 data/test_assets/dataset_tests/dataset_0/semantics/test_semantic_descriptor_path1.semantic_config.json create mode 100644 data/test_assets/dataset_tests/dataset_0/semantics/test_semantic_descriptor_path2.semantic_config.json create mode 100644 src/tests/SemanticTest.cpp diff --git a/data/test_assets/dataset_tests/dataset_0/scenes/dataset_test_scene.scene_instance.json b/data/test_assets/dataset_tests/dataset_0/scenes/dataset_test_scene.scene_instance.json index 14b8024e29..e5d5e02589 100644 --- a/data/test_assets/dataset_tests/dataset_0/scenes/dataset_test_scene.scene_instance.json +++ b/data/test_assets/dataset_tests/dataset_0/scenes/dataset_test_scene.scene_instance.json @@ -18,6 +18,5 @@ } ], "default_lighting":"modified_test_lights", - "navmesh_instance":"navmesh_path1", - "semantic_scene_instance":"semantic_descriptor_path1" + "navmesh_instance":"navmesh_path1" } diff --git a/data/test_assets/dataset_tests/dataset_0/semantics/dataset_test_scene.semantic_config.json b/data/test_assets/dataset_tests/dataset_0/semantics/dataset_test_scene.semantic_config.json new file mode 100644 index 0000000000..dd48746ce6 --- /dev/null +++ b/data/test_assets/dataset_tests/dataset_0/semantics/dataset_test_scene.semantic_config.json @@ -0,0 +1,36 @@ +{ + "region_annotations": [ + { + "name": "test_region_negativeX", + "label": "bedroom", + "poly_loop": [ + [-20.0, -2.0,-10.0], + [-25.0, -2.0, 0.0], + [-20.0, -2.0, 10.0], + [-10.0, -2.0, 10.0], + [-5.0, -2.0, 0.0], + [-10.0, -2.0,-10.0] + ], + "floor_height": -2.0, + "extrusion_height": 4.0, + "min_bounds": [-25.0, -2.0, -10.0], + "max_bounds": [-5.0, 2.0, 10.0] + }, + { + "name": "test_region_positiveX", + "label": "bathroom", + "poly_loop": [ + [10.0, -2.0,-10.0], + [5.0, -2.0, 0.0], + [10.0, -2.0, 10.0], + [20.0, -2.0, 10.0], + [25.0, -2.0, 0.0], + [20.0, -2.0,-10.0] + ], + "floor_height": -2.0, + "extrusion_height": 4.0, + "min_bounds": [5.0, -2.0, -10.0], + "max_bounds": [25.0, 2.0, 10.0] + } + ] +} diff --git a/data/test_assets/dataset_tests/dataset_0/semantics/test_semantic_descriptor_path1.semantic_config.json b/data/test_assets/dataset_tests/dataset_0/semantics/test_semantic_descriptor_path1.semantic_config.json new file mode 100644 index 0000000000..dd48746ce6 --- /dev/null +++ b/data/test_assets/dataset_tests/dataset_0/semantics/test_semantic_descriptor_path1.semantic_config.json @@ -0,0 +1,36 @@ +{ + "region_annotations": [ + { + "name": "test_region_negativeX", + "label": "bedroom", + "poly_loop": [ + [-20.0, -2.0,-10.0], + [-25.0, -2.0, 0.0], + [-20.0, -2.0, 10.0], + [-10.0, -2.0, 10.0], + [-5.0, -2.0, 0.0], + [-10.0, -2.0,-10.0] + ], + "floor_height": -2.0, + "extrusion_height": 4.0, + "min_bounds": [-25.0, -2.0, -10.0], + "max_bounds": [-5.0, 2.0, 10.0] + }, + { + "name": "test_region_positiveX", + "label": "bathroom", + "poly_loop": [ + [10.0, -2.0,-10.0], + [5.0, -2.0, 0.0], + [10.0, -2.0, 10.0], + [20.0, -2.0, 10.0], + [25.0, -2.0, 0.0], + [20.0, -2.0,-10.0] + ], + "floor_height": -2.0, + "extrusion_height": 4.0, + "min_bounds": [5.0, -2.0, -10.0], + "max_bounds": [25.0, 2.0, 10.0] + } + ] +} diff --git a/data/test_assets/dataset_tests/dataset_0/semantics/test_semantic_descriptor_path2.semantic_config.json b/data/test_assets/dataset_tests/dataset_0/semantics/test_semantic_descriptor_path2.semantic_config.json new file mode 100644 index 0000000000..dd48746ce6 --- /dev/null +++ b/data/test_assets/dataset_tests/dataset_0/semantics/test_semantic_descriptor_path2.semantic_config.json @@ -0,0 +1,36 @@ +{ + "region_annotations": [ + { + "name": "test_region_negativeX", + "label": "bedroom", + "poly_loop": [ + [-20.0, -2.0,-10.0], + [-25.0, -2.0, 0.0], + [-20.0, -2.0, 10.0], + [-10.0, -2.0, 10.0], + [-5.0, -2.0, 0.0], + [-10.0, -2.0,-10.0] + ], + "floor_height": -2.0, + "extrusion_height": 4.0, + "min_bounds": [-25.0, -2.0, -10.0], + "max_bounds": [-5.0, 2.0, 10.0] + }, + { + "name": "test_region_positiveX", + "label": "bathroom", + "poly_loop": [ + [10.0, -2.0,-10.0], + [5.0, -2.0, 0.0], + [10.0, -2.0, 10.0], + [20.0, -2.0, 10.0], + [25.0, -2.0, 0.0], + [20.0, -2.0,-10.0] + ], + "floor_height": -2.0, + "extrusion_height": 4.0, + "min_bounds": [5.0, -2.0, -10.0], + "max_bounds": [25.0, 2.0, 10.0] + } + ] +} diff --git a/data/test_assets/dataset_tests/dataset_0/test_dataset_0.scene_dataset_config.json b/data/test_assets/dataset_tests/dataset_0/test_dataset_0.scene_dataset_config.json index 2199e8eac1..514901111f 100644 --- a/data/test_assets/dataset_tests/dataset_0/test_dataset_0.scene_dataset_config.json +++ b/data/test_assets/dataset_tests/dataset_0/test_dataset_0.scene_dataset_config.json @@ -12,90 +12,91 @@ "original_file": "stages/dataset_test_stage.stage_config.json", "template_handle": "modified_test_stage", "attributes": { - "scale":[1,1,1], - "gravity":[0,-9.8,0], - "margin":0.041, - "friction_coefficient": 0.4, - "restitution_coefficient": 0.5, - "units_to_meters":1.0 + "scale":[1,1,1], + "gravity":[0,-9.8,0], + "margin":0.041, + "friction_coefficient": 0.4, + "restitution_coefficient": 0.5, + "units_to_meters":1.0 } }, { "template_handle": "new_test_stage", "attributes": { - "render_asset": "stages/dataset_test_stage.glb", - "up":[0,-1,0], - "scale":[2,2,2], - "gravity":[0,9.8,0], - "friction_coefficient": 0.35, - "restitution_coefficient": 0.25, - "units_to_meters":2.0, - "force_flat_shading": true + "render_asset": "stages/dataset_test_stage.glb", + "up":[0,-1,0], + "scale":[2,2,2], + "gravity":[0,9.8,0], + "friction_coefficient": 0.35, + "restitution_coefficient": 0.25, + "units_to_meters":2.0, + "force_flat_shading": true } } ] }, "objects":{ "default_attributes": { - "mass" : 10.0, - "inertia": [3,2,1] + "mass" : 10.0, + "inertia": [3,2,1] }, "paths": { - ".json" : ["objects"] - }, + ".json" : ["objects"] + }, "configs" : [ { "original_file": "objects/dataset_test_object1.object_config.json", "template_handle": "modified_test_object1_1_slick_heavy", "attributes": { - "friction_coefficient": 0.2, - "rolling_friction_coefficient": 0.0002, - "spinning_friction_coefficient": 0.0003, - "mass": 3.5 + "friction_coefficient": 0.2, + "rolling_friction_coefficient": 0.0002, + "spinning_friction_coefficient": 0.0003, + "mass": 3.5 } }, - { + { "template_handle": "new_test_object3", "attributes": { - "render_asset": "objects/dataset_test_object3.glb", - "friction_coefficient": 0.1, - "mass": 1.1 + "render_asset": "objects/dataset_test_object3.glb", + "friction_coefficient": 0.1, + "mass": 1.1 } } - ] }, "light_setups":{ - "default_attributes": { - }, + "default_attributes": {}, "paths": { - ".json" : ["lights"] - }, + ".json" : ["lights"] + }, "configs" : [ { "original_file":"lights/dataset_test_lights.lighting_config.json", "template_handle": "modified_test_lights", "attributes": { - "lights":{ - "0": { "position": [1.5,0.1,1.5], "intensity": 2.4, "color": [0.5,1,0.95], "type": "point"}, - "1": { "position": [2.5,-0.1,2.5], "intensity": 2.1, "color": [0.5,1,0.95], "type": "point"}, - "11": { "position": [3.5,-0.7,-3.5], "intensity": -0.5, "color": [1,0.5,1], "type": "point"} - } + "lights":{ + "0": { "position": [1.5,0.1,1.5], "intensity": 2.4, "color": [0.5,1,0.95], "type": "point"}, + "1": { "position": [2.5,-0.1,2.5], "intensity": 2.1, "color": [0.5,1,0.95], "type": "point"}, + "11": { "position": [3.5,-0.7,-3.5], "intensity": -0.5, "color": [1,0.5,1], "type": "point"} + } } }, { "template_handle": "new_test_lights_0", "attributes": { - "lights":{ - "3": { "position": [11.5,10.1,11.5], "intensity": 1.4, "color": [0.5,1,0.95], "type": "point"}, - "4": { "position": [12.5,-10.1,12.5], "intensity": 1.1, "color": [0.5,1,0.95], "type": "point"}, - "5": { "position": [13.5,-10.7,-13.5], "intensity": -1.5, "color": [1,0.5,1], "type": "point"} - } + "lights":{ + "3": { "position": [11.5,10.1,11.5], "intensity": 1.4, "color": [0.5,1,0.95], "type": "point"}, + "4": { "position": [12.5,-10.1,12.5], "intensity": 1.1, "color": [0.5,1,0.95], "type": "point"}, + "5": { "position": [13.5,-10.7,-13.5], "intensity": -1.5, "color": [1,0.5,1], "type": "point"} + } } } ] }, "scene_instances":{ + "default_attributes": { + "semantic_scene_instance": "%%CONFIG_NAME_AS_ASSET_FILENAME%%.semantic_config.json" + }, "paths": { ".json" : ["scenes"] } @@ -105,6 +106,11 @@ "navmesh_path2":"test_navmesh_path2" }, "semantic_scene_descriptor_instances": { + "paths": { + ".json": [ + "semantics/*" + ] + }, "semantic_descriptor_path1":"test_semantic_descriptor_path1", "semantic_descriptor_path2":"test_semantic_descriptor_path2" } diff --git a/data/test_assets/dataset_tests/dataset_1/test_dataset_1.scene_dataset_config.json b/data/test_assets/dataset_tests/dataset_1/test_dataset_1.scene_dataset_config.json index 38ef15b55d..e281a14aaa 100644 --- a/data/test_assets/dataset_tests/dataset_1/test_dataset_1.scene_dataset_config.json +++ b/data/test_assets/dataset_tests/dataset_1/test_dataset_1.scene_dataset_config.json @@ -4,8 +4,8 @@ "origin":[1.0,2.0,3.0] }, "paths": { - ".json" : ["stages/*"] - }, + ".json" : ["stages/*"] + }, "configs" : [ { "original_file": "stages/stage_0/dataset_test_stage_0.stage_config.json", @@ -52,8 +52,8 @@ "inertia": [3,2,1] }, "paths": { - ".json" : ["objects/*"] - }, + ".json" : ["objects/*"] + }, "configs" : [ { "original_file": "objects/object_0/dataset_test_object1_0.object_config.json", @@ -63,7 +63,7 @@ "mass": 3.5 } }, - { + { "template_handle": "new_test_object3", "attributes": { "render_asset": "objects/object_1/dataset_test_object3_1.glb", @@ -71,15 +71,14 @@ "mass": 1.1 } } - ] }, "light_setups":{ "default_attributes": { }, "paths": { - ".json" : ["lights/*"] - }, + ".json" : ["lights/*"] + }, "configs" : [ { "original_file":"lights/lights_0/dataset_test_lights_0.lighting_config.json", diff --git a/data/test_assets/semantic/test_regions.semantic_config.json b/data/test_assets/semantic/test_regions.semantic_config.json index 6eaf7f233f..dd48746ce6 100644 --- a/data/test_assets/semantic/test_regions.semantic_config.json +++ b/data/test_assets/semantic/test_regions.semantic_config.json @@ -1,280 +1,36 @@ { "region_annotations": [ { - "name": "bedroom", + "name": "test_region_negativeX", "label": "bedroom", "poly_loop": [ - [ - -18.876018524169922, - 0.0, - -10.983560562133789 - ], - [ - -15.061599731445312, - 0.0, - -10.983560562133789 - ], - [ - -15.061599731445312, - 0.0, - -8.543549537658691 - ], - [ - -18.876018524169922, - 0.0, - -8.543549537658691 - ] - ], - "floor_height": 0.0, - "extrusion_height": 4.0, - "min_bounds": [ - -18.876018524169922, - 0.0, - -10.983560562133789 - ], - "max_bounds": [ - -15.061599731445312, - 4.0, - -8.543549537658691 - ] - }, - { - "name": "kitchen", - "label": "kitchen", - "poly_loop": [ - [ - -18.876018524169922, - 0.0, - -8.393549919128418 - ], - [ - -15.316800117492676, - 0.0, - -8.393549919128418 - ], - [ - -18.876018524169922, - 0.0, - -5.939759731292725 - ], - [ - -15.316800117492676, - 0.0, - -5.939759731292725 - ] - ], - "floor_height": 0.0, - "extrusion_height": 4.0, - "min_bounds": [ - -18.876018524169922, - 0.0, - -8.393549919128418 - ], - "max_bounds": [ - -15.316800117492676, - 4.0, - -5.939759731292725 - ] - }, - { - "name": "living room", - "label": "living room", - "poly_loop": [ - [ - -18.876018524169922, - 0.0, - -5.789760112762451 - ], - [ - -13.965399742126465, - 0.0, - -5.789760112762451 - ], - [ - -18.876018524169922, - 0.0, - -2.040019989013672 - ], - [ - -13.965399742126465, - 0.0, - -2.040019989013672 - ] - ], - "floor_height": 0.0, + [-20.0, -2.0,-10.0], + [-25.0, -2.0, 0.0], + [-20.0, -2.0, 10.0], + [-10.0, -2.0, 10.0], + [-5.0, -2.0, 0.0], + [-10.0, -2.0,-10.0] + ], + "floor_height": -2.0, "extrusion_height": 4.0, - "min_bounds": [ - -18.876018524169922, - 0.0, - -5.789760112762451 - ], - "max_bounds": [ - -13.965399742126465, - 4.0, - -2.040019989013672 - ] + "min_bounds": [-25.0, -2.0, -10.0], + "max_bounds": [-5.0, 2.0, 10.0] }, { - "name": "rec/game", - "label": "rec/game", - "poly_loop": [ - [ - -15.166800498962402, - 0.0, - -8.393549919128418 - ], - [ - -10.796329498291016, - 0.0, - -8.393549919128418 - ], - [ - -15.166800498962402, - 0.0, - -6.908959865570068 - ], - [ - -13.815400123596191, - 0.0, - -6.908959865570068 - ], - [ - -13.815400123596191, - 0.0, - -2.040019989013672 - ], - [ - -10.796329498291016, - 0.0, - -2.040019989013672 - ] - ], - "floor_height": 0.0, - "extrusion_height": 4.0, - "min_bounds": [ - -15.166800498962402, - 0.0, - -8.393549919128418 - ], - "max_bounds": [ - -10.796329498291016, - 4.0, - -2.040019989013672 - ] - }, - { - "name": "bathroom", + "name": "test_region_positiveX", "label": "bathroom", "poly_loop": [ - [ - -15.166800498962402, - 0.0, - -6.758960247039795 - ], - [ - -13.965399742126465, - 0.0, - -6.758960247039795 - ], - [ - -15.166800498962402, - 0.0, - -5.939759731292725 - ], - [ - -13.965399742126465, - 0.0, - -5.939759731292725 - ] - ], - "floor_height": 0.0, - "extrusion_height": 4.0, - "min_bounds": [ - -15.166800498962402, - 0.0, - -6.758960247039795 - ], - "max_bounds": [ - -13.965399742126465, - 4.0, - -5.939759731292725 - ] - }, - { - "name": "tv", - "label": "tv", - "poly_loop": [ - [ - -14.911600112915039, - 0.0, - -10.983560562133789 - ], - [ - -11.86514949798584, - 0.0, - -10.983560562133789 - ], - [ - -14.911600112915039, - 0.0, - -8.543549537658691 - ], - [ - -11.86514949798584, - 0.0, - -8.543549537658691 - ] - ], - "floor_height": 0.0, + [10.0, -2.0,-10.0], + [5.0, -2.0, 0.0], + [10.0, -2.0, 10.0], + [20.0, -2.0, 10.0], + [25.0, -2.0, 0.0], + [20.0, -2.0,-10.0] + ], + "floor_height": -2.0, "extrusion_height": 4.0, - "min_bounds": [ - -14.911600112915039, - 0.0, - -10.983560562133789 - ], - "max_bounds": [ - -11.86514949798584, - 4.0, - -8.543549537658691 - ] - }, - { - "name": "bathroom.001", - "label": "bathroom", - "poly_loop": [ - [ - -11.715149879455566, - 0.0, - -10.983560562133789 - ], - [ - -10.796329498291016, - 0.0, - -10.983560562133789 - ], - [ - -11.715149879455566, - 0.0, - -8.543549537658691 - ], - [ - -10.796329498291016, - 0.0, - -8.543549537658691 - ] - ], - "floor_height": 0.0, - "extrusion_height": 4.0, - "min_bounds": [ - -11.715149879455566, - 0.0, - -10.983560562133789 - ], - "max_bounds": [ - -10.796329498291016, - 4.0, - -8.543549537658691 - ] + "min_bounds": [5.0, -2.0, -10.0], + "max_bounds": [25.0, 2.0, 10.0] } ] } diff --git a/examples/viewer.py b/examples/viewer.py index 0fd7d1a540..0439e76c45 100644 --- a/examples/viewer.py +++ b/examples/viewer.py @@ -76,6 +76,9 @@ def __init__(self, sim_settings: Dict[str, Any]) -> None: self.debug_bullet_draw = False # draw active contact point debug line visualizations self.contact_debug_draw = False + # draw semantic region debug visualizations if present + self.semantic_region_debug_draw = False + # cache most recently loaded URDF file for quick-reload self.cached_urdf = "" @@ -176,6 +179,7 @@ def __init__(self, sim_settings: Dict[str, Any]) -> None: self.replay_renderer_cfg: Optional[ReplayRendererConfiguration] = None self.replay_renderer: Optional[ReplayRenderer] = None self.reconfigure_sim() + self.debug_semantic_colors = {} # compute NavMesh if not already loaded by the scene. if ( @@ -189,7 +193,7 @@ def __init__(self, sim_settings: Dict[str, Any]) -> None: logger.setLevel("INFO") self.print_help_text() - def draw_contact_debug(self): + def draw_contact_debug(self, debug_line_render: Any): """ This method is called to render a debug line overlay displaying active contact points and normals. Yellow lines show the contact distance along the normal and red lines show the contact normal at a fixed length. @@ -197,32 +201,46 @@ def draw_contact_debug(self): yellow = mn.Color4.yellow() red = mn.Color4.red() cps = self.sim.get_physics_contact_points() - self.sim.get_debug_line_render().set_line_width(1.5) + debug_line_render.set_line_width(1.5) camera_position = self.render_camera.render_camera.node.absolute_translation # only showing active contacts active_contacts = (x for x in cps if x.is_active) for cp in active_contacts: # red shows the contact distance - self.sim.get_debug_line_render().draw_transformed_line( + debug_line_render.draw_transformed_line( cp.position_on_b_in_ws, cp.position_on_b_in_ws + cp.contact_normal_on_b_in_ws * -cp.contact_distance, red, ) # yellow shows the contact normal at a fixed length for visualization - self.sim.get_debug_line_render().draw_transformed_line( + debug_line_render.draw_transformed_line( cp.position_on_b_in_ws, # + cp.contact_normal_on_b_in_ws * cp.contact_distance, cp.position_on_b_in_ws + cp.contact_normal_on_b_in_ws * 0.1, yellow, ) - self.sim.get_debug_line_render().draw_circle( + debug_line_render.draw_circle( translation=cp.position_on_b_in_ws, radius=0.005, color=yellow, normal=camera_position - cp.position_on_b_in_ws, ) + def draw_region_debug(self, debug_line_render: Any) -> None: + """ + Draw the semantic region wireframes. + """ + + for region in self.sim.semantic_scene.regions: + color = self.debug_semantic_colors.get(region.id, mn.Color4.magenta()) + for edge in region.volume_edges: + debug_line_render.draw_transformed_line( + edge[0], + edge[1], + color, + ) + def debug_draw(self): """ Additional draw commands to be called during draw_event. @@ -231,8 +249,18 @@ def debug_draw(self): render_cam = self.render_camera.render_camera proj_mat = render_cam.projection_matrix.__matmul__(render_cam.camera_matrix) self.sim.physics_debug_draw(proj_mat) + + debug_line_render = self.sim.get_debug_line_render() if self.contact_debug_draw: - self.draw_contact_debug() + self.draw_contact_debug(debug_line_render) + + if self.semantic_region_debug_draw: + if len(self.debug_semantic_colors) != len(self.sim.semantic_scene.regions): + for region in self.sim.semantic_scene.regions: + self.debug_semantic_colors[region.id] = mn.Color4( + mn.Vector3(np.random.random(3)) + ) + self.draw_region_debug(debug_line_render) def draw_event( self, @@ -473,6 +501,12 @@ def key_press_event(self, event: Application.KeyEvent) -> None: elif key == pressed.H: self.print_help_text() + elif key == pressed.J: + logger.info( + f"Toggle Region Draw from {self.semantic_region_debug_draw } to {not self.semantic_region_debug_draw}" + ) + # Toggle visualize semantic bboxes. Currently only regions supported + self.semantic_region_debug_draw = not self.semantic_region_debug_draw elif key == pressed.TAB: # NOTE: (+ALT) - reconfigure without cycling scenes @@ -590,7 +624,6 @@ def key_press_event(self, event: Application.KeyEvent) -> None: elif key == pressed.V: self.invert_gravity() logger.info("Command: gravity inverted") - elif key == pressed.N: # (default) - toggle navmesh visualization # NOTE: (+ALT) - re-sample the agent position on the NavMesh @@ -973,6 +1006,7 @@ def print_help_text(self) -> None: ',': Render a Bullet collision shape debug wireframe overlay (white=active, green=sleeping, blue=wants sleeping, red=can't sleep). 'c': Run a discrete collision detection pass and render a debug wireframe overlay showing active contact points and normals (yellow=fixed length normals, red=collision distances). (+SHIFT) Toggle the contact point debug render overlay on/off. + 'j' Toggle Semantic visualization bounds (currently only Semantic Region annotations) Object Interactions: SPACE: Toggle physics simulation on/off. diff --git a/src/esp/assets/ResourceManager.cpp b/src/esp/assets/ResourceManager.cpp index 52460aec15..2b06623bf5 100644 --- a/src/esp/assets/ResourceManager.cpp +++ b/src/esp/assets/ResourceManager.cpp @@ -305,77 +305,30 @@ std::vector ResourceManager::buildVertexColorMapReport( semanticScene_); } // ResourceManager::buildVertexColorMapReport -bool ResourceManager::loadSemanticSceneDescriptor( - const std::string& ssdFilename, +bool ResourceManager::loadSemanticScene( + const std::shared_ptr& + semanticAttr, const std::string& activeSceneName) { - namespace FileUtil = Cr::Utility::Path; + const std::string ssdFilename = + semanticAttr != nullptr ? semanticAttr->getSemanticDescriptorFilename() + : ""; semanticScene_ = nullptr; - if (ssdFilename != "") { + if ((ssdFilename != "") || ((semanticAttr != nullptr) && + (semanticAttr->getNumRegionInstances() > 0))) { bool success = false; - // semantic scene descriptor might not exist + // semantic scene descriptor might not exist so (re)create it. semanticScene_ = scene::SemanticScene::create(); - ESP_DEBUG(Mn::Debug::Flag::NoSpace) - << "SceneInstance : `" << activeSceneName - << "` proposed Semantic Scene Descriptor filename : `" << ssdFilename - << "`."; - - bool fileExists = FileUtil::exists(ssdFilename); - if (fileExists) { - // Attempt to load semantic scene descriptor specified in scene instance - // file, agnostic to file type inferred by name, if file exists. - success = scene::SemanticScene::loadSemanticSceneDescriptor( - ssdFilename, *semanticScene_); - if (success) { - ESP_DEBUG(Mn::Debug::Flag::NoSpace) - << "SSD with SceneInstanceAttributes-provided name `" << ssdFilename - << "` successfully found and loaded."; - } else { - // here if provided file exists but does not correspond to appropriate - // SSD - ESP_ERROR(Mn::Debug::Flag::NoSpace) - << "SSD Load Failure! File with " - "SceneInstanceAttributes-provided name `" - << ssdFilename << "` exists but failed to load."; - } - return success; - // if not success then try to construct a name - } else { - // attempt to look for specified file failed, attempt to build new file - // name by searching in path specified of specified file for - // info_semantic.json file for replica dataset - const std::string constructedFilename = FileUtil::join( - FileUtil::split(ssdFilename).first(), "info_semantic.json"); - fileExists = FileUtil::exists(constructedFilename); - if (fileExists) { - success = scene::SemanticScene::loadReplicaHouse(constructedFilename, - *semanticScene_); - if (success) { - ESP_DEBUG(Mn::Debug::Flag::NoSpace) - << "SSD for Replica using constructed file : `" - << constructedFilename << "` in directory with `" << ssdFilename - << "` loaded successfully"; - } else { - // here if constructed file exists but does not correspond to - // appropriate SSD or some loading error occurred. - ESP_ERROR(Mn::Debug::Flag::NoSpace) - << "SSD Load Failure! Replica file with constructed name `" - << ssdFilename << "` exists but failed to load."; - } - return success; - } else { - // neither provided non-empty filename nor constructed filename - // exists. This is probably due to an incorrect naming in the - // SceneInstanceAttributes - ESP_WARNING(Mn::Debug::Flag::NoSpace) - << "SSD File Naming Issue! Neither " - "SceneInstanceAttributes-provided name : `" - << ssdFilename << "` nor constructed filename : `" - << constructedFilename << "` exist on disk."; - return false; - } - } // if given SSD file name specified exists - } // if semantic scene descriptor specified in scene instance + success = scene::SemanticScene::loadSemanticSceneDescriptor( + semanticAttr, *semanticScene_); + + ESP_VERY_VERBOSE(Mn::Debug::Flag::NoSpace) + << "Attempt to create SemanticScene for Current Scene :`" + << activeSceneName << "` with ssdFilename : `" << ssdFilename + << "` :" << (success ? " " : "Not ") << "Successful"; + + return success; + } return false; } // ResourceManager::loadSemanticSceneDescriptor diff --git a/src/esp/assets/ResourceManager.h b/src/esp/assets/ResourceManager.h index 37cc2928f2..7faf7b6d7d 100644 --- a/src/esp/assets/ResourceManager.h +++ b/src/esp/assets/ResourceManager.h @@ -52,6 +52,7 @@ namespace attributes { class ObjectAttributes; class PbrShaderAttributes; class PhysicsManagerAttributes; +class SemanticAttributes; class SceneObjectInstanceAttributes; class StageAttributes; } // namespace attributes @@ -193,15 +194,19 @@ class ResourceManager { bool semanticSceneExists() const { return (semanticScene_ != nullptr); } /** - * @brief Load semantic scene descriptor file specified by @p ssdFilename , - * for the passed @p activeSceneName . - * @param ssdFilename The fully qualified filename candidate for the ssd file. + * @brief Load semantic scene data from descriptor file and metadata specified + * in @p semanticAttr , for the passed @p activeSceneName . + * @param semanticAttr Pointer to semantic attributes, if they exist. This + * will hold fully-qualified filename along with other attributes required to + * create the semantic scene. * @param activeSceneName Name of the currently active scene that we will be * loading the SSD for. * @return whether loaded successfully or not. */ - bool loadSemanticSceneDescriptor(const std::string& ssdFilename, - const std::string& activeSceneName); + bool loadSemanticScene( + const std::shared_ptr& + semanticAttr, + const std::string& activeSceneName); /** * @brief Load a scene mesh and add it to the specified @ref DrawableGroup as diff --git a/src/esp/bindings/SceneBindings.cpp b/src/esp/bindings/SceneBindings.cpp index 0d0bccc5a5..db9bf7ab35 100644 --- a/src/esp/bindings/SceneBindings.cpp +++ b/src/esp/bindings/SceneBindings.cpp @@ -140,6 +140,12 @@ void initSceneBindings(py::module& m) { .def("index", &Mp3dRegionCategory::index, "mapping"_a = "") .def("name", &Mp3dRegionCategory::name, "mapping"_a = ""); + // === Polyloop-based Semantic Region Category === + py::class_( + m, "LoopRegionCategory") + .def("index", &LoopRegionCategory::index, "mapping"_a = "") + .def("name", &LoopRegionCategory::name, "mapping"_a = ""); + // These two are (cyclically) referenced by multiple classes below, define // the classes first so pybind has the type definition available when binding // functions @@ -159,13 +165,30 @@ void initSceneBindings(py::module& m) { semanticRegion .def_property_readonly( "id", &SemanticRegion::id, - "The ID of the region, of the form ``_``") + "The ID of the region, either as the region's unique name, or of the " + "form ``_``") .def_property_readonly("level", &SemanticRegion::level) .def_property_readonly("aabb", &SemanticRegion::aabb) .def_property_readonly("category", &SemanticRegion::category, "The semantic category of the region") .def_property_readonly("objects", &SemanticRegion::objects, - "All objects in the region"); + "All objects in the region") + .def_property_readonly("poly_loop_points", + &SemanticRegion::getPolyLoopPoints, + "The points making up the polyloop for this " + "region, coplanar and parallel to the floor.") + .def_property_readonly( + "volume_edges", &SemanticRegion::getVisEdges, + "The edges, as pairs of points, that determine " + "the boundaries of the region. For visualizations.") + .def_property_readonly("floor_height", &SemanticRegion::getFloorHeight, + "The height above the x-z plane for the floor of " + "the semantic region.") + .def_property_readonly("extrusion_height", + &SemanticRegion::getExtrusionHeight, + "The height of the extrusion above the floor.") + .def("contains", &SemanticRegion::contains, "point"_a, + "Check whether the given point is contained in the given region."); // ==== SemanticObject ==== semanticObject diff --git a/src/esp/metadata/attributes/SceneDatasetAttributes.cpp b/src/esp/metadata/attributes/SceneDatasetAttributes.cpp index efd74ef2e8..e5794e6bb6 100644 --- a/src/esp/metadata/attributes/SceneDatasetAttributes.cpp +++ b/src/esp/metadata/attributes/SceneDatasetAttributes.cpp @@ -129,14 +129,15 @@ bool SceneDatasetAttributes::addNewSceneInstanceToDataset( } // SceneDatasetAttributes::addSceneInstanceToDataset void SceneDatasetAttributes::createSemanticAttribsFromDS( - const std::string& semanticHandle) { + const std::string& semanticHandle, + const std::string& dbgSourceAttribs) { const std::string infoPrefix = - Cr::Utility::formatString("Dataset : '{}' : Semantic Attributes '{}`", + Cr::Utility::formatString("Dataset : `{}` : Semantic Attributes `{}`", this->getSimplifiedHandle(), semanticHandle); // Check if exists or build a new SemanticAttributes with the passed Stage's - // semantic data set - if (semanticAttributesManager_->getFullAttrNameFromStr(semanticHandle) == - "") { + // or Scene Dataset Config's semantic data set/file + // Should match -exactly- + if (!semanticAttributesManager_->getObjectLibHasHandle(semanticHandle)) { // DNE, create a new one ESP_VERY_VERBOSE(Mn::Debug::Flag::NoSpace) << infoPrefix @@ -144,8 +145,8 @@ void SceneDatasetAttributes::createSemanticAttribsFromDS( semanticAttributesManager_->createObject(semanticHandle, true); } else { ESP_VERY_VERBOSE(Mn::Debug::Flag::NoSpace) - << infoPrefix - << "specified in Stage Attributes already exists in dataset library. "; + << infoPrefix << " specified in " << dbgSourceAttribs + << " already exists in dataset library. "; } } // SceneDatasetAttributes::createSemanticAttribsFromDS @@ -157,7 +158,7 @@ std::string SceneDatasetAttributes::addSemanticSceneDescrPathEntry( bool setSemanticAssetData = (semanticAssetFilename != ""); bool setSSDFilename = (ssdFilename != ""); // create a semantic attributes if DNE with given handle - this->createSemanticAttribsFromDS(semanticHandle); + this->createSemanticAttribsFromDS(semanticHandle, "Stage Attributes"); // Get actual object and set semantic data if appropriate auto semanticAttr = @@ -186,7 +187,8 @@ void SceneDatasetAttributes::setSemanticAttrSSDFilenames( const std::string ssdFilename = entry.second; // create a semantic attributes if DNE with given handle - this->createSemanticAttribsFromDS(semanticHandle); + this->createSemanticAttribsFromDS(semanticHandle, + "Dataset Config Map Entry"); // Get actual object and set semantic data if appropriate auto semanticAttr = semanticAttributesManager_->getObjectByHandle(semanticHandle); diff --git a/src/esp/metadata/attributes/SceneDatasetAttributes.h b/src/esp/metadata/attributes/SceneDatasetAttributes.h index 10e51f3829..ccb9a81136 100644 --- a/src/esp/metadata/attributes/SceneDatasetAttributes.h +++ b/src/esp/metadata/attributes/SceneDatasetAttributes.h @@ -401,8 +401,11 @@ class SceneDatasetAttributes : public AbstractAttributes { * @brief will create a new @ref esp::metadata::attributes::SemanticAttributes * with the given handle if one does not exist. * @param semanticHandle The name of the attributes to create. + * @param dbgSourceAttribs The name of the caller, for debug purposes should + * this fail. */ - void createSemanticAttribsFromDS(const std::string& semanticHandle); + void createSemanticAttribsFromDS(const std::string& semanticHandle, + const std::string& dbgSourceAttribs); /** * @brief Retrieve a comma-separated string holding the header values for the diff --git a/src/esp/metadata/attributes/SemanticAttributes.cpp b/src/esp/metadata/attributes/SemanticAttributes.cpp index 932ab2feff..855958dabe 100644 --- a/src/esp/metadata/attributes/SemanticAttributes.cpp +++ b/src/esp/metadata/attributes/SemanticAttributes.cpp @@ -8,17 +8,17 @@ namespace metadata { namespace attributes { ///////////////////////////////// -// SemanticRegionAttributes +// SemanticVolumeAttributes -SemanticRegionAttributes::SemanticRegionAttributes(const std::string& handle) - : AbstractAttributes("SemanticRegionAttributes", handle) { +SemanticVolumeAttributes::SemanticVolumeAttributes(const std::string& handle) + : AbstractAttributes("SemanticVolumeAttributes", handle) { // Initialize values set("floor_height", 0.0); set("extrusion_height", 2.5); -} // SemanticRegionAttributes ctor +} // SemanticVolumeAttributes ctor -std::string SemanticRegionAttributes::getObjectInfoHeaderInternal() const { +std::string SemanticVolumeAttributes::getObjectInfoHeaderInternal() const { std::string res = "Name,Label,Floor Height,Extrusion Height,Min Bounds,Max Bounds,"; int iter = 0; @@ -29,7 +29,7 @@ std::string SemanticRegionAttributes::getObjectInfoHeaderInternal() const { return res; } -std::string SemanticRegionAttributes::getObjectInfoInternal() const { +std::string SemanticVolumeAttributes::getObjectInfoInternal() const { std::string res = Cr::Utility::formatString( "{},{},{},{},{},{},", getAsString("handle"), getAsString("label"), getAsString("floor_height"), getAsString("extrusion_height"), @@ -42,12 +42,12 @@ std::string SemanticRegionAttributes::getObjectInfoInternal() const { Cr::Utility::formatInto(res, res.size(), "],"); return res; -} // SemanticRegionAttributes::getObjectInfoInternal() +} // SemanticVolumeAttributes::getObjectInfoInternal() -void SemanticRegionAttributes::writeValuesToJson( +void SemanticVolumeAttributes::writeValuesToJson( io::JsonGenericValue& jsonObj, io::JsonAllocator& allocator) const { - // map "handle" to "name" key in json - this SemanticRegionAttributes' handle + // map "handle" to "name" key in json - this SemanticVolumeAttributes' handle // is its unique name writeValueToJson("handle", "name", jsonObj, allocator); writeValueToJson("label", jsonObj, allocator); @@ -60,7 +60,7 @@ void SemanticRegionAttributes::writeValuesToJson( io::addMember(jsonObj, "poly_loop", polyLoop_, allocator); } -} // SemanticRegionAttributes::writeValuesToJson +} // SemanticVolumeAttributes::writeValuesToJson ///////////////////////////////// // SemanticAttributes @@ -85,7 +85,7 @@ SemanticAttributes::SemanticAttributes(const SemanticAttributes& otr) availableRegionInstIDs_(otr.availableRegionInstIDs_) { // get refs to internal subconfigs for semantic region attributes regionAnnotationConfig_ = editSubconfig("region_annotations"); - copySubconfigIntoMe(otr.regionAnnotationConfig_, + copySubconfigIntoMe(otr.regionAnnotationConfig_, regionAnnotationConfig_); } // SemanticAttributes copy ctor @@ -162,7 +162,7 @@ SemanticAttributes& SemanticAttributes::operator=( // get refs to internal subconfigs for semantic region attributes regionAnnotationConfig_ = editSubconfig("region_annotations"); - copySubconfigIntoMe(otr.regionAnnotationConfig_, + copySubconfigIntoMe(otr.regionAnnotationConfig_, regionAnnotationConfig_); } return *this; diff --git a/src/esp/metadata/attributes/SemanticAttributes.h b/src/esp/metadata/attributes/SemanticAttributes.h index 2a465e8270..c78242e64c 100644 --- a/src/esp/metadata/attributes/SemanticAttributes.h +++ b/src/esp/metadata/attributes/SemanticAttributes.h @@ -15,20 +15,20 @@ namespace attributes { // Semantic attributes /** - * @brief This class describes the semantic attributes for a specific region - * annotation. + * @brief This class describes the attributes describing some Semantic + * Volume. Currently only used for region annotations. */ -class SemanticRegionAttributes : public AbstractAttributes { +class SemanticVolumeAttributes : public AbstractAttributes { public: - explicit SemanticRegionAttributes(const std::string& handle); + explicit SemanticVolumeAttributes(const std::string& handle); /** - * @brief Get the label assigned to this semantic region. + * @brief Get the label assigned to this semantic volume. */ std::string getLabel() const { return get("label"); } /** - * @brief Set the label assigned to this semantic region. + * @brief Set the label assigned to this semantic volume. */ void setLabel(const std::string& _label) { set("label", _label); } @@ -103,8 +103,8 @@ class SemanticRegionAttributes : public AbstractAttributes { /** * @brief Populate a JSON object with all the first-level values held in this - * SemanticRegionAttributes. Default is overridden to handle special - * cases for SemanticRegionAttributes. + * SemanticVolumeAttributes. Default is overridden to handle special + * cases for SemanticVolumeAttributes. */ void writeValuesToJson(io::JsonGenericValue& jsonObj, io::JsonAllocator& allocator) const override; @@ -129,7 +129,7 @@ class SemanticRegionAttributes : public AbstractAttributes { std::vector polyLoop_{}; public: - ESP_SMART_POINTERS(SemanticRegionAttributes) + ESP_SMART_POINTERS(SemanticVolumeAttributes) }; // class SemanticAttributes @@ -241,8 +241,8 @@ class SemanticAttributes : public AbstractAttributes { /** * @brief Add an object instance attributes to this scene instance. */ - void addRegionInstanceAttrs(SemanticRegionAttributes::ptr _regionInstance) { - setSubAttributesInternal( + void addRegionInstanceAttrs(SemanticVolumeAttributes::ptr _regionInstance) { + setSubAttributesInternal( _regionInstance, availableRegionInstIDs_, regionAnnotationConfig_, "region_desc_"); } @@ -250,12 +250,12 @@ class SemanticAttributes : public AbstractAttributes { /** * @brief Get the object instance descriptions for this scene */ - std::vector getRegionInstances() const { - return getSubAttributesListInternal( + std::vector getRegionInstances() const { + return getSubAttributesListInternal( regionAnnotationConfig_); } /** - * @brief Return the number of defined @ref SemanticRegionAttributes + * @brief Return the number of defined @ref SemanticVolumeAttributes * subconfigs in this scene instance. */ int getNumRegionInstances() const { @@ -272,8 +272,8 @@ class SemanticAttributes : public AbstractAttributes { /** * @brief Populate a JSON object with all the first-level values held in this - * SemanticRegionAttributes. Default is overridden to handle special - * cases for SemanticRegionAttributes. + * SemanticVolumeAttributes. Default is overridden to handle special + * cases for SemanticVolumeAttributes. */ void writeValuesToJson(io::JsonGenericValue& jsonObj, io::JsonAllocator& allocator) const override; @@ -308,7 +308,7 @@ class SemanticAttributes : public AbstractAttributes { * @brief Retrieve a comma-separated string holding the header values for * the info returned for this managed object, type-specific. Don't use this * method, since we have both SemanticAttributes data to save and the - * individual SemanticRegionAttributes data to save. + * individual SemanticVolumeAttributes data to save. */ std::string getObjectInfoHeaderInternal() const override { return ""; } diff --git a/src/esp/metadata/managers/AttributesManagerBase.h b/src/esp/metadata/managers/AttributesManagerBase.h index 7efded9da3..3a35c143bb 100644 --- a/src/esp/metadata/managers/AttributesManagerBase.h +++ b/src/esp/metadata/managers/AttributesManagerBase.h @@ -262,12 +262,18 @@ class AttributesManager : public ManagedFileBasedContainer { /** * @brief Set a filename attribute to hold the appropriate data if the * existing attribute's given path contains the sentinel tag value defined at - * @ref esp::metadata::CONFIG_NAME_AS_ASSET_FILENAME. This will be used in + * @ref esp::metadata::CONFIG_NAME_AS_ASSET_FILENAME. This will be called from * the Scene Dataset configuration file in the "default_attributes" tag for * any attributes which consume file names to specify that the name specified * as the instanced attributes should also be used to build the name of the * specified asset. The tag value will be replaced by the attributes object's - * simplified handle. + * simplified handle, or if unable to be found, with an empty string. + * + * If the given data does not contain the @ref esp::metadata::CONFIG_NAME_AS_ASSET_FILENAME + * tag, this will attempt to find the file referenced in @p srcAssetFilename + * directly, first as it is given, and then by prefixing it with the current + * attributes' source file directory. If found it will call @p filenameSetter + * with the successful string. * * This will only be called from the specified manager's initNewObjectInternal * function, where the attributes is initially built from a default attributes @@ -550,9 +556,16 @@ bool AttributesManager::setFilenameFromDefaultTag( if (srcAssetFilename.empty()) { return false; } - std::string tempStr(srcAssetFilename); + // First check if tag references a file that already exists on disk and is + // able to be found + if (Cr::Utility::Path::exists(srcAssetFilename)) { + // set filename with verified filepath + filenameSetter(srcAssetFilename); + return true; + } const auto loc = srcAssetFilename.find(CONFIG_NAME_AS_ASSET_FILENAME); if (loc != std::string::npos) { + std::string tempStr(srcAssetFilename); // sentinel tag is found - replace tag with simplified handle of // attributes and use filenameSetter and return whether this file exists or // not. @@ -574,9 +587,19 @@ bool AttributesManager::setFilenameFromDefaultTag( // out of options, clear out the wild-card default so that init-based // default is derived and used. filenameSetter(""); + return false; + } + // no sentinel tag found - check if existing non-empty field exists. + std::string tempStr = + Cr::Utility::Path::join(attributes->getFileDirectory(), srcAssetFilename); + if (Cr::Utility::Path::exists(tempStr)) { + // path-prefixed filename exists on disk, so set as filename + filenameSetter(tempStr); + return true; } - // no tag found - check if existing non-empty field exists. - return Cr::Utility::Path::exists(srcAssetFilename); + // No file exists with passed name, either as relative path or as + // wild-card-equipped relative path + return false; } // AttributesManager::setFilenameFromDefaultTag template @@ -593,8 +616,12 @@ bool AttributesManager::setAttributesHandleFromDefaultTag( // sentinel tag is found - replace tag with simplified handle of // attributes and use filenameSetter and return whether this file exists or // not. - tempStr.replace(loc, strlen(CONFIG_NAME_AS_ASSET_FILENAME), - attributes->getSimplifiedHandle()); + auto simpleHandle = attributes->getSimplifiedHandle(); + if (simpleHandle == "NONE") { + tempStr = ""; + } else { + tempStr.replace(loc, strlen(CONFIG_NAME_AS_ASSET_FILENAME), simpleHandle); + } // DOES NOT VERIFY THAT HANDLE REFERENCES ACTUAL OBJECT handleSetter(tempStr); return true; diff --git a/src/esp/metadata/managers/SemanticAttributesManager.cpp b/src/esp/metadata/managers/SemanticAttributesManager.cpp index 04e094f080..25f0ed70cb 100644 --- a/src/esp/metadata/managers/SemanticAttributesManager.cpp +++ b/src/esp/metadata/managers/SemanticAttributesManager.cpp @@ -13,7 +13,7 @@ namespace esp { namespace metadata { using attributes::SemanticAttributes; -using attributes::SemanticRegionAttributes; +using attributes::SemanticVolumeAttributes; namespace managers { SemanticAttributes::ptr SemanticAttributesManager::createObject( @@ -76,8 +76,8 @@ SemanticAttributes::ptr SemanticAttributesManager::initNewObjectInternal( return newAttributes; } // SemanticAttributesManager::initNewObjectInternal -void SemanticAttributesManager::setSemanticRegionAttributesFromJson( - const SemanticRegionAttributes::ptr& instanceAttrs, +void SemanticAttributesManager::setSemanticVolumeAttributesFromJson( + const SemanticVolumeAttributes::ptr& instanceAttrs, const io::JsonGenericValue& jCell) { // unique name for region io::jsonIntoConstSetter( @@ -138,14 +138,14 @@ void SemanticAttributesManager::setSemanticRegionAttributesFromJson( // check for user defined attributes this->parseUserDefinedJsonVals(instanceAttrs, jCell); -} // SemanticAttributesManager::setSemanticRegionAttributesFromJson +} // SemanticAttributesManager::setSemanticVolumeAttributesFromJson -SemanticRegionAttributes::ptr +SemanticVolumeAttributes::ptr SemanticAttributesManager::createRegionAttributesFromJSON( const io::JsonGenericValue& jCell) { - SemanticRegionAttributes::ptr regionAttrs = createEmptyRegionAttributes(""); + SemanticVolumeAttributes::ptr regionAttrs = createEmptyRegionAttributes(""); // populate attributes - this->setSemanticRegionAttributesFromJson(regionAttrs, jCell); + this->setSemanticVolumeAttributesFromJson(regionAttrs, jCell); return regionAttrs; } // SemanticAttributesManager::createRegionAttributesFromJSON diff --git a/src/esp/metadata/managers/SemanticAttributesManager.h b/src/esp/metadata/managers/SemanticAttributesManager.h index 76cc85c691..af1230b129 100644 --- a/src/esp/metadata/managers/SemanticAttributesManager.h +++ b/src/esp/metadata/managers/SemanticAttributesManager.h @@ -63,35 +63,35 @@ class SemanticAttributesManager /** * @brief This will return a @ref - * attributes::SemanticRegionAttributes object with passed handle. + * attributes::SemanticVolumeAttributes object with passed handle. */ - attributes::SemanticRegionAttributes::ptr createEmptyRegionAttributes( + attributes::SemanticVolumeAttributes::ptr createEmptyRegionAttributes( const std::string& handle) { - return attributes::SemanticRegionAttributes::create(handle); + return attributes::SemanticVolumeAttributes::create(handle); } protected: /** * @brief Used Internally. Create a @ref - * esp::metadata::attributes::SemanticRegionAttributes object from the + * esp::metadata::attributes::SemanticVolumeAttributes object from the * passed JSON doc. * @param jCell JSON object containing the description of the semantic retion * @return the constructed @ref - * esp::metadata::attributes::SemanticRegionAttributes object + * esp::metadata::attributes::SemanticVolumeAttributes object */ - attributes::SemanticRegionAttributes::ptr createRegionAttributesFromJSON( + attributes::SemanticVolumeAttributes::ptr createRegionAttributesFromJSON( const io::JsonGenericValue& jCell); /** * @brief Populate an existing @ref - * metadata::attributes::SemanticRegionAttributes from a JSON config. + * metadata::attributes::SemanticVolumeAttributes from a JSON config. * * @param attributes (out) the attributes to populate with JSON values * @param jCell JSON document to parse to populate the - * @ref metadata::attributes::SemanticRegionAttributes + * @ref metadata::attributes::SemanticVolumeAttributes */ - void setSemanticRegionAttributesFromJson( - const attributes::SemanticRegionAttributes::ptr& attributes, + void setSemanticVolumeAttributesFromJson( + const attributes::SemanticVolumeAttributes::ptr& attributes, const io::JsonGenericValue& jCell); /** diff --git a/src/esp/scene/SemanticScene.cpp b/src/esp/scene/SemanticScene.cpp index 537c9b0073..558687fd6e 100644 --- a/src/esp/scene/SemanticScene.cpp +++ b/src/esp/scene/SemanticScene.cpp @@ -3,6 +3,7 @@ // LICENSE file in the root directory of this source tree. #include "SemanticScene.h" +#include #include "GibsonSemanticScene.h" #include "Mp3dSemanticScene.h" #include "ReplicaSemanticScene.h" @@ -17,93 +18,252 @@ #include "esp/io/Io.h" #include "esp/io/Json.h" +#include "esp/metadata/attributes/SemanticAttributes.h" namespace esp { namespace scene { bool SemanticScene:: - loadSemanticSceneDescriptor(const std::string& ssdFileName, SemanticScene& scene, const quatf& rotation /* = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY) */) { - bool success = false; - bool exists = checkFileExists(ssdFileName, "loadSemanticSceneDescriptor"); - if (exists) { - // TODO: we need to investigate the possibility of adding an identifying tag - // to the SSD config files. - // Try file load mechanisms for various types - // first try Mp3d - try { - // only returns false if file does not exist, or attempting to open it - // fails - // open stream and determine house format version + loadSemanticSceneDescriptor(const std::shared_ptr& semanticAttr, SemanticScene& scene, const quatf& rotation /* = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY) */) { + const std::string ssdFileName = + semanticAttr != nullptr ? semanticAttr->getSemanticDescriptorFilename() + : ""; + + bool loadSuccess = false; + if (ssdFileName != "") { + bool fileExists = + checkFileExists(ssdFileName, "loadSemanticSceneDescriptor"); + if (fileExists) { + // TODO: we need to investigate the possibility of adding an identifying + // tag to the SSD config files. Try file load mechanisms for various types + // first try Mp3d try { - std::ifstream ifs = std::ifstream(ssdFileName); - std::string header; - std::getline(ifs, header); - if (header.find("ASCII 1.1") != std::string::npos) { - success = buildMp3dHouse(ifs, scene, rotation); - } else if (header.find("HM3D Semantic Annotations") != - std::string::npos) { - success = buildHM3DHouse(ifs, scene, rotation); + // only returns false if file does not exist, or attempting to open it + // fails + // open stream and determine house format version + try { + std::ifstream ifs = std::ifstream(ssdFileName); + std::string header; + std::getline(ifs, header); + if (header.find("ASCII 1.1") != std::string::npos) { + loadSuccess = buildMp3dHouse(ifs, scene, rotation); + } else if (header.find("HM3D Semantic Annotations") != + std::string::npos) { + loadSuccess = buildHM3DHouse(ifs, scene, rotation); + } + } catch (...) { + loadSuccess = false; + } + if (!loadSuccess) { + // if not successful then attempt to load known json files + const io::JsonDocument& jsonDoc = io::parseJsonFile(ssdFileName); + // if no error thrown, then we have loaded a json file of given name + + io::JsonGenericValue::ConstMemberIterator hasJsonObjIter = + jsonDoc.FindMember("objects"); + + bool hasCorrectObjects = (hasJsonObjIter != jsonDoc.MemberEnd() && + hasJsonObjIter->value.IsArray()); + + io::JsonGenericValue::ConstMemberIterator hasJsonClassIter = + jsonDoc.FindMember("classes"); + + // check if also has "classes" tag, otherwise will assume it is a + // gibson file + if (hasJsonClassIter != jsonDoc.MemberEnd() && + hasJsonClassIter->value.IsArray()) { + // attempt to load replica or replicaCAD if has classes (replicaCAD + // does not have objects in SSDescriptor) + loadSuccess = + buildReplicaHouse(jsonDoc, scene, hasCorrectObjects, rotation); + } else if (hasCorrectObjects) { + // attempt to load gibson if has objects but not classes + loadSuccess = buildGibsonHouse(jsonDoc, scene, rotation); + } } } catch (...) { - success = false; + // if error thrown, assume it is because file load attempt fails + loadSuccess = false; } - if (!success) { - // if not successful then attempt to load known json files - const io::JsonDocument& jsonDoc = io::parseJsonFile(ssdFileName); - // if no error thrown, then we have loaded a json file of given name - - io::JsonGenericValue::ConstMemberIterator hasJsonObjIter = - jsonDoc.FindMember("objects"); - - bool hasCorrectObjects = (hasJsonObjIter != jsonDoc.MemberEnd() && - hasJsonObjIter->value.IsArray()); - - io::JsonGenericValue::ConstMemberIterator hasJsonClassIter = - jsonDoc.FindMember("classes"); - - // check if also has "classes" tag, otherwise will assume it is a - // gibson file - if (hasJsonClassIter != jsonDoc.MemberEnd() && - hasJsonClassIter->value.IsArray()) { - // attempt to load replica or replicaCAD if has classes (replicaCAD - // does not have objects in SSDescriptor) - success = - buildReplicaHouse(jsonDoc, scene, hasCorrectObjects, rotation); - } else if (hasCorrectObjects) { - // attempt to load gibson if has objects but not classes - success = buildGibsonHouse(jsonDoc, scene, rotation); + } + if (!loadSuccess) { + // should only reach here if either specified file exists but was not + // loaded successfully or file does not exist. + + // attempt to look for specified file failed, so attempt to build new file + // name by searching in path specified of specified file for + // info_semantic.json file for replica dataset + namespace FileUtil = Cr::Utility::Path; + // check if constructed replica file exists in directory of passed + // ssdFileName + const std::string constructedFilename = FileUtil::join( + FileUtil::split(ssdFileName).first(), "info_semantic.json"); + if (FileUtil::exists(constructedFilename)) { + loadSuccess = + scene::SemanticScene::loadReplicaHouse(constructedFilename, scene); + + if (loadSuccess) { + ESP_DEBUG(Mn::Debug::Flag::NoSpace) + << "SSD for Replica using constructed file : `" + << constructedFilename << "` in directory with `" << ssdFileName + << "` loaded successfully"; + } else { + // here if constructed file exists but does not correspond to + // appropriate SSD or some loading error occurred. + ESP_ERROR(Mn::Debug::Flag::NoSpace) + << "SSD Load Failure! Replica file with constructed name `" + << ssdFileName << "` exists but failed to load."; } + } else { + // neither provided non-empty filename nor constructed filename + // exists. This is probably due to an incorrect naming in the + // SemanticAttributes + ESP_WARNING(Mn::Debug::Flag::NoSpace) + << "SSD File Naming Issue! Neither " + "SemanticAttributes-provided name : `" + << ssdFileName << "` nor constructed filename : `" + << constructedFilename << "` exist on disk."; + loadSuccess = false; } - if (success) { - // if successfully loaded, return true; - return true; + } + + if (loadSuccess) { + ESP_DEBUG(Mn::Debug::Flag::NoSpace) + << "SSD with SemanticAttributes-provided name `" << ssdFileName + << "` successfully found and loaded."; + } else { + // here if provided file exists but does not correspond to appropriate + // SSD + ESP_ERROR(Mn::Debug::Flag::NoSpace) + << "SSD Load Failure! File with " + "SemanticAttributes-provided name `" + << ssdFileName << "` exists but failed to load."; + } + } + + if (semanticAttr != nullptr) { + if (semanticAttr->getNumRegionInstances() > 0) { + ESP_DEBUG(Mn::Debug::Flag::NoSpace) + << "Semantic Attributes : `" << semanticAttr->getHandle() << "` has " + << semanticAttr->getNumRegionInstances() << " regions defined."; + // Build Semantic regions for each SemanticRegion attributes instance + const auto regionInstances = semanticAttr->getRegionInstances(); + + scene.regions_.clear(); + scene.regions_.reserve(regionInstances.size()); + int idx = 0; + for (const auto& regionInstance : regionInstances) { + auto regionPtr = SemanticRegion::create(); + // Unique name + regionPtr->name_ = regionInstance->getHandle(); + // Build a category + regionPtr->category_ = + LoopRegionCategory::create(idx++, regionInstance->getLabel()); + // Set y heights + regionPtr->extrusionHeight_ = regionInstance->getExtrusionHeight(); + regionPtr->floorHeight_ = regionInstance->getFloorHeight(); + // Set bbox + const Mn::Vector3 min = regionInstance->getMinBounds(); + const Mn::Vector3 max = regionInstance->getMaxBounds(); + regionPtr->setBBox(min, max); + // Set polyloop points and precalc polyloop edge vectors + const std::vector& loopPoints = + regionInstance->getPolyLoop(); + + std::size_t numPts = loopPoints.size(); + regionPtr->polyLoopPoints_ = std::vector(numPts); + regionPtr->visEdges_ = + std::vector>(4 * numPts); + + // Save points and edges + int eIdx = 0; + float yExtrusion = static_cast(regionPtr->extrusionHeight_ + + regionPtr->floorHeight_); + for (std::size_t i = 0; i < numPts; ++i) { + Mn::Vector3 currPoint = loopPoints[i]; + regionPtr->polyLoopPoints_[i] = {currPoint.x(), currPoint.z()}; + // Edges + Mn::Vector3 currExtPt = {currPoint.x(), yExtrusion, currPoint.z()}; + + std::size_t nextIdx = ((i + 1) % numPts); + Mn::Vector3 nextPoint = loopPoints[nextIdx]; + Mn::Vector3 nextExtPt = {nextPoint.x(), yExtrusion, nextPoint.z()}; + // Horizontal edge + regionPtr->visEdges_[eIdx++] = {currPoint, nextPoint}; + // Vertical edge + regionPtr->visEdges_[eIdx++] = {currPoint, currExtPt}; + // Extruded horizontal edge + regionPtr->visEdges_[eIdx++] = {currExtPt, nextExtPt}; + // Diagonal edge + regionPtr->visEdges_[eIdx++] = {currPoint, nextExtPt}; + } + + scene.regions_.emplace_back(std::move(regionPtr)); } - } catch (...) { - // if error thrown, assume it is because file load attempt fails - success = false; + } else { // if semantic attributes specifes region annotations + ESP_DEBUG(Mn::Debug::Flag::NoSpace) + << "Semantic Attributes : `" << semanticAttr->getHandle() + << "` does not have any regions defined."; } + } else { + ESP_DEBUG(Mn::Debug::Flag::NoSpace) << "Semantic attributes do not exist."; + } // if semanticAttrs exist or not + + return loadSuccess; + +} // SemanticScene::loadSemanticSceneDescriptor + +bool SemanticRegion::contains(const Mn::Vector3& pt) const { + auto checkPt = [&](float x, float x0, float x1, float y, float y0, + float y1) -> bool { + float interp = ((y - y0) / (y1 - y0)); + return ((y < y0) != (y < y1)) && (x < x0 + interp * (x1 - x0)); + }; + + // First check height + if ((pt.y() < floorHeight_) || (pt.y() > (floorHeight_ + extrusionHeight_))) { + return false; } - // should only reach here if not successfully loaded - namespace FileUtil = Cr::Utility::Path; - // check if constructed replica file exists in directory of passed - // ssdFileName - const std::string tmpFName = FileUtil::join( - FileUtil::split(ssdFileName).first(), "info_semantic.json"); - if (FileUtil::exists(tmpFName)) { - success = scene::SemanticScene::loadReplicaHouse(tmpFName, scene); + // Next check bbox + if (!bbox_.contains(Mn::EigenIntegration::cast(pt))) { + return false; + } + // Lastly, count casts across edges. + int count = 0; + int numPts = polyLoopPoints_.size(); + for (int i = 0; i < numPts; ++i) { + const auto stPt = polyLoopPoints_[i]; + const auto endPt = polyLoopPoints_[(i + 1) % numPts]; + if (stPt == endPt) { + // Skip points that are equal. + continue; + } + // use x-z in point (i.e. projecting to ground plane) + bool checkCrossing = + checkPt(pt.x(), stPt.x(), endPt.x(), pt.z(), stPt.y(), endPt.y()); + if (checkCrossing) { + ++count; + } } - return success; -} // SemanticScene::loadSemanticSceneDescriptor + // Want odd crossings for being inside + return (count % 2 == 1); +} // SemanticRegion::contains + +void SemanticRegion::setBBox(const Mn::Vector3& min, const Mn::Vector3& max) { + bbox_ = box3f(Mn::EigenIntegration::cast(min), + Mn::EigenIntegration::cast(max)); +} // SemanticRegion::setBBox namespace { /** - * @brief Build an AABB for a given set of vertex indices in @p verts list, and - * return in a std::pair, along with the count of verts used to build the AABB. + * @brief Build an AABB for a given set of vertex indices in @p verts list, + * and return in a std::pair, along with the count of verts used to build the + * AABB. * @param colorInt Semantic Color of object * @param verts The mesh's vertex buffer. - * @param setOfIDXs set of vertex IDXs in the vertex buffer being used to build - * the resultant AABB. + * @param setOfIDXs set of vertex IDXs in the vertex buffer being used to + * build the resultant AABB. */ CCSemanticObject::ptr buildCCSemanticObjForSetOfVerts( uint32_t colorInt, @@ -132,8 +292,8 @@ CCSemanticObject::ptr buildCCSemanticObjForSetOfVerts( } // buildCCSemanticObjForSetOfVerts /** - * @brief build per-SSD object vector of known semantic IDs - doing this in case - * semanticIDs are not contiguous. + * @brief build per-SSD object vector of known semantic IDs - doing this in + * case semanticIDs are not contiguous. */ std::vector getObjsIdxToIDMap( @@ -179,8 +339,8 @@ SemanticScene::buildCCBasedSemanticObjs( } } - // only map to semantic ID if semanticScene exists, otherwise return map with - // objects keyed by hex color + // only map to semantic ID if semanticScene exists, otherwise return map + // with objects keyed by hex color if (!semanticScene) { return semanticCCObjsByVertTag; } @@ -375,8 +535,9 @@ std::vector SemanticScene::buildSemanticOBBs( // number of unique ssdObjs mappings. for (int vertIdx = 0; vertIdx < vertSemanticIDs.size(); ++vertIdx) { - // semantic ID on vertex - valid values are 1->semanticIDToSSOBJidx.size(). - // Invalid/unknown semantic ids are > semanticIDToSSOBJidx.size() + // semantic ID on vertex - valid values are + // 1->semanticIDToSSOBJidx.size(). Invalid/unknown semantic ids are > + // semanticIDToSSOBJidx.size() const auto semanticID = vertSemanticIDs[vertIdx]; if ((semanticID >= 0) && (semanticID < semanticIDToSSOBJidx.size())) { const auto vert = vertices[vertIdx]; @@ -416,7 +577,8 @@ std::vector SemanticScene::buildSemanticOBBs( center = .5f * (vertMax[semanticID] + vertMin[semanticID]); dims = vertMax[semanticID] - vertMin[semanticID]; ESP_VERY_VERBOSE() << Cr::Utility::formatString( - "{} Semantic ID : {} : color : {} tag : {} present in {} verts | BB " + "{} Semantic ID : {} : color : {} tag : {} present in {} verts | " + "BB " "Center [{} {} {}] Dims [{} {} {}]", msgPrefix, semanticID, geo::getColorAsString(ssdObj.getColor()), ssdObj.id(), vertCounts[semanticID], center.x(), center.y(), diff --git a/src/esp/scene/SemanticScene.h b/src/esp/scene/SemanticScene.h index 2cd1ab5379..561bd65bc4 100644 --- a/src/esp/scene/SemanticScene.h +++ b/src/esp/scene/SemanticScene.h @@ -18,6 +18,11 @@ #include "esp/io/Json.h" namespace esp { +namespace metadata { +namespace attributes { +class SemanticAttributes; +} +} // namespace metadata namespace scene { //! Represents a semantic category @@ -91,14 +96,17 @@ class SemanticScene { /** * @brief Attempt to load SemanticScene descriptor from an unknown file type. - * @param filename the name of the semantic scene descriptor (house file) to - * attempt to load + * @param semanticAttr The semantic attributes containing a reference to the + * name of the semantic scene descriptor (house file) to attempt to load, + * along with informationd describing semantic constructions (region + * annotations). * @param scene reference to sceneNode to assign semantic scene to * @param rotation rotation to apply to semantic scene upon load. * @return successfully loaded */ static bool loadSemanticSceneDescriptor( - const std::string& filename, + const std::shared_ptr& + semanticAttr, SemanticScene& scene, const quatf& rotation = quatf::FromTwoVectors(-vec3f::UnitZ(), geo::ESP_GRAVITY)); @@ -424,17 +432,44 @@ class SemanticLevel { ESP_SMART_POINTERS(SemanticLevel) }; +class LoopRegionCategory : public SemanticCategory { + public: + LoopRegionCategory(const int id, const std::string& name) + : id_(id), name_(name) {} + + int index(const std::string& /*mapping*/) const override { return id_; } + + std::string name(const std::string& mapping) const override { + if (mapping == "category" || mapping.empty()) { + return name_; + } else { + ESP_ERROR() << "Unknown mapping type:" << mapping; + return "UNKNOWN"; + } + } + + protected: + int id_; + std::string name_; + ESP_SMART_POINTERS(LoopRegionCategory) + +}; // class LoopRegionCategory + //! Represents a region (typically room) in a level of a house class SemanticRegion { public: virtual ~SemanticRegion() = default; virtual std::string id() const { + if (!name_.empty()) { + return name_; + } if (level_ != nullptr) { return level_->id() + "_" + std::to_string(index_); } else { return "_" + std::to_string(index_); } } + int getIndex() const { return index_; } SemanticLevel::ptr level() const { return level_; } @@ -442,8 +477,30 @@ class SemanticRegion { return objects_; } + /** + * @brief Test whether this region contains the passed point + */ + virtual bool contains(const Mn::Vector3& point) const; + + void setBBox(const Mn::Vector3& min, const Mn::Vector3& max); + box3f aabb() const { return bbox_; } + const std::vector& getPolyLoopPoints() const { + return polyLoopPoints_; + } + + /** + * @brief Return a list of the semantic region's bounding volume edges. + */ + const std::vector>& getVisEdges() const { + return visEdges_; + } + + double getExtrusionHeight() const { return extrusionHeight_; } + + double getFloorHeight() const { return floorHeight_; } + SemanticCategory::ptr category() const { return category_; } protected: @@ -452,8 +509,20 @@ class SemanticRegion { std::shared_ptr category_; vec3f position_; box3f bbox_; - vec3f floorNormal_; - std::vector floorPoints_; + + std::string name_; + + // Height of extrusion for Extruded poly-loop-based volumes + double extrusionHeight_{}; + // Floor height + double floorHeight_{}; + + // poly loop points for base extrusion + std::vector polyLoopPoints_; + + // Edges for visualization of volume + std::vector> visEdges_; + std::vector> objects_; std::shared_ptr level_; friend SemanticScene; diff --git a/src/esp/sim/Simulator.cpp b/src/esp/sim/Simulator.cpp index 7283a11644..fd5bf7b224 100644 --- a/src/esp/sim/Simulator.cpp +++ b/src/esp/sim/Simulator.cpp @@ -43,6 +43,7 @@ namespace sim { using metadata::attributes::PhysicsManagerAttributes; using metadata::attributes::SceneAOInstanceAttributes; using metadata::attributes::SceneObjectInstanceAttributes; +using metadata::attributes::SemanticAttributes; using metadata::attributes::StageAttributes; Simulator::Simulator(const SimulatorConfiguration& cfg, @@ -299,20 +300,46 @@ bool Simulator::createSceneInstance(const std::string& activeSceneName) { // 3. Load the Semantic Scene Descriptor file appropriate for the current // scene instance. - // - Get semantic attributes - this handle may only be a tag - const auto currSemanticAttrHandle = + // - Get semantic attributes - this handle will either be empty or an existing + // handle for a SemanticAttributes in the SemanticAttributesManager. If not + // found then an error has occurred. + const std::string currSemanticAttrHandle = curSceneInstanceAttributes_->getSemanticSceneHandle(); - std::string semanticSceneDescFilename = ""; - if (currSemanticAttrHandle != "") { - const auto semanticAttr = - metadataMediator_->getSemanticAttributesManager() - ->getFirstMatchingObjectCopyByHandle(currSemanticAttrHandle); - // - Get Semantic Scene Descriptor Filename - semanticSceneDescFilename = semanticAttr->getSemanticDescriptorFilename(); + + const std::shared_ptr + semanticAttr = + (currSemanticAttrHandle != "") + ? metadataMediator_->getSemanticAttributesManager() + ->getFirstMatchingObjectCopyByHandle(currSemanticAttrHandle) + : nullptr; + + // This check verifies that either the `semantic_scene_instance' tag specified + // in the scene instance config was empty, or else it corresponds to an actual + // SemanticAttributes found in the SemanticAttributesManager. If this tag was + // not found even as a substring in the SemanticAttributesManager as being + // mapped to an existing semantic attributes,it would signify an error in the + // scene dataset config specifications. + ESP_CHECK( + (semanticAttr || (currSemanticAttrHandle == "")), + Cr::Utility::formatString( + "Simulator::createSceneInstance() : Attempt to reference " + "semantic attributes specified in current scene instance : `{}` " + "failed due to specified semantic tag `{}` not being found in " + "SemanticAttributesManager. Aborting", + activeSceneName, currSemanticAttrHandle)); + if (semanticAttr) { + ESP_VERY_VERBOSE(Mn::Debug::Flag::NoSpace) + << "Scene Instance `" << activeSceneName + << "` has Semantic attr handle : `" << currSemanticAttrHandle + << "` which tagged attributes : `" << semanticAttr->getHandle() << "`"; + } else { + ESP_VERY_VERBOSE(Mn::Debug::Flag::NoSpace) + << "Scene Instance `" << activeSceneName + << "` has Semantic attr handle : `" << currSemanticAttrHandle + << "` which did not reference any attributes"; } - // - Load semantic scene descriptor - resourceManager_->loadSemanticSceneDescriptor(semanticSceneDescFilename, - activeSceneName); + // - Load semantic scene + resourceManager_->loadSemanticScene(semanticAttr, activeSceneName); // 4. Specify frustumCulling based on value from config frustumCulling_ = config_.frustumCulling; @@ -388,7 +415,8 @@ bool Simulator::createSceneInstance(const std::string& activeSceneName) { resourceManager_->loadAllIBLAssets(); // 8. Load stage specified by Scene Instance Attributes - bool success = instanceStageForSceneAttributes(curSceneInstanceAttributes_); + bool success = instanceStageForSceneAttributes(curSceneInstanceAttributes_, + semanticAttr); // 9. Load object instances as specified by Scene Instance Attributes. if (success) { success = instanceObjectsForSceneAttributes(curSceneInstanceAttributes_); @@ -410,7 +438,9 @@ bool Simulator::createSceneInstance(const std::string& activeSceneName) { bool Simulator::instanceStageForSceneAttributes( const metadata::attributes::SceneInstanceAttributes::cptr& - curSceneInstanceAttributes_) { + curSceneInstanceAttributes_, + const std::shared_ptr& + semanticAttr) { // Load stage specified by Scene Instance Attributes // Get Stage Instance Attributes - contains name of stage and initial // transformation of stage in scene. @@ -464,14 +494,9 @@ bool Simulator::instanceStageForSceneAttributes( // from the dataset config stageAttributes->setUseSemanticTextures(config_.useSemanticTexturesIfFound); // set stage's ref to ssd file - // - Get semantic attributes if any referenced by scene instance - const auto currSemanticAttrHandle = - curSceneInstanceAttributes_->getSemanticSceneHandle(); - if (currSemanticAttrHandle != "") { - // Must exist already by here. - const auto semanticAttr = - metadataMediator_->getSemanticAttributesManager() - ->getFirstMatchingObjectCopyByHandle(currSemanticAttrHandle); + // - Get pertinent semantic values if any referenced in existing + // SemanticAttributes + if (semanticAttr != nullptr) { const std::string semanticAttrSSDName = semanticAttr->getSemanticDescriptorFilename(); const std::string semanticAttrAssetName = diff --git a/src/esp/sim/Simulator.h b/src/esp/sim/Simulator.h index 46e42fe6f1..14aca8fb7f 100644 --- a/src/esp/sim/Simulator.h +++ b/src/esp/sim/Simulator.h @@ -867,11 +867,15 @@ class Simulator { * esp::metadata::attributes::SceneInstanceAttributes * @param curSceneInstanceAttributes The attributes describing the current * scene instance. + * @param curSemanticAttr The SemanticAttributes referenced by the current + * scene instance, or nullptr if none. * @return whether stage creation is completed successfully */ bool instanceStageForSceneAttributes( const metadata::attributes::SceneInstanceAttributes::cptr& - curSceneInstanceAttributes); + curSceneInstanceAttributes, + const std::shared_ptr& + semanticAttr); /** * @brief Instance all the objects in the scene based on diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 66991e7175..d276dee3af 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -159,7 +159,14 @@ corrade_add_test( ) target_include_directories(MetadataMediatorTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) -corrade_add_test(Mp3dTest Mp3dTest.cpp LIBRARIES scene) +corrade_add_test( + Mp3dTest + Mp3dTest.cpp + LIBRARIES + core + metadata + scene +) target_include_directories(Mp3dTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) corrade_add_test( @@ -202,6 +209,9 @@ target_include_directories(ResourceManagerTest PRIVATE ${CMAKE_CURRENT_BINARY_DI corrade_add_test(SceneGraphTest SceneGraphTest.cpp LIBRARIES scene) +corrade_add_test(SemanticTest SemanticTest.cpp LIBRARIES metadata scene) +target_include_directories(SemanticTest PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + corrade_add_test(SensorTest SensorTest.cpp LIBRARIES sensor sim) corrade_add_test( diff --git a/src/tests/MetadataMediatorTest.cpp b/src/tests/MetadataMediatorTest.cpp index 55b085e077..07d743db3e 100644 --- a/src/tests/MetadataMediatorTest.cpp +++ b/src/tests/MetadataMediatorTest.cpp @@ -360,7 +360,8 @@ void MetadataMediatorTest::testDataset0() { CORRADE_VERIFY(navmeshHandle.find("navmesh_path1") != std::string::npos); // ssd const std::string ssdHandle = sceneAttrs->getSemanticSceneHandle(); - CORRADE_VERIFY(ssdHandle.find("semantic_descriptor_path1") != + // Ssd handle is made from scene instance simplified handle + CORRADE_VERIFY(ssdHandle.find(sceneAttrs->getSimplifiedHandle()) != std::string::npos); // @@ -431,9 +432,10 @@ void MetadataMediatorTest::testDataset0() { const auto semanticMgr = MM_->getSemanticAttributesManager(); - // should have 2 - CORRADE_COMPARE(semanticMgr->getNumObjects(), 2); - // should hold 2 keys + // should have 5 + CORRADE_COMPARE(semanticMgr->getNumObjects(), 5); + + // should hold these 2 keys + semantic scene-named file-based json CORRADE_VERIFY( semanticMgr->getObjectLibHasHandle("semantic_descriptor_path1")); CORRADE_VERIFY( @@ -442,8 +444,10 @@ void MetadataMediatorTest::testDataset0() { // each key should hold specific value const auto semanticAttr1 = semanticMgr->getObjectCopyByHandle("semantic_descriptor_path1"); + CORRADE_COMPARE(semanticAttr1->getSemanticDescriptorFilename(), "test_semantic_descriptor_path1"); + const auto semanticAttr2 = semanticMgr->getObjectCopyByHandle("semantic_descriptor_path2"); CORRADE_COMPARE(semanticAttr2->getSemanticDescriptorFilename(), diff --git a/src/tests/Mp3dTest.cpp b/src/tests/Mp3dTest.cpp index 8c75c63912..45e85459f2 100644 --- a/src/tests/Mp3dTest.cpp +++ b/src/tests/Mp3dTest.cpp @@ -6,6 +6,7 @@ #include #include +#include "esp/metadata/attributes/SemanticAttributes.h" #include "esp/scene/SemanticScene.h" #include "configure.h" @@ -32,14 +33,16 @@ void Mp3dTest::testLoad() { if (!Cr::Utility::Path::exists(filename)) { CORRADE_SKIP("MP3D dataset not found."); } - + auto mp3dAttr = + esp::metadata::attributes::SemanticAttributes::create("mp3dTestAttrs"); + mp3dAttr->setSemanticDescriptorFilename(filename); esp::scene::SemanticScene house; const esp::quatf alignGravity = esp::quatf::FromTwoVectors(-esp::vec3f::UnitZ(), esp::geo::ESP_GRAVITY); const esp::quatf alignFront = esp::quatf::FromTwoVectors(-esp::vec3f::UnitX(), esp::geo::ESP_FRONT); - esp::scene::SemanticScene::loadMp3dHouse(filename, house, - alignFront * alignGravity); + esp::scene::SemanticScene::loadSemanticSceneDescriptor( + mp3dAttr, house, alignFront * alignGravity); ESP_DEBUG() << "House{nobjects:" << house.count("objects") << ",nlevels:" << house.count("levels") << ",nregions:" << house.count("regions") diff --git a/src/tests/SemanticTest.cpp b/src/tests/SemanticTest.cpp new file mode 100644 index 0000000000..d468ad1875 --- /dev/null +++ b/src/tests/SemanticTest.cpp @@ -0,0 +1,132 @@ +// Copyright (c) Meta Platforms, Inc. and its affiliates. +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +#include +#include + +#include "esp/core/Esp.h" +#include "esp/metadata/MetadataMediator.h" +#include "esp/scene/SemanticScene.h" + +#include "configure.h" + +namespace Cr = Corrade; +namespace AttrMgrs = esp::metadata::managers; +namespace Attrs = esp::metadata::attributes; +using Attrs::SemanticAttributes; +using esp::metadata::MetadataMediator; + +namespace { + +struct SemanticTest : Cr::TestSuite::Tester { + explicit SemanticTest(); + + /** + * @brief This test will validate semantic region creation and containment + * checks. + */ + void TestRegionCreation(); + + esp::logging::LoggingContext loggingContext_; + + // + std::shared_ptr semanticAttr_ = nullptr; + +}; // struct SemanticTest + +SemanticTest::SemanticTest() { + // set up a default simulation config to initialize MM + auto cfg = esp::sim::SimulatorConfiguration{}; + esp::metadata::MetadataMediator::uptr MM = + MetadataMediator::create_unique(cfg); + + const std::string semanticConfigFile = Cr::Utility::Path::join( + TEST_ASSETS, "semantic/test_regions.semantic_config.json"); + + // Build an attributes based on the json file + semanticAttr_ = MM->getSemanticAttributesManager()->createObject( + semanticConfigFile, true); + + addTests({&SemanticTest::TestRegionCreation}); +} + +void SemanticTest::TestRegionCreation() { + std::shared_ptr semanticScene = + esp::scene::SemanticScene::create(); + + esp::scene::SemanticScene::loadSemanticSceneDescriptor(semanticAttr_, + *semanticScene); + // Verify semantic scene values + const auto& regions = semanticScene->regions(); + // Verify there are 2 regions + CORRADE_COMPARE(regions.size(), 2); + // The regions are symmetric, so both lists will work for both, if the signs + // are flipped. + // Build a list of points within the region + std::vector hitTestPoints(6); + hitTestPoints[0] = Mn::Vector3{-5.1, 0.0, 0.01}; + hitTestPoints[1] = Mn::Vector3{-5.1, 1.0, 0.01}; + hitTestPoints[2] = Mn::Vector3{-5.1, -1.0, 0.01}; + hitTestPoints[3] = Mn::Vector3{-24.9, 0.0, 0.01}; + hitTestPoints[4] = Mn::Vector3{-15.1, 0.0, 9.9}; + hitTestPoints[5] = Mn::Vector3{-15.1, 0.0, -9.9}; + // Build a list of points outside the region + std::vector missTestPoints(6); + missTestPoints[0] = Mn::Vector3{0.0, 0.0, 0.01}; + missTestPoints[1] = Mn::Vector3{-6.0, 0.0, 10.01}; + missTestPoints[2] = Mn::Vector3{-5.1, -2.1, 0.01}; + missTestPoints[3] = Mn::Vector3{-5.1, 2.1, 0.01}; + missTestPoints[4] = Mn::Vector3{-15.1, -1.5, 10.01}; + missTestPoints[5] = Mn::Vector3{-15.1, 1.5, -10.01}; + + // Check region construction + { // negative X region + const auto region = regions[0]; + CORRADE_COMPARE(region->id(), "test_region_negativeX"); + CORRADE_COMPARE(region->getPolyLoopPoints().size(), 6); + CORRADE_COMPARE(region->getExtrusionHeight(), 4.0); + CORRADE_COMPARE(region->getFloorHeight(), -2.0); + + const auto regionCat = region->category(); + CORRADE_COMPARE(regionCat->name(), "bedroom"); + CORRADE_COMPARE(regionCat->index(), 0); + + int idx = 0; + for (const auto& pt : hitTestPoints) { + CORRADE_ITERATION(idx++); + CORRADE_VERIFY(region->contains(pt)); + } + idx = 0; + for (const auto& pt : missTestPoints) { + CORRADE_ITERATION(idx++); + CORRADE_VERIFY(!region->contains(pt)); + } + } + + { // positive X region + const auto region = regions[1]; + CORRADE_COMPARE(region->id(), "test_region_positiveX"); + CORRADE_COMPARE(region->getPolyLoopPoints().size(), 6); + CORRADE_COMPARE(region->getExtrusionHeight(), 4.0); + CORRADE_COMPARE(region->getFloorHeight(), -2.0); + const auto regionCat = region->category(); + CORRADE_COMPARE(regionCat->name(), "bathroom"); + CORRADE_COMPARE(regionCat->index(), 1); + int idx = 0; + for (const auto& pt : hitTestPoints) { + CORRADE_ITERATION(idx++); + CORRADE_VERIFY(region->contains(-pt)); + } + idx = 0; + for (const auto& pt : missTestPoints) { + CORRADE_ITERATION(idx++); + CORRADE_VERIFY(!region->contains(-pt)); + } + } + +} // namespace + +} // namespace + +CORRADE_TEST_MAIN(SemanticTest) diff --git a/tests/test_attributes_managers.py b/tests/test_attributes_managers.py index fd44fdb9fe..48dd328568 100644 --- a/tests/test_attributes_managers.py +++ b/tests/test_attributes_managers.py @@ -4,11 +4,22 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +from os import path as osp + import magnum as mn import habitat_sim import habitat_sim.utils.settings +# Scene to load for tests. +# TODO: evolve these tests to be scene dataset config-driven. +TEST_SCENE = osp.abspath( + osp.join( + osp.dirname(__file__), + "../data/scene_datasets/habitat-test-scenes/van-gogh-room.glb", + ) +) + def perform_general_tests(attr_mgr, search_string): # get size of template library @@ -256,17 +267,21 @@ def perform_add_blank_template_test(attr_mgr, valid_render_handle=None): assert curr_num_templates == orig_num_templates -def test_physics_attributes_managers(): +def build_cfg(_scene): cfg_settings = habitat_sim.utils.settings.default_sim_settings.copy() - cfg_settings["scene"] = "data/scene_datasets/habitat-test-scenes/van-gogh-room.glb" - hab_cfg = habitat_sim.utils.settings.make_cfg(cfg_settings) + cfg_settings["scene"] = _scene + return habitat_sim.utils.settings.make_cfg(cfg_settings) + + +def test_physics_attributes_managers(): + hab_cfg = build_cfg(TEST_SCENE) with habitat_sim.Simulator(hab_cfg) as sim: # get attribute managers phys_attr_mgr = sim.get_physics_template_manager() # perform general tests for this attributes manager template0, _ = perform_general_tests( - phys_attr_mgr, cfg_settings["physics_config_file"] + phys_attr_mgr, hab_cfg.sim_cfg.physics_config_file ) # verify that physics template matches expected values in file @@ -278,11 +293,9 @@ def test_physics_attributes_managers(): def test_stage_attributes_managers(): - cfg_settings = habitat_sim.utils.settings.default_sim_settings.copy() - cfg_settings["scene"] = "data/scene_datasets/habitat-test-scenes/van-gogh-room.glb" - hab_cfg = habitat_sim.utils.settings.make_cfg(cfg_settings) + hab_cfg = build_cfg(TEST_SCENE) with habitat_sim.Simulator(hab_cfg) as sim: - stage_name = cfg_settings["scene"] + stage_name = hab_cfg.sim_cfg.scene_id # get attribute managers stage_mgr = sim.get_stage_template_manager() @@ -298,9 +311,7 @@ def test_stage_attributes_managers(): def test_object_attributes_managers(): - cfg_settings = habitat_sim.utils.settings.default_sim_settings.copy() - cfg_settings["scene"] = "data/scene_datasets/habitat-test-scenes/van-gogh-room.glb" - hab_cfg = habitat_sim.utils.settings.make_cfg(cfg_settings) + hab_cfg = build_cfg(TEST_SCENE) with habitat_sim.Simulator(hab_cfg) as sim: # get object attribute managers obj_mgr = sim.get_object_template_manager() @@ -371,9 +382,7 @@ def perform_asset_attrib_mgr_tests(attr_mgr, default_attribs, legalVal, illegalV def test_asset_attributes_managers(): - cfg_settings = habitat_sim.utils.settings.default_sim_settings.copy() - cfg_settings["scene"] = "data/scene_datasets/habitat-test-scenes/van-gogh-room.glb" - hab_cfg = habitat_sim.utils.settings.make_cfg(cfg_settings) + hab_cfg = build_cfg(TEST_SCENE) with habitat_sim.Simulator(hab_cfg) as sim: # legal and illegal vals for primitives based on wireframe or solid legal_mod_val_wf = 64 diff --git a/tests/test_semantic_scene.py b/tests/test_semantic_scene.py index 8f2e80ab53..f5f573e4e1 100644 --- a/tests/test_semantic_scene.py +++ b/tests/test_semantic_scene.py @@ -6,12 +6,12 @@ from os import path as osp +import magnum as mn import pytest -import quaternion # noqa: F401 import habitat_sim import habitat_sim.errors -from habitat_sim.utils.settings import make_cfg +from habitat_sim.utils.settings import default_sim_settings, make_cfg _test_scenes = [ osp.abspath( @@ -28,6 +28,95 @@ ), ] +TEST_SCENE_DATASET = osp.abspath( + osp.join( + osp.dirname(__file__), + "../data/test_assets/dataset_tests/dataset_0/test_dataset_0.scene_dataset_config.json", + ) +) + + +def test_semantic_regions(): + ## load scene dataset + cfg_settings = default_sim_settings.copy() + cfg_settings["scene_dataset_config_file"] = TEST_SCENE_DATASET + cfg_settings["scene"] = "dataset_test_scene" + hab_cfg = make_cfg(cfg_settings) + with habitat_sim.Simulator(hab_cfg) as sim: + semantic_scene = sim.semantic_scene + assert semantic_scene != None + regions = semantic_scene.regions + assert regions != None + assert len(regions) == 2 + + # Build a list of points within the region + hit_test_points = 6 * [None] + hit_test_points[0] = mn.Vector3(-5.1, 0.0, 0.01) + hit_test_points[1] = mn.Vector3(-5.1, 1.0, 0.01) + hit_test_points[2] = mn.Vector3(-5.1, -1.0, 0.01) + hit_test_points[3] = mn.Vector3(-24.9, 0.0, 0.01) + hit_test_points[4] = mn.Vector3(-15.1, 0.0, 9.9) + hit_test_points[5] = mn.Vector3(-15.1, 0.0, -9.9) + # Build a list of points outside the region + miss_test_points = 6 * [None] + miss_test_points[0] = mn.Vector3(0.0, 0.0, 0.01) + miss_test_points[1] = mn.Vector3(-6.0, 0.0, 10.01) + miss_test_points[2] = mn.Vector3(-5.1, -2.1, 0.01) + miss_test_points[3] = mn.Vector3(-5.1, 2.1, 0.01) + miss_test_points[4] = mn.Vector3(-15.1, -1.5, 10.01) + miss_test_points[5] = mn.Vector3(-15.1, 1.5, -10.01) + + # Check region construction + # negative X region + region = regions[0] + assert region.id == "test_region_negativeX" + assert len(region.poly_loop_points) == 6 + assert region.extrusion_height == 4.0 + assert region.floor_height == -2.0 + + regionCat = region.category + assert regionCat.name() == "bedroom" + assert regionCat.index() == 0 + # verify containment + assert region.contains(hit_test_points[0]) + assert region.contains(hit_test_points[1]) + assert region.contains(hit_test_points[2]) + assert region.contains(hit_test_points[3]) + assert region.contains(hit_test_points[4]) + assert region.contains(hit_test_points[5]) + # verify non-containment + assert not region.contains(miss_test_points[0]) + assert not region.contains(miss_test_points[1]) + assert not region.contains(miss_test_points[2]) + assert not region.contains(miss_test_points[3]) + assert not region.contains(miss_test_points[4]) + assert not region.contains(miss_test_points[5]) + + # positive X region + region = regions[1] + assert region.id == "test_region_positiveX" + assert len(region.poly_loop_points) == 6 + assert region.extrusion_height == 4.0 + assert region.floor_height == -2.0 + + regionCat = region.category + assert regionCat.name() == "bathroom" + assert regionCat.index() == 1 + # verify containment + assert region.contains(-1 * hit_test_points[0]) + assert region.contains(-1 * hit_test_points[1]) + assert region.contains(-1 * hit_test_points[2]) + assert region.contains(-1 * hit_test_points[3]) + assert region.contains(-1 * hit_test_points[4]) + assert region.contains(-1 * hit_test_points[5]) + # verify non-containment + assert not region.contains(-1 * miss_test_points[0]) + assert not region.contains(-1 * miss_test_points[1]) + assert not region.contains(-1 * miss_test_points[2]) + assert not region.contains(-1 * miss_test_points[3]) + assert not region.contains(-1 * miss_test_points[4]) + assert not region.contains(-1 * miss_test_points[5]) + @pytest.mark.parametrize("scene", _test_scenes) def test_semantic_scene(scene, make_cfg_settings):