diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88e67406fa..b0ef44ce27 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -136,6 +136,7 @@ repos: hooks: - id: eslint args: [--fix, --ext .html,.js] + language_version: 14.21.3 additional_dependencies: - eslint@6.4.0 - eslint-config-prettier@6.3.0 diff --git a/data/test_assets/objects/skinned_prism.glb b/data/test_assets/objects/skinned_prism.glb new file mode 100644 index 0000000000..fb86290ef4 Binary files /dev/null and b/data/test_assets/objects/skinned_prism.glb differ diff --git a/data/test_assets/screenshots/SimTestInvertScaleImageExpected.png b/data/test_assets/screenshots/SimTestInvertScaleImageExpected.png new file mode 100644 index 0000000000..efb897f5e9 Binary files /dev/null and b/data/test_assets/screenshots/SimTestInvertScaleImageExpected.png differ diff --git a/data/test_assets/screenshots/SimTestSkinnedAOInitialPose.png b/data/test_assets/screenshots/SimTestSkinnedAOInitialPose.png new file mode 100644 index 0000000000..6da741d5a2 Binary files /dev/null and b/data/test_assets/screenshots/SimTestSkinnedAOInitialPose.png differ diff --git a/data/test_assets/screenshots/SimTestSkinnedAOPose.png b/data/test_assets/screenshots/SimTestSkinnedAOPose.png new file mode 100644 index 0000000000..c3a7f156b1 Binary files /dev/null and b/data/test_assets/screenshots/SimTestSkinnedAOPose.png differ diff --git a/data/test_assets/urdf/skinned_prism.ao_config.json b/data/test_assets/urdf/skinned_prism.ao_config.json new file mode 100644 index 0000000000..c14c0199f9 --- /dev/null +++ b/data/test_assets/urdf/skinned_prism.ao_config.json @@ -0,0 +1,5 @@ +{ + "render_asset": "../objects/skinned_prism.glb", + "semantic_id": 100, + "debug_render_primitives": false +} diff --git a/data/test_assets/urdf/skinned_prism.urdf b/data/test_assets/urdf/skinned_prism.urdf new file mode 100644 index 0000000000..0c1912ab71 --- /dev/null +++ b/data/test_assets/urdf/skinned_prism.urdf @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/tutorials/lighting_tutorial.py b/examples/tutorials/lighting_tutorial.py index 34e9390c94..9a2abbace8 100644 --- a/examples/tutorials/lighting_tutorial.py +++ b/examples/tutorials/lighting_tutorial.py @@ -149,7 +149,7 @@ def main(show_imgs=True, save_imgs=False): # create a custom light setup my_default_lighting = [ - LightInfo(vector=[2.0, 2.0, 1.0, 0.0], model=LightPositionModel.Camera) + LightInfo(vector=[-2.0, -2.0, -1.0, 0.0], model=LightPositionModel.Camera) ] # overwrite the default DEFAULT_LIGHTING_KEY light setup sim.set_light_setup(my_default_lighting) @@ -208,7 +208,7 @@ def main(show_imgs=True, save_imgs=False): # create a new setup with an additional light new_light_setup = existing_light_setup + [ LightInfo( - vector=[0.0, 0.0, 1.0, 0.0], + vector=[0.0, 0.0, -1.0, 0.0], color=[1.6, 1.6, 1.4], model=LightPositionModel.Camera, ) diff --git a/examples/viewer.py b/examples/viewer.py index c479482554..d72e3b5157 100644 --- a/examples/viewer.py +++ b/examples/viewer.py @@ -53,24 +53,25 @@ def __init__(self, sim_settings: Dict[str, Any]) -> None: ) # Compute environment camera resolution based on the number of environments to render in the window. - surface_size: tuple(int, int) = ( + window_size: mn.Vector2 = ( self.sim_settings["window_width"], self.sim_settings["window_height"], ) - grid_size: tuple(int, int) = ReplayRenderer.environment_grid_size(self.num_env) - camera_resolution: tuple(int, int) = ( - surface_size[0] / grid_size[0], - surface_size[1] / grid_size[1], - ) - self.sim_settings["width"] = camera_resolution[0] - self.sim_settings["height"] = camera_resolution[1] configuration = self.Configuration() configuration.title = "Habitat Sim Interactive Viewer" - configuration.size = surface_size + configuration.size = window_size Application.__init__(self, configuration) self.fps: float = 60.0 + # Compute environment camera resolution based on the number of environments to render in the window. + grid_size: mn.Vector2i = ReplayRenderer.environment_grid_size(self.num_env) + camera_resolution: mn.Vector2 = mn.Vector2(self.framebuffer_size) / mn.Vector2( + grid_size + ) + self.sim_settings["width"] = camera_resolution[0] + self.sim_settings["height"] = camera_resolution[1] + # draw Bullet debug line visualizations (e.g. collision meshes) self.debug_bullet_draw = False # draw active contact point debug line visualizations @@ -138,11 +139,12 @@ def __init__(self, sim_settings: Dict[str, Any]) -> None: # text object transform in window space is Projection matrix times Translation Matrix # put text in top left of window self.window_text_transform = mn.Matrix3.projection( - mn.Vector2(surface_size) + self.framebuffer_size ) @ mn.Matrix3.translation( - mn.Vector2( - surface_size[0] * -HabitatSimInteractiveViewer.TEXT_DELTA_FROM_CENTER, - surface_size[1] * HabitatSimInteractiveViewer.TEXT_DELTA_FROM_CENTER, + mn.Vector2(self.framebuffer_size) + * mn.Vector2( + -HabitatSimInteractiveViewer.TEXT_DELTA_FROM_CENTER, + HabitatSimInteractiveViewer.TEXT_DELTA_FROM_CENTER, ) ) self.shader = shaders.VectorGL2D() @@ -558,13 +560,31 @@ def key_press_event(self, event: Application.KeyEvent) -> None: self.cached_urdf = urdf_file_path aom = self.sim.get_articulated_object_manager() ao = aom.add_articulated_object_from_urdf( - urdf_file_path, fixed_base, 1.0, 1.0, True + urdf_file_path, + fixed_base, + 1.0, + 1.0, + True, + maintain_link_order=False, + intertia_from_urdf=False, ) ao.translation = ( self.default_agent.scene_node.transformation.transform_point( [0.0, 1.0, -1.5] ) ) + # check removal and auto-creation + joint_motor_settings = habitat_sim.physics.JointMotorSettings( + position_target=0.0, + position_gain=1.0, + velocity_target=0.0, + velocity_gain=1.0, + max_impulse=1000.0, + ) + existing_motor_ids = ao.existing_joint_motor_ids + for motor_id in existing_motor_ids: + ao.remove_joint_motor(motor_id) + ao.create_all_motors(joint_motor_settings) else: logger.warn("Load URDF: input file not found. Aborting.") diff --git a/src/cmake/FindMagnum.cmake b/src/cmake/FindMagnum.cmake index 98ce451a41..3e17e0a684 100644 --- a/src/cmake/FindMagnum.cmake +++ b/src/cmake/FindMagnum.cmake @@ -128,7 +128,7 @@ # # Features of found Magnum library are exposed in these variables: # -# MAGNUM_BUILD_DEPRECATED - Defined if compiled with deprecated APIs +# MAGNUM_BUILD_DEPRECATED - Defined if compiled with deprecated features # included # MAGNUM_BUILD_STATIC - Defined if compiled as static libraries # MAGNUM_BUILD_STATIC_UNIQUE_GLOBALS - Defined if static libraries keep the @@ -910,7 +910,7 @@ foreach(_component ${Magnum_FIND_COMPONENTS}) # SceneTools library elseif(_component STREQUAL SceneTools) - set(_MAGNUM_${_COMPONENT}_INCLUDE_PATH_NAMES FlattenMeshHierarchy.h) + set(_MAGNUM_${_COMPONENT}_INCLUDE_PATH_NAMES FlattenTransformationHierarchy.h) # ShaderTools library elseif(_component STREQUAL ShaderTools) diff --git a/src/cmake/FindMagnumPlugins.cmake b/src/cmake/FindMagnumPlugins.cmake index 19280bb466..aa97036914 100644 --- a/src/cmake/FindMagnumPlugins.cmake +++ b/src/cmake/FindMagnumPlugins.cmake @@ -39,6 +39,7 @@ # PngImporter - PNG importer # PrimitiveImporter - Primitive importer # SpirvToolsShaderConverter - SPIR-V Tools shader converter +# SpngImporter - PNG importer using libspng # StanfordImporter - Stanford PLY importer # StanfordSceneConverter - Stanford PLY converter # StbDxtImageConverter - BC1/BC3 image compressor using stb_dxt @@ -168,10 +169,10 @@ set(_MAGNUMPLUGINS_PLUGIN_COMPONENTS JpegImporter KtxImageConverter KtxImporter MeshOptimizerSceneConverter MiniExrImageConverter OpenExrImageConverter OpenExrImporter OpenGexImporter PngImageConverter PngImporter PrimitiveImporter - SpirvToolsShaderConverter StanfordImporter StanfordSceneConverter - StbDxtImageConverter StbImageConverter StbImageImporter - StbResizeImageConverter StbTrueTypeFont StbVorbisAudioImporter StlImporter - UfbxImporter WebPImporter) + SpirvToolsShaderConverter SpngImporter StanfordImporter + StanfordSceneConverter StbDxtImageConverter StbImageConverter + StbImageImporter StbResizeImageConverter StbTrueTypeFont + StbVorbisAudioImporter StlImporter UfbxImporter WebPImporter) # Nothing is enabled by default right now set(_MAGNUMPLUGINS_IMPLICITLY_ENABLED_COMPONENTS ) @@ -471,6 +472,12 @@ foreach(_component ${MagnumPlugins_FIND_COMPONENTS}) set_property(TARGET MagnumPlugins::${_component} APPEND PROPERTY INTERFACE_LINK_LIBRARIES SpirvTools::SpirvTools SpirvTools::Opt) + # SpngImporter plugin dependencies + elseif(_component STREQUAL SpngImporter) + find_package(Spng REQUIRED) + set_property(TARGET MagnumPlugins::${_component} APPEND PROPERTY + INTERFACE_LINK_LIBRARIES Spng::Spng) + # StanfordImporter has no dependencies # StanfordSceneConverter has no dependencies # StbDxtImageConverter has no dependencies diff --git a/src/cmake/dependencies.cmake b/src/cmake/dependencies.cmake index 825a73137a..425544089a 100644 --- a/src/cmake/dependencies.cmake +++ b/src/cmake/dependencies.cmake @@ -199,7 +199,9 @@ if(NOT USE_SYSTEM_MAGNUM) # These are enabled by default but we don't need them for anything yet set(MAGNUM_WITH_SHADERTOOLS OFF CACHE BOOL "" FORCE) - set(MAGNUM_WITH_MATERIALTOOLS OFF CACHE BOOL "" FORCE) + # These used to be disabled here but now aren't, explicitly enable them to + # update options in existing builds + set(MAGNUM_WITH_MATERIALTOOLS ON CACHE BOOL "" FORCE) # These are enabled by default but we don't need them if not building GUI # viewers -- disabling for slightly faster builds. If you need any of these @@ -347,16 +349,25 @@ if(NOT USE_SYSTEM_MAGNUM) # Make Magnum text rendering plugins (used by the native viewer) available # for Python as well; and reset that back to strange build procedures that # turn some features off again later can still work. + set( + common_plugins + Magnum::AnyImageConverter + Magnum::AnyImageImporter + Magnum::AnySceneImporter + MagnumPlugins::AssimpImporter + MagnumPlugins::BasisImporter + MagnumPlugins::GltfImporter + MagnumPlugins::StanfordImporter + MagnumPlugins::StbImageConverter + MagnumPlugins::StbImageImporter + ) if(BUILD_GUI_VIEWERS) - set(MAGNUM_PYTHON_BINDINGS_STATIC_PLUGINS - MagnumPlugins::StbTrueTypeFont Magnum::AnySceneImporter - MagnumPlugins::AssimpImporter CACHE STRING "" FORCE - ) - else() - set(MAGNUM_PYTHON_BINDINGS_STATIC_PLUGINS Magnum::AnySceneImporter - MagnumPlugins::AssimpImporter + set(MAGNUM_PYTHON_BINDINGS_STATIC_PLUGINS ${common_plugins} + MagnumPlugins::StbTrueTypeFont CACHE STRING "" FORCE ) + else() + set(MAGNUM_PYTHON_BINDINGS_STATIC_PLUGINS ${common_plugins} CACHE STRING "" FORCE) endif() add_subdirectory("${DEPS_DIR}/magnum-bindings") endif() diff --git a/src/deps/corrade b/src/deps/corrade index 68d0216383..1408175dbf 160000 --- a/src/deps/corrade +++ b/src/deps/corrade @@ -1 +1 @@ -Subproject commit 68d0216383edaf2c8f03da9c404e2158c4790c27 +Subproject commit 1408175dbff4c72c547aa44969f2f53769c01c40 diff --git a/src/deps/magnum b/src/deps/magnum index b38d3eea89..bc1859f653 160000 --- a/src/deps/magnum +++ b/src/deps/magnum @@ -1 +1 @@ -Subproject commit b38d3eea89d20e3c6ff67864df041f7c76a6c7be +Subproject commit bc1859f653e4dde58df1a1291cd75ce0325bcf30 diff --git a/src/deps/magnum-bindings b/src/deps/magnum-bindings index e2642033b3..ee284aa4aa 160000 --- a/src/deps/magnum-bindings +++ b/src/deps/magnum-bindings @@ -1 +1 @@ -Subproject commit e2642033b3e77559e7d521c2e5714876555661fe +Subproject commit ee284aa4aa201e6eb0b6c4b31d8990879a362501 diff --git a/src/deps/magnum-integration b/src/deps/magnum-integration index d1eebad0cd..1a66b05bd7 160000 --- a/src/deps/magnum-integration +++ b/src/deps/magnum-integration @@ -1 +1 @@ -Subproject commit d1eebad0cd71cb80b0d3631a45f112af533f8a6d +Subproject commit 1a66b05bd7db0a5484366054ddc678bebf79921e diff --git a/src/deps/magnum-plugins b/src/deps/magnum-plugins index b7d3438887..8e5cf7ac98 160000 --- a/src/deps/magnum-plugins +++ b/src/deps/magnum-plugins @@ -1 +1 @@ -Subproject commit b7d34388879cfcc5e7f1108032a4d3a13b723723 +Subproject commit 8e5cf7ac9820be5cf1f8728db83a284eb9120090 diff --git a/src/esp/assets/CMakeLists.txt b/src/esp/assets/CMakeLists.txt index a3ba82d3c3..b477ed6a24 100644 --- a/src/esp/assets/CMakeLists.txt +++ b/src/esp/assets/CMakeLists.txt @@ -31,6 +31,7 @@ find_package( AnyImageImporter AnySceneImporter GL + MaterialTools MeshTools SceneGraph SceneTools @@ -82,7 +83,7 @@ target_link_libraries( MagnumPlugins::StanfordImporter MagnumPlugins::StbImageImporter MagnumPlugins::StbImageConverter - PRIVATE geo io + PRIVATE geo io Magnum::MaterialTools ) if(BUILD_WITH_VHACD) diff --git a/src/esp/assets/MeshMetaData.h b/src/esp/assets/MeshMetaData.h index f94fca9acd..c174bf30b0 100644 --- a/src/esp/assets/MeshMetaData.h +++ b/src/esp/assets/MeshMetaData.h @@ -45,6 +45,9 @@ struct MeshTransformNode { /** @brief Default constructor. */ MeshTransformNode() = default; + + /** @brief Node name in the original file. */ + std::string name{}; }; /** @@ -74,6 +77,10 @@ struct MeshMetaData { std::pair textureIndex = std::make_pair(ID_UNDEFINED, ID_UNDEFINED); + /** @brief Index range (inclusive) of skin data for the asset in the global + * asset datastructure. */ + std::pair skinIndex = std::make_pair(ID_UNDEFINED, ID_UNDEFINED); + /** @brief The root of the mesh component transformation hierarchy tree which * stores the relationship between components of the asset.*/ MeshTransformNode root; @@ -116,6 +123,19 @@ struct MeshMetaData { textureIndex.second = textureEnd; } + /** + * @brief Sets the skin indices for the asset. See @ref + * ResourceManager::skins_. + * @param skinStart First index for asset skin data in the global + * skin datastructure. + * @param skinEnd Final index for asset skin data in the global skin + * datastructure. + */ + void setSkinIndices(int skinStart, int skinEnd) { + skinIndex.first = skinStart; + skinIndex.second = skinEnd; + } + /** * @brief Set the root frame orientation based on passed frame * @param frame target frame in world space diff --git a/src/esp/assets/RenderAssetInstanceCreationInfo.h b/src/esp/assets/RenderAssetInstanceCreationInfo.h index 49a06bf678..101dc6c0be 100644 --- a/src/esp/assets/RenderAssetInstanceCreationInfo.h +++ b/src/esp/assets/RenderAssetInstanceCreationInfo.h @@ -9,9 +9,14 @@ #include #include #include + +#include #include namespace esp { +namespace physics { +class ArticulatedObject; +} namespace assets { // parameters to control how a render asset instance is created @@ -44,6 +49,7 @@ struct RenderAssetInstanceCreationInfo { Corrade::Containers::Optional scale; Flags flags; std::string lightSetupKey; + std::shared_ptr rig; }; } // namespace assets diff --git a/src/esp/assets/ResourceManager.cpp b/src/esp/assets/ResourceManager.cpp index 090e4585fb..35de131edd 100644 --- a/src/esp/assets/ResourceManager.cpp +++ b/src/esp/assets/ResourceManager.cpp @@ -5,6 +5,7 @@ #include "ResourceManager.h" #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -37,13 +39,14 @@ #include #include #include -#include +#include #include #include #include #include #include #include +#include #include #include @@ -57,8 +60,8 @@ #include "esp/assets/RenderAssetInstanceCreationInfo.h" #include "esp/geo/Geo.h" #include "esp/gfx/GenericDrawable.h" -#include "esp/gfx/MaterialUtil.h" #include "esp/gfx/PbrDrawable.h" +#include "esp/gfx/SkinData.h" #include "esp/gfx/replay/Recorder.h" #include "esp/io/Json.h" #include "esp/io/URDFParser.h" @@ -722,29 +725,6 @@ esp::geo::CoordinateFrame ResourceManager::buildFrameFromAttributes( } } // ResourceManager::buildFrameFromAttributes -std::string ResourceManager::createColorMaterial( - const esp::assets::PhongMaterialColor& materialColor) { - std::ostringstream matHandleStream; - matHandleStream << "phong_amb_" << materialColor.ambientColor.toSrgbAlphaInt() - << "_dif_" << materialColor.diffuseColor.toSrgbAlphaInt() - << "_spec_" << materialColor.specularColor.toSrgbAlphaInt(); - std::string newMaterialID = matHandleStream.str(); - auto materialResource = shaderManager_.get(newMaterialID); - if (materialResource.state() == Mn::ResourceState::NotLoadedFallback) { - gfx::PhongMaterialData::uptr phongMaterial = - gfx::PhongMaterialData::create_unique(); - phongMaterial->ambientColor = materialColor.ambientColor; - // NOTE: This multiplication is a hack to roughly balance the Phong and PBR - // light intensity reactions. - phongMaterial->diffuseColor = materialColor.diffuseColor * 0.175; - phongMaterial->specularColor = materialColor.specularColor * 0.175; - - shaderManager_.set(newMaterialID, static_cast( - phongMaterial.release())); - } - return newMaterialID; -} // ResourceManager::createColorMaterial - scene::SceneNode* ResourceManager::loadAndCreateRenderAssetInstance( const AssetInfo& assetInfo, const RenderAssetInstanceCreationInfo& creation, @@ -1282,14 +1262,19 @@ void ResourceManager::buildPrimitiveAssetData( MeshMetaData meshMetaData{meshStart, meshEnd}; meshes_.emplace(meshStart, std::move(primMeshData)); + meshMetaData.root.materialID = std::to_string(nextMaterialID_++); // default material for now - std::unique_ptr phongMaterial = - gfx::PhongMaterialData::create_unique(); + // Populate with defaults from sim's gfx::PhongMaterialData + Mn::Trade::MaterialData materialData = buildDefaultPhongMaterial(); + + // Set expected user-defined attributes + materialData = setMaterialDefaultUserAttributes( + materialData, ObjectInstanceShaderType::Phong); + + shaderManager_.set(meshMetaData.root.materialID, + std::move(materialData)); - meshMetaData.root.materialID = std::to_string(nextMaterialID_++); - shaderManager_.set(meshMetaData.root.materialID, - static_cast(phongMaterial.release())); meshMetaData.root.meshIDLocal = 0; meshMetaData.root.componentID = 0; @@ -1464,16 +1449,25 @@ ResourceManager::flattenImportedMeshAndBuildSemantic(Importer& fileImporter, Cr::Containers::Optional scene = fileImporter.scene(sceneID); + // To access the mesh id + Cr::Containers::Array>> + meshesMaterials = scene->meshesMaterialsAsArray(); + // All the transformations, flattened and indexed by mesh id, with + // reframeTransform applied to each + Cr::Containers::Array transformations = + Mn::SceneTools::flattenTransformationHierarchy3D( + *scene, Mn::Trade::SceneField::Mesh, reframeTransform); + Cr::Containers::Array flattenedMeshes; - for (const Cr::Containers::Triple& - meshTransformation : - Mn::SceneTools::flattenMeshHierarchy3D(*scene)) { - int iMesh = meshTransformation.first(); + Cr::Containers::arrayReserve(flattenedMeshes, meshesMaterials.size()); + + for (std::size_t i = 0; i != meshesMaterials.size(); ++i) { + Mn::UnsignedInt iMesh = meshesMaterials[i].second().first(); if (Cr::Containers::Optional mesh = fileImporter.mesh(iMesh)) { - const auto transform = reframeTransform * meshTransformation.third(); arrayAppend(flattenedMeshes, - Mn::MeshTools::transform3D(*mesh, transform)); + Mn::MeshTools::transform3D(*mesh, transformations[i])); } } @@ -1756,6 +1750,8 @@ bool ResourceManager::loadRenderAssetGeneral(const AssetInfo& info) { loadMaterials(*fileImporter_, loadedAssetData); } loadMeshes(*fileImporter_, loadedAssetData); + loadSkins(*fileImporter_, loadedAssetData); + auto inserted = resourceDict_.emplace(filename, std::move(loadedAssetData)); MeshMetaData& meshMetaData = inserted.first->second.meshMetaData; @@ -1794,6 +1790,7 @@ bool ResourceManager::loadRenderAssetGeneral(const AssetInfo& info) { scene->parentsAsArray()) { nodes[parent.first()].emplace(); nodes[parent.first()]->componentID = parent.first(); + nodes[parent.first()]->name = fileImporter_->objectName(parent.first()); } // Set transformations. Objects that are not part of the hierarchy are @@ -1879,14 +1876,31 @@ scene::SceneNode* ResourceManager::createRenderAssetInstanceGeneralPrimitive( : scene::SceneNodeType::OBJECT; bool computeAbsoluteAABBs = creation.isStatic(); - addComponent(loadedAssetData.meshMetaData, // mesh metadata - newNode, // parent scene node - creation.lightSetupKey, // lightSetup key - drawables, // drawable group - loadedAssetData.meshMetaData.root, // mesh transform node + // If the object has a skin and a rig (articulated object), link them together + // such as the model bones are driven by the articulated object links. + std::shared_ptr instanceSkinData = nullptr; + const auto& meshMetaData = loadedAssetData.meshMetaData; + if (creation.rig && meshMetaData.skinIndex.first != ID_UNDEFINED) { + ESP_CHECK( + !skins_.empty(), + "Cannot instantiate skinned model because no skin data is imported."); + const auto& skinData = skins_[meshMetaData.skinIndex.first]; + instanceSkinData = std::make_shared(skinData); + mapSkinnedModelToArticulatedObject(meshMetaData.root, creation.rig, + instanceSkinData); + ESP_CHECK(instanceSkinData->rootArticulatedObjectNode, + "Could not map skinned model to articulated object."); + } + + addComponent(meshMetaData, // mesh metadata + newNode, // parent scene node + creation.lightSetupKey, // lightSetup key + drawables, // drawable group + meshMetaData.root, // mesh transform node visNodeCache, // a vector of scene nodes, the visNodeCache computeAbsoluteAABBs, // compute absolute AABBs - staticDrawableInfo); // a vector of static drawable info + staticDrawableInfo, // a vector of static drawable info + instanceSkinData); // instance skinning data if (computeAbsoluteAABBs) { // now compute aabbs by constructed staticDrawableInfo @@ -1963,15 +1977,21 @@ bool ResourceManager::buildTrajectoryVisualization( meshes_.emplace(meshStart, std::move(visMeshData)); // default material for now - auto phongMaterial = gfx::PhongMaterialData::create_unique(); - phongMaterial->specularColor = {1.0, 1.0, 1.0, 1.0}; - phongMaterial->shininess = 160.f; - phongMaterial->ambientColor = {1.0, 1.0, 1.0, 1.0}; - phongMaterial->perVertexObjectId = true; - - meshMetaData.root.materialID = std::to_string(nextMaterialID_++); - shaderManager_.set(meshMetaData.root.materialID, - static_cast(phongMaterial.release())); + // Populate with defaults from sim's gfx::PhongMaterialData + Mn::Trade::MaterialData materialData = buildDefaultPhongMaterial(); + // Override default values + materialData.mutableAttribute( + Mn::Trade::MaterialAttribute::AmbientColor) = Mn::Color4{1.0}; + materialData.mutableAttribute( + Mn::Trade::MaterialAttribute::SpecularColor) = Mn::Color4{1.0}; + materialData.mutableAttribute( + Mn::Trade::MaterialAttribute::Shininess) = 160.0f; + // Set expected user-defined attributes + materialData = setMaterialDefaultUserAttributes( + materialData, ObjectInstanceShaderType::Phong, true); + + shaderManager_.set(meshMetaData.root.materialID, + std::move(materialData)); meshMetaData.root.meshIDLocal = 0; meshMetaData.root.componentID = 0; @@ -2090,8 +2110,320 @@ bool compareShaderTypeToMnMatType(const ObjectInstanceShaderType typeToCheck, } } // compareShaderTypeToMnMatType +/** + * @brief This function will take an existing @ref Mn::Trade::MaterialData and add + * the missing attributes for the types it does not support, so that it will + * have attributes for all habitat-supported shader types. This should only be + * called if the user has specified a desired shader type that the material does + * not natively support. + * @param origMaterialData The original material from the importer + * @return The new material with attribute support for all supported shader + * types. + */ +Mn::Trade::MaterialData createUniversalMaterial( + const Mn::Trade::MaterialData& origMaterialData) { + // create a material, based on the passed material, that will have reasonable + // attributes to support any possible shader type. should only be called if + // we are not using the Material's natively specified shaderType + + // NOLINTNEXTLINE(google-build-using-namespace) + using namespace Mn::Math::Literals; + + // get source attributes from original material + Cr::Containers::Array newAttributes; + + const auto origMatTypes = origMaterialData.types(); + // add appropriate attributes based on what is missing + // flat material already recognizes Phong and pbr, so don't have to do + // anything + + // multiplicative magic number for scaling from PBR to phong, + // hacky method to attempt to balance Phong and PBR light intensity reactions + const float magicPhongScaling = 1.5f; + + // whether the MaterialAttribute::TextureMatrix has been set already + bool setTexMatrix = false; + if (!(origMatTypes & Mn::Trade::MaterialType::Phong)) { + // add appropriate values for expected attributes to support Phong from + // PbrMetallicRoughnessMaterialData + + const auto& pbrMaterial = + origMaterialData.as(); + + ///////////////// + // calculate Phong values from PBR material values + //////////////// + + // derive ambient color from pbr baseColor + const Mn::Color4 ambientColor = pbrMaterial.baseColor(); + + // If there's a roughness texture, we have no way to use it here. The safest + // fallback is to assume roughness == 1, thus producing no spec highlights. + // If pbrMaterial does not have a roughness value, it returns a 1 by + // default. + const float roughness = + pbrMaterial.hasRoughnessTexture() ? 1.0f : pbrMaterial.roughness(); + + // If there's a metalness texture, we have no way to use it here. + // The safest fallback is to assume non-metal. + const float metalness = + pbrMaterial.hasMetalnessTexture() ? 0.0f : pbrMaterial.metalness(); + + // Heuristic to map roughness to spec power. + // Higher exponent makes the spec highlight larger (lower power) + // example calc : + // https://www.wolframalpha.com/input/?i=1.1+%2B+%281+-+x%29%5E4.5+*+180.0+for+x+from+0+to+1 + // lower power for metal + const float maxShininess = Mn::Math::lerp(250.0f, 120.0f, metalness); + const float shininess = + 1.1f + Mn::Math::pow(1.0f - roughness, 4.5f) * maxShininess; + + // increase spec intensity for metal + // example calc : + // https://www.wolframalpha.com/input/?i=1.0+%2B+10*%28x%5E1.7%29++from+0+to+1 + const float specIntensityScale = + (metalness > 0.0f + ? (metalness < 1.0f + ? (1.0f + 10.0f * Mn::Math::pow(metalness, 1.7f)) + : 11.0f) + : 1.0f); + // Heuristic to map roughness to spec intensity. + // higher exponent decreases intensity. + // example calc : + // https://www.wolframalpha.com/input/?i=1.4+*+%281-x%29%5E2.5++from+0+to+1 + const float specIntensity = + Mn::Math::pow(1.0f - roughness, 2.5f) * 1.4f * specIntensityScale; + + // NOTE: The magic-number multiplication at the end is a hack to + // roughly balance the Phong and PBR light intensity reactions + const Mn::Color4 diffuseColor = ambientColor * magicPhongScaling; + + // Set spec base color to white or material base color, depending on + // metalness. + const Mn::Color4 specBaseColor = + (metalness > 0.0f + ? (metalness < 1.0f + ? Mn::Math::lerp(0xffffffff_rgbaf, ambientColor, + Mn::Math::pow(metalness, 0.5f)) + : ambientColor) + : 0xffffffff_rgbaf); + // NOTE: The magic-number multiplication at the end is a hack to + // roughly balance the Phong and PBR light intensity reactions + const Mn::Color4 specColor = + specBaseColor * specIntensity * magicPhongScaling; + + ///////////////// + // set Phong attributes appropriately from precalculated value + //////////////// + // normal mapping is already present in copied array if present in + // original material. + + arrayAppend(newAttributes, {{MaterialAttribute::Shininess, shininess}, + {MaterialAttribute::AmbientColor, ambientColor}, + {MaterialAttribute::DiffuseColor, diffuseColor}, + {MaterialAttribute::SpecularColor, specColor}}); + // texture transforms, if there's none the returned matrix is an + // identity only copy if we don't already have TextureMatrix + // attribute (if original pbrMaterial does not have that specific + // matrix) + if (!setTexMatrix && + (!pbrMaterial.hasAttribute(MaterialAttribute::TextureMatrix))) { + arrayAppend(newAttributes, {MaterialAttribute::TextureMatrix, + pbrMaterial.commonTextureMatrix()}); + setTexMatrix = true; + } + + if (pbrMaterial.hasAttribute(MaterialAttribute::BaseColorTexture)) { + // only provide texture indices if BaseColorTexture attribute + // exists + const Mn::UnsignedInt BCTexture = pbrMaterial.baseColorTexture(); + arrayAppend(newAttributes, + {{MaterialAttribute::AmbientTexture, BCTexture}, + {MaterialAttribute::DiffuseTexture, BCTexture}}); + if (metalness >= 0.5) { + arrayAppend(newAttributes, + {MaterialAttribute::SpecularTexture, BCTexture}); + } + } + } // if no phong material support exists in material + + if (!(origMatTypes & Mn::Trade::MaterialType::PbrMetallicRoughness)) { + // add appropriate values for expected attributes for PbrMetallicRoughness + // derived from Phong attributes + const auto& phongMaterial = + origMaterialData.as(); + + ///////////////// + // calculate PBR values from Phong material values + //////////////// + + // derive base color from Phong diffuse or ambient color, depending on which + // is present. set to white if neither is present + const Mn::Color4 baseColor = phongMaterial.diffuseColor(); + + // Experimental metalness heuristic using saturation of spec color + // to derive approximation of metalness + const Mn::Color4 specColor = phongMaterial.specularColor(); + + // if specColor alpha == 0 then no metalness + float metalness = 0.0f; + // otherwise, this hacky heuristic will derive a value for metalness based + // on how non-grayscale the specular color is (HSV Saturation). + if (specColor.a() != 0.0f) { + metalness = specColor.saturation(); + } + + ///////////////// + // set PbrMetallicRoughness attributes appropriately from precalculated + // values + //////////////// + + // normal mapping is already present in copied array if present in + // original material. + arrayAppend(newAttributes, {{MaterialAttribute::BaseColor, baseColor}, + {MaterialAttribute::Metalness, metalness}}); + + // if diffuse texture is present, use as base color texture in pbr. + if (phongMaterial.hasAttribute(MaterialAttribute::DiffuseTexture)) { + uint32_t bcTextureVal = phongMaterial.diffuseTexture(); + arrayAppend(newAttributes, + {MaterialAttribute::BaseColorTexture, bcTextureVal}); + } + // texture transforms, if there's none the returned matrix is an + // identity Only copy if we don't already have TextureMatrix attribute + // (if original phongMaterial does not have that specific attribute) + if (!setTexMatrix && + (!phongMaterial.hasAttribute(MaterialAttribute::TextureMatrix))) { + arrayAppend(newAttributes, {MaterialAttribute::TextureMatrix, + phongMaterial.commonTextureMatrix()}); + // setTexMatrix = true; + } + + // base texture + + } // if no PbrMetallicRoughness material support exists in material + + // build flags to support all materials + constexpr auto flags = Mn::Trade::MaterialType::Flat | + Mn::Trade::MaterialType::Phong | + Mn::Trade::MaterialType::PbrMetallicRoughness; + + // create new material from attributes array + Mn::Trade::MaterialData newMaterialData{flags, std::move(newAttributes)}; + + return newMaterialData; +} // namespace + } // namespace +// Specifically for building materials that relied on old defaults +Mn::Trade::MaterialData ResourceManager::buildDefaultPhongMaterial() { + Mn::Trade::MaterialData materialData{ + Mn::Trade::MaterialType::Phong, + {{Mn::Trade::MaterialAttribute::AmbientColor, Mn::Color4{0.1}}, + {Mn::Trade::MaterialAttribute::DiffuseColor, Mn::Color4{0.7 * 0.175}}, + {Mn::Trade::MaterialAttribute::SpecularColor, Mn::Color4{0.2 * 0.175}}, + {Mn::Trade::MaterialAttribute::Shininess, 80.0f}}}; + return materialData; +} // ResourceManager::buildDefaultPhongMaterial + +Mn::Trade::MaterialData ResourceManager::setMaterialDefaultUserAttributes( + const Mn::Trade::MaterialData& material, + ObjectInstanceShaderType shaderTypeToUse, + bool hasVertObjID, + bool hasTxtrObjID, + int txtrIdx) const { + // New material's attributes + Cr::Containers::Array newAttributes; + arrayAppend(newAttributes, Cr::InPlaceInit, "hasPerVertexObjectId", + hasVertObjID); + if (hasTxtrObjID) { + arrayAppend(newAttributes, Cr::InPlaceInit, "objectIdTexturePointer", + textures_.at(txtrIdx).get()); + } + arrayAppend(newAttributes, Cr::InPlaceInit, "shaderTypeToUse", + static_cast(shaderTypeToUse)); + + Cr::Containers::Optional finalMaterial = + Mn::MaterialTools::merge( + material, Mn::Trade::MaterialData{{}, std::move(newAttributes), {}}); + + return std::move(*finalMaterial); +} // ResourceManager::setMaterialDefaultUserAttributes + +std::string ResourceManager::createColorMaterial( + const esp::assets::PhongMaterialColor& materialColor) { + std::string newMaterialID = + Cr::Utility::formatString("phong_amb_{}_dif_{}_spec_{}", + materialColor.ambientColor.toSrgbAlphaInt(), + materialColor.diffuseColor.toSrgbAlphaInt(), + materialColor.specularColor.toSrgbAlphaInt()); + + auto materialResource = + shaderManager_.get(newMaterialID); + + if (materialResource.state() == Mn::ResourceState::NotLoadedFallback) { + // Build a new default phong material + Mn::Trade::MaterialData materialData = buildDefaultPhongMaterial(); + materialData.mutableAttribute( + Mn::Trade::MaterialAttribute::AmbientColor) = + materialColor.ambientColor; + materialData.mutableAttribute( + Mn::Trade::MaterialAttribute::DiffuseColor) = + materialColor.diffuseColor * 0.175; + materialData.mutableAttribute( + Mn::Trade::MaterialAttribute::SpecularColor) = + materialColor.specularColor * 0.175; + + // Set expected user-defined attributes + materialData = setMaterialDefaultUserAttributes( + materialData, ObjectInstanceShaderType::Phong); + shaderManager_.set(newMaterialID, + std::move(materialData)); + } + return newMaterialID; +} // ResourceManager::createColorMaterial + +void ResourceManager::initDefaultMaterials() { + // Build default phong materials + Mn::Trade::MaterialData dfltMaterialData = buildDefaultPhongMaterial(); + // Set expected user-defined attributes + dfltMaterialData = setMaterialDefaultUserAttributes( + dfltMaterialData, ObjectInstanceShaderType::Phong); + // Add to shaderManager at specified key location + shaderManager_.set(DEFAULT_MATERIAL_KEY, + std::move(dfltMaterialData)); + // Build white material + Mn::Trade::MaterialData whiteMaterialData = buildDefaultPhongMaterial(); + whiteMaterialData.mutableAttribute( + Mn::Trade::MaterialAttribute::AmbientColor) = Mn::Color4{1.0}; + // Set expected user-defined attributes + whiteMaterialData = setMaterialDefaultUserAttributes( + whiteMaterialData, ObjectInstanceShaderType::Phong); + // Add to shaderManager at specified key location + shaderManager_.set(WHITE_MATERIAL_KEY, + std::move(whiteMaterialData)); + // Buiild white vertex ID material + Mn::Trade::MaterialData vertIdMaterialData = buildDefaultPhongMaterial(); + vertIdMaterialData.mutableAttribute( + Mn::Trade::MaterialAttribute::AmbientColor) = Mn::Color4{1.0}; + // Set expected user-defined attributes + vertIdMaterialData = setMaterialDefaultUserAttributes( + vertIdMaterialData, ObjectInstanceShaderType::Phong, true); + // Add to shaderManager at specified key location + shaderManager_.set(PER_VERTEX_OBJECT_ID_MATERIAL_KEY, + std::move(vertIdMaterialData)); + + // Build default material for fallback material + auto fallBackMaterial = buildDefaultPhongMaterial(); + // Set expected user-defined attributes + fallBackMaterial = setMaterialDefaultUserAttributes( + fallBackMaterial, ObjectInstanceShaderType::Phong); + // Add to shaderManager as fallback material + shaderManager_.setFallback( + std::move(fallBackMaterial)); +} // ResourceManager::initDefaultMaterials + void ResourceManager::loadMaterials(Importer& importer, LoadedAssetData& loadedAssetData) { // Specify the shaderType to use to render the materials being imported @@ -2106,11 +2438,14 @@ void ResourceManager::loadMaterials(Importer& importer, << "Building " << numMaterials << " materials for asset named '" << assetName << "' : "; + // starting index for all textures corresponding to this material + int textureBaseIndex = loadedAssetData.meshMetaData.textureIndex.first; + if (loadedAssetData.assetInfo.hasSemanticTextures) { - int textureBaseIndex = loadedAssetData.meshMetaData.textureIndex.first; - // TODO: Verify this is correct process for building individual materials - // for each semantic int texture. for (int iMaterial = 0; iMaterial < numMaterials; ++iMaterial) { + // Build material key + std::string materialKey = std::to_string(nextMaterialID_++); + // Retrieving the material just to verify it exists Cr::Containers::Optional materialData = importer.material(iMaterial); @@ -2120,31 +2455,34 @@ void ResourceManager::loadMaterials(Importer& importer, << "."; continue; } - // Semantic texture-based - std::unique_ptr finalMaterial = - gfx::PhongMaterialData::create_unique(); - - // const auto& material = materialData->as(); - - finalMaterial->ambientColor = Mn::Color4{1.0}; - finalMaterial->diffuseColor = Mn::Color4{}; - finalMaterial->specularColor = Mn::Color4{}; - finalMaterial->shaderTypeSpec = - static_cast(ObjectInstanceShaderType::Flat); - // has texture-based semantic annotations - finalMaterial->textureObjectId = true; - // get semantic int texture - finalMaterial->objectIdTexture = - textures_.at(textureBaseIndex + iMaterial).get(); - - shaderManager_.set(std::to_string(nextMaterialID_++), - finalMaterial.release()); + // Semantic texture-based mapping + + // Build a phong material for semantics. TODO: Should this be a + // FlatMaterialData? Populate with defaults from deprecated + // gfx::PhongMaterialData + Mn::Trade::MaterialData newMaterialData = buildDefaultPhongMaterial(); + // Override default values + newMaterialData.mutableAttribute( + Mn::Trade::MaterialAttribute::AmbientColor) = Mn::Color4{1.0}; + newMaterialData.mutableAttribute( + Mn::Trade::MaterialAttribute::DiffuseColor) = Mn::Color4{}; + newMaterialData.mutableAttribute( + Mn::Trade::MaterialAttribute::SpecularColor) = Mn::Color4{}; + + // Set expected user-defined attributes - force to use phong shader for + // semantics + newMaterialData = setMaterialDefaultUserAttributes( + newMaterialData, ObjectInstanceShaderType::Phong, false, true, + textureBaseIndex + iMaterial); + + shaderManager_.set(materialKey, + std::move(newMaterialData)); } } else { for (int iMaterial = 0; iMaterial < numMaterials; ++iMaterial) { - // TODO: - // it seems we have a way to just load the material once in this case, - // as long as the materialName includes the full path to the material + // Build material key + std::string materialKey = std::to_string(nextMaterialID_++); + Cr::Containers::Optional materialData = importer.material(iMaterial); @@ -2155,11 +2493,10 @@ void ResourceManager::loadMaterials(Importer& importer, continue; } - std::unique_ptr finalMaterial; - std::string debugStr = - Cr::Utility::formatString("Idx {:.02d}:", iMaterial); + int numMaterialLayers = materialData->layerCount(); + std::string debugStr = Cr::Utility::formatString( + "Idx {:.02d} has {:.02} layers:", iMaterial, numMaterialLayers); - int textureBaseIndex = loadedAssetData.meshMetaData.textureIndex.first; // If we are not using the material's native shadertype, or flat (Which // all materials already support), expand the Mn::Trade::MaterialData with // appropriate data for all possible shadertypes @@ -2171,32 +2508,45 @@ void ResourceManager::loadMaterials(Importer& importer, "(Expanding existing materialData to support requested shaderType `" "{}`) ", metadata::attributes::getShaderTypeName(shaderTypeToUse)); - materialData = esp::gfx::createUniversalMaterial(*materialData); + materialData = createUniversalMaterial(*materialData); } + // This material data has any per-shader as well as global custom + // user-defined attributes set excluding texture pointer mappings + Corrade::Containers::Optional + custMaterialData; + + // Build based on desired shader to use // pbr shader spec, of material-specified and material specifies pbr if (checkForPassedShaderType( shaderTypeToUse, *materialData, ObjectInstanceShaderType::PBR, Mn::Trade::MaterialType::PbrMetallicRoughness)) { Cr::Utility::formatInto(debugStr, debugStr.size(), "PBR."); - finalMaterial = - buildPbrShadedMaterialData(*materialData, textureBaseIndex); + + // Material with custom settings appropriately set for PBR material + custMaterialData = + buildCustomAttributePbrMaterial(*materialData, textureBaseIndex); // phong shader spec, of material-specified and material specifies phong } else if (checkForPassedShaderType(shaderTypeToUse, *materialData, ObjectInstanceShaderType::Phong, Mn::Trade::MaterialType::Phong)) { Cr::Utility::formatInto(debugStr, debugStr.size(), "Phong."); - finalMaterial = - buildPhongShadedMaterialData(*materialData, textureBaseIndex); + + // Material with custom settings appropriately set for Phong material + custMaterialData = + buildCustomAttributePhongMaterial(*materialData, textureBaseIndex); // flat shader spec or material-specified and material specifies flat } else if (checkForPassedShaderType(shaderTypeToUse, *materialData, ObjectInstanceShaderType::Flat, Mn::Trade::MaterialType::Flat)) { Cr::Utility::formatInto(debugStr, debugStr.size(), "Flat."); - finalMaterial = - buildFlatShadedMaterialData(*materialData, textureBaseIndex); + + // Material with custom settings appropriately set for Flat materials to + // be used in our Phong shader + custMaterialData = + buildCustomAttributeFlatMaterial(*materialData, textureBaseIndex); } else { ESP_CHECK( @@ -2207,15 +2557,188 @@ void ResourceManager::loadMaterials(Importer& importer, metadata::attributes::getShaderTypeName(shaderTypeToUse), iMaterial, assetName)); } + + // Merge all custom attribute except remapped texture pointers with + // original material for final material. custMaterialData should never be + // Cr::Containers::NullOpt since every non-error branch is covered. + Cr::Containers::Optional mergedCustomMaterial = + Mn::MaterialTools::merge( + *custMaterialData, *materialData, + Mn::MaterialTools::MergeConflicts::KeepFirstIfSameType); + + // Now build texture pointer array, with appropriate layers based on + // number of layers in original material + + // New txtrptr-holding material's attributes + Cr::Containers::Array newAttributes{}; + // New txtrptr-holding material's layers + Cr::Containers::Array newLayers{}; + + // Copy all texture pointers into array + // list of all this material's attributes + const Cr::Containers::ArrayView + materialAttributes = materialData->attributeData(); + // Returns nullptr if has no attributes + if (materialAttributes != nullptr) { + // For all layers + for (int layerIdx = 0; layerIdx < numMaterialLayers; ++layerIdx) { + // Add all material texture pointers into new attributes + // find start and end idxs for each layer + int stIdx = materialData->attributeDataOffset(layerIdx); + int endIdx = materialData->attributeDataOffset(layerIdx + 1); + + for (int mIdx = stIdx; mIdx < endIdx; ++mIdx) { + const Mn::Trade::MaterialAttributeData& materialAttribute = + materialAttributes[mIdx]; + auto attrName = materialAttribute.name(); + const auto matType = materialAttribute.type(); + // Find textures and add them to newAttributes + // bool found = (std::string::npos != key.find(strToLookFor)); + if ((matType == Mn::Trade::MaterialAttributeType::UnsignedInt) && + attrName.hasSuffix("Texture")) { + // texture index, copy texture pointer into newAttributes + const Mn::UnsignedInt txtrIdx = + materialAttribute.value(); + // copy texture into new attributes tagged with lowercase material + // name + auto newAttrName = Cr::Utility::formatString( + "{}{}Pointer", + Cr::Utility::String::lowercase(attrName.slice(0, 1)), + attrName.slice(1, attrName.size())); + // Debug display of layer pointers + Cr::Utility::formatInto(debugStr, debugStr.size(), + "| txtr ptr name:{} | idx :{} Layer {}", + newAttrName, (textureBaseIndex + txtrIdx), + layerIdx); + arrayAppend(newAttributes, + {newAttrName, + textures_.at(textureBaseIndex + txtrIdx).get()}); + } // if texture found + } // for each material attribute in layer + // Save this layer's offset + arrayAppend(newLayers, newAttributes.size()); + } // for each layer + } // if material has attributes + + // Merge all texture-pointer custom attributes with material holding + // original attributes + non-texture-pointer custom attributes for final + // material + Cr::Containers::Optional finalMaterial = + Mn::MaterialTools::merge( + *mergedCustomMaterial, + Mn::Trade::MaterialData{ + {}, std::move(newAttributes), std::move(newLayers)}); + ESP_DEBUG() << debugStr; // for now, just use unique ID for material key. This may change if we // expose materials to user for post-load modification - shaderManager_.set(std::to_string(nextMaterialID_++), - finalMaterial.release()); + + shaderManager_.set(materialKey, + std::move(*finalMaterial)); } } } // ResourceManager::loadMaterials +Mn::Trade::MaterialData ResourceManager::buildCustomAttributeFlatMaterial( + const Mn::Trade::MaterialData& materialData, + int textureBaseIndex) { + // NOLINTNEXTLINE(google-build-using-namespace) + using namespace Mn::Math::Literals; + // Custom/remapped attributes for material, to match required Phong shader + // mapping. + Cr::Containers::Array custAttributes; + + // To save on shader switching, a Phong shader with zero lights is used for + // flat materials. This requires custom mapping of material quantities so that + // the Phong shader can find what it is looking for. + const auto& flatMat = materialData.as(); + // Populate base/diffuse color and texture (if present) into flat + // material array + arrayAppend( + custAttributes, + {// Set ambient color from flat material's base/diffuse color + {Mn::Trade::MaterialAttribute::AmbientColor, flatMat.color()}, + // Clear out diffuse and specular colors for flat material + {Mn::Trade::MaterialAttribute::DiffuseColor, 0x00000000_rgbaf}, + // No default shininess in Magnum materials + {Mn::Trade::MaterialAttribute::Shininess, 80.0f}, + + {Mn::Trade::MaterialAttribute::SpecularColor, 0x00000000_rgbaf}}); + // Only populate into ambient texture if present in original + // material + if (flatMat.hasTexture()) { + arrayAppend(custAttributes, + {"ambientTexturePointer", + textures_.at(textureBaseIndex + flatMat.texture()).get()}); + } + // Merge new attributes with those specified in original material + // overridding original ambient, diffuse and specular colors + // Using owning MaterialData constructor to handle potential + // layers + auto finalMaterial = Mn::MaterialTools::merge( + Mn::Trade::MaterialData{{}, std::move(custAttributes), {}}, materialData, + Mn::MaterialTools::MergeConflicts::KeepFirstIfSameType); + + // Set default, expected user attributes for the final material + // and return + return setMaterialDefaultUserAttributes(*finalMaterial, + ObjectInstanceShaderType::Flat); +} // ResourceManager::buildFlatShadedMaterialData + +Mn::Trade::MaterialData ResourceManager::buildCustomAttributePhongMaterial( + const Mn::Trade::MaterialData& materialData, + CORRADE_UNUSED int textureBaseIndex) const { + // Custom/remapped attributes for material, to match required Phong shader + // mapping. + Cr::Containers::Array custAttributes; + + // TODO : specify custom non-texture pointer mappings for Phong materials + // here. + + // Merge new attributes with those specified in original material + // overridding original ambient, diffuse and specular colors + if (custAttributes.size() > 0) { + // Using owning MaterialData constructor to handle potential layers + auto finalMaterial = Mn::MaterialTools::merge( + Mn::Trade::MaterialData{{}, std::move(custAttributes), {}}, + materialData, Mn::MaterialTools::MergeConflicts::KeepFirstIfSameType); + + // Set default, expected user attributes for the final material and return + return setMaterialDefaultUserAttributes(*finalMaterial, + ObjectInstanceShaderType::Phong); + } + return setMaterialDefaultUserAttributes(materialData, + ObjectInstanceShaderType::Phong); + +} // ResourceManager::buildPhongShadedMaterialData + +Mn::Trade::MaterialData ResourceManager::buildCustomAttributePbrMaterial( + const Mn::Trade::MaterialData& materialData, + CORRADE_UNUSED int textureBaseIndex) const { + // Custom/remapped attributes for material, to match required PBR shader + // mapping. + Cr::Containers::Array custAttributes; + + // TODO : specify custom non-texture pointer mappings for PBR materials + // here. + + // Merge new attributes with those specified in original material + // overridding original ambient, diffuse and specular colors + + if (custAttributes.size() > 0) { + // Using owning MaterialData constructor to handle potential layers + auto finalMaterial = Mn::MaterialTools::merge( + Mn::Trade::MaterialData{{}, std::move(custAttributes), {}}, + materialData, Mn::MaterialTools::MergeConflicts::KeepFirstIfSameType); + + // Set default, expected user attributes for the final material and return + return setMaterialDefaultUserAttributes(*finalMaterial, + ObjectInstanceShaderType::PBR); + } + return setMaterialDefaultUserAttributes(materialData, + ObjectInstanceShaderType::PBR); +} // ResourceManager::buildPbrShadedMaterialData + ObjectInstanceShaderType ResourceManager::getMaterialShaderType( const AssetInfo& info) const { // if specified to be force-flat, then should be flat shaded, regardless of @@ -2247,181 +2770,6 @@ bool ResourceManager::checkForPassedShaderType( ((materialData.types() & mnVerificationType) == mnVerificationType))); } -gfx::PhongMaterialData::uptr ResourceManager::buildFlatShadedMaterialData( - const Mn::Trade::MaterialData& materialData, - int textureBaseIndex) { - // NOLINTNEXTLINE(google-build-using-namespace) - using namespace Mn::Math::Literals; - - const auto& material = materialData.as(); - - // To save on shader switching, a Phong shader with zero lights is used for - // flat materials - auto finalMaterial = gfx::PhongMaterialData::create_unique(); - - // texture transform, if the material has a texture; if there's none the - // matrix is an identity - // TODO this check shouldn't be needed, remove when this no longer asserts - // in magnum for untextured materials - if (material.hasTexture()) { - finalMaterial->textureMatrix = material.textureMatrix(); - } - - finalMaterial->ambientColor = material.color(); - if (material.hasTexture()) { - finalMaterial->ambientTexture = - textures_.at(textureBaseIndex + material.texture()).get(); - } - finalMaterial->diffuseColor = 0x00000000_rgbaf; - finalMaterial->specularColor = 0x00000000_rgbaf; - - finalMaterial->shaderTypeSpec = - static_cast(ObjectInstanceShaderType::Flat); - - return finalMaterial; -} // ResourceManager::buildFlatShadedMaterialData - -gfx::PhongMaterialData::uptr ResourceManager::buildPhongShadedMaterialData( - const Mn::Trade::MaterialData& materialData, - int textureBaseIndex) const { - // NOLINTNEXTLINE(google-build-using-namespace) - using namespace Mn::Math::Literals; - const auto& material = materialData.as(); - - auto finalMaterial = gfx::PhongMaterialData::create_unique(); - finalMaterial->shininess = material.shininess(); - - // texture transform, if there's none the matrix is an identity - finalMaterial->textureMatrix = material.commonTextureMatrix(); - - // ambient material properties - finalMaterial->ambientColor = material.ambientColor(); - if (material.hasAttribute(MaterialAttribute::AmbientTexture)) { - finalMaterial->ambientTexture = - textures_.at(textureBaseIndex + material.ambientTexture()).get(); - } - - // diffuse material properties - finalMaterial->diffuseColor = material.diffuseColor(); - if (material.hasAttribute(MaterialAttribute::DiffuseTexture)) { - finalMaterial->diffuseTexture = - textures_.at(textureBaseIndex + material.diffuseTexture()).get(); - } - - // specular material properties - finalMaterial->specularColor = material.specularColor(); - if (material.hasSpecularTexture()) { - finalMaterial->specularTexture = - textures_.at(textureBaseIndex + material.specularTexture()).get(); - } - - // normal mapping - if (material.hasAttribute(MaterialAttribute::NormalTexture)) { - finalMaterial->normalTexture = - textures_.at(textureBaseIndex + material.normalTexture()).get(); - } - - finalMaterial->shaderTypeSpec = - static_cast(ObjectInstanceShaderType::Phong); - - return finalMaterial; -} // ResourceManager::buildPhongShadedMaterialData - -gfx::PbrMaterialData::uptr ResourceManager::buildPbrShadedMaterialData( - const Mn::Trade::MaterialData& materialData, - int textureBaseIndex) const { - // NOLINTNEXTLINE(google-build-using-namespace) - using namespace Mn::Math::Literals; - const auto& material = - materialData.as(); - - auto finalMaterial = gfx::PbrMaterialData::create_unique(); - - // texture transform, if there's none the matrix is an identity - finalMaterial->textureMatrix = material.commonTextureMatrix(); - - // base color (albedo) - finalMaterial->baseColor = material.baseColor(); - if (material.hasAttribute(MaterialAttribute::BaseColorTexture)) { - finalMaterial->baseColorTexture = - textures_.at(textureBaseIndex + material.baseColorTexture()).get(); - } - - // normal map - if (material.hasAttribute(MaterialAttribute::NormalTexture)) { - // must be inside the if clause otherwise assertion fails if no normal - // texture is presented - finalMaterial->normalTextureScale = material.normalTextureScale(); - - finalMaterial->normalTexture = - textures_.at(textureBaseIndex + material.normalTexture()).get(); - } - - // emission - finalMaterial->emissiveColor = material.emissiveColor(); - if (material.hasAttribute(MaterialAttribute::EmissiveTexture)) { - finalMaterial->emissiveTexture = - textures_.at(textureBaseIndex + material.emissiveTexture()).get(); - if (!material.hasAttribute(MaterialAttribute::EmissiveColor)) { - finalMaterial->emissiveColor = Mn::Vector3{1.0f}; - } - } - - // roughness - finalMaterial->roughness = material.roughness(); - if (material.hasRoughnessTexture()) { - finalMaterial->roughnessTexture = - textures_.at(textureBaseIndex + material.roughnessTexture()).get(); - } - - // metallic - finalMaterial->metallic = material.metalness(); - if (material.hasMetalnessTexture()) { - finalMaterial->metallicTexture = - textures_.at(textureBaseIndex + material.metalnessTexture()).get(); - } - - // sanity check when both metallic and roughness materials are presented - if (material.hasMetalnessTexture() && material.hasRoughnessTexture()) { - /* - sanity check using hasNoneRoughnessMetallicTexture() to ensure that: - - both use the same texture coordinate attribute, - - both have the same texture transformation, and - - the metalness is in B and roughness is in G - - It checks for a subset of hasOcclusionRoughnessMetallicTexture(), - so hasOcclusionRoughnessMetallicTexture() is not needed here. - - The normal/roughness/metallic is a totally different packing (in BA - instead of GB), and it is NOT supported in the current version. - so hasNormalRoughnessMetallicTexture() is not needed here. - - */ - CORRADE_ASSERT(material.hasNoneRoughnessMetallicTexture(), - "::buildPbrShadedMaterialData(): if both the metallic " - "and roughness texture exist, they must be packed in the " - "same texture " - "based on glTF 2.0 Spec.", - finalMaterial); - } - - // TODO: - // Support NormalRoughnessMetallicTexture packing - CORRADE_ASSERT(!material.hasNormalRoughnessMetallicTexture(), - "::buildPbrShadedMaterialData(): " - "Sorry. NormalRoughnessMetallicTexture is not supported in " - "the current version. We will work on it.", - finalMaterial); - - // double-sided - finalMaterial->doubleSided = material.isDoubleSided(); - - finalMaterial->shaderTypeSpec = - static_cast(ObjectInstanceShaderType::PBR); - - return finalMaterial; -} // ResourceManager::buildPbrShadedMaterialData - void ResourceManager::loadMeshes(Importer& importer, LoadedAssetData& loadedAssetData) { int meshStart = nextMeshID_; @@ -2444,6 +2792,34 @@ void ResourceManager::loadMeshes(Importer& importer, } } // ResourceManager::loadMeshes +void ResourceManager::loadSkins(Importer& importer, + LoadedAssetData& loadedAssetData) { + if (importer.skin3DCount() == 0) + return; + + const int skinStart = nextSkinID_; + const int skinEnd = skinStart + importer.skin3DCount() - 1; + nextSkinID_ = skinEnd + 1; + loadedAssetData.meshMetaData.setSkinIndices(skinStart, skinEnd); + + for (int iSkin = 0; iSkin < importer.skin3DCount(); ++iSkin) { + auto skinData = std::make_shared(); + + Cr::Containers::Optional skin = + importer.skin3D(iSkin); + CORRADE_INTERNAL_ASSERT(skin); + + // Cache bone names for later association with instance transforms + for (auto jointIt : skin->joints()) { + const auto gfxBoneName = fileImporter_->objectName(jointIt); + skinData->boneNameJointIdMap[gfxBoneName] = jointIt; + } + + skinData->skin = std::make_shared(std::move(*skin)); + skins_.emplace(skinStart + iSkin, std::move(skinData)); + } +} // ResourceManager::loadSkins + Mn::Image2D ResourceManager::convertRGBToSemanticId( const Mn::ImageView2D& srcImage, Cr::Containers::Array& clrToSemanticId) { @@ -2585,7 +2961,30 @@ void ResourceManager::loadTextures(Importer& importer, if (image->isCompressed()) { format = Mn::GL::textureFormat(image->compressedFormat()); } else { - format = Mn::GL::textureFormat(image->format()); + const auto pixelFormat = image->format(); + format = Mn::GL::textureFormat(pixelFormat); + // Modify swizzle for single channel textures so that they are + // greyscale + Mn::UnsignedInt channelCount = pixelFormatChannelCount(pixelFormat); + if (channelCount == 1) { +#ifdef MAGNUM_TARGET_WEBGL + ESP_WARNING() << "Single Channel Greyscale Texture : ID" << iTexture + << "incorrectly displays as red instead of " + "greyscale due to greyscale expansion" + "not yet implemented in WebGL."; +#else + currentTexture->setSwizzle<'r', 'r', 'r', '1'>(); +#endif + } else if (channelCount == 2) { +#ifdef MAGNUM_TARGET_WEBGL + ESP_WARNING() << "Two Channel Greyscale + Alpha Texture : ID" + << iTexture + << "incorrectly displays due to greyscale expansion" + "not yet implemented in WebGL."; +#else + currentTexture->setSwizzle<'r', 'r', 'r', 'g'>(); +#endif + } } // For the very first level, allocate the texture @@ -2785,7 +3184,8 @@ void ResourceManager::addComponent( const MeshTransformNode& meshTransformNode, std::vector& visNodeCache, bool computeAbsoluteAABBs, - std::vector& staticDrawableInfo) { + std::vector& staticDrawableInfo, + const std::shared_ptr& skinData /* = nullptr */) { // Add the object to the scene and set its transformation scene::SceneNode& node = parent.createChild(); visNodeCache.push_back(&node); @@ -2830,7 +3230,8 @@ void ResourceManager::addComponent( node, // scene node lightSetupKey, // lightSetup Key materialKey, // material key - drawables); // drawable group + drawables, // drawable group + skinData); // instance skinning data // compute the bounding box for the mesh we are adding if (computeAbsoluteAABBs) { @@ -2849,10 +3250,48 @@ void ResourceManager::addComponent( child, // mesh transform node visNodeCache, // a vector of scene nodes, the visNodeCache computeAbsoluteAABBs, // compute absolute aabbs - staticDrawableInfo); // a vector of static drawable info + staticDrawableInfo, // a vector of static drawable info + skinData); // instance skinning data } } // addComponent +void ResourceManager::mapSkinnedModelToArticulatedObject( + const MeshTransformNode& meshTransformNode, + const std::shared_ptr& rig, + const std::shared_ptr& skinData) { + // Find skin joint ID that matches the node + const auto& gfxBoneName = meshTransformNode.name; + const auto& boneNameJointIdMap = skinData->skinData->boneNameJointIdMap; + const auto jointIt = boneNameJointIdMap.find(gfxBoneName); + if (jointIt != boneNameJointIdMap.end()) { + int jointId = jointIt->second; + + // Find articulated object link ID that matches the node + const auto linkIds = rig->getLinkIdsWithBase(); + const auto linkId = + std::find_if(linkIds.begin(), linkIds.end(), + [&](int i) { return gfxBoneName == rig->getLinkName(i); }); + + // Map the articulated object link associated with the skin joint + if (linkId != linkIds.end()) { + auto* articulatedObjectNode = &rig->getLink(*linkId.base()).node(); + + // This node will be used for rendering. + auto& transformNode = articulatedObjectNode->createChild(); + skinData->jointIdToTransformNode[jointId] = &transformNode; + + // First node found is the root + if (!skinData->rootArticulatedObjectNode) { + skinData->rootArticulatedObjectNode = articulatedObjectNode; + } + } + } + + for (const auto& child : meshTransformNode.children) { + mapSkinnedModelToArticulatedObject(child, rig, skinData); + } +} // mapSkinnedModelToArticulatedObject + void ResourceManager::addPrimitiveToDrawables(int primitiveID, scene::SceneNode& node, DrawableGroup* drawables) { @@ -2877,28 +3316,44 @@ void ResourceManager::removePrimitiveMesh(int primitiveID) { primitive_meshes_.erase(primMeshIter); } -void ResourceManager::createDrawable(Mn::GL::Mesh* mesh, - gfx::Drawable::Flags& meshAttributeFlags, - scene::SceneNode& node, - const Mn::ResourceKey& lightSetupKey, - const Mn::ResourceKey& materialKey, - DrawableGroup* group /* = nullptr */) { - const auto& materialDataType = - shaderManager_.get(materialKey)->type; +void ResourceManager::createDrawable( + Mn::GL::Mesh* mesh, + gfx::Drawable::Flags& meshAttributeFlags, + scene::SceneNode& node, + const Mn::ResourceKey& lightSetupKey, + const Mn::ResourceKey& materialKey, + DrawableGroup* group /* = nullptr */, + const std::shared_ptr& skinData /* = nullptr */) { + auto material = shaderManager_.get(materialKey); + + ObjectInstanceShaderType materialDataType = + static_cast( + material->mutableAttribute("shaderTypeToUse")); + + // If shader type to use has not been explicitly specified, use best shader + // supported by material + if ((materialDataType < ObjectInstanceShaderType::Flat) || + (materialDataType > ObjectInstanceShaderType::PBR)) { + const auto types = material->types(); + if (types >= Mn::Trade::MaterialType::PbrMetallicRoughness) { + materialDataType = ObjectInstanceShaderType::PBR; + } else { + materialDataType = ObjectInstanceShaderType::Phong; + } + } switch (materialDataType) { - case gfx::MaterialDataType::None: - CORRADE_INTERNAL_ASSERT_UNREACHABLE(); - break; - case gfx::MaterialDataType::Phong: + case ObjectInstanceShaderType::Flat: + case ObjectInstanceShaderType::Phong: node.addFeature( mesh, // render mesh meshAttributeFlags, // mesh attribute flags shaderManager_, // shader manager lightSetupKey, // lightSetup key materialKey, // material key - group); // drawable group + group, // drawable group + skinData); // instance skinning data break; - case gfx::MaterialDataType::Pbr: + case ObjectInstanceShaderType::PBR: node.addFeature( mesh, // render mesh meshAttributeFlags, // mesh attribute flags @@ -2908,8 +3363,17 @@ void ResourceManager::createDrawable(Mn::GL::Mesh* mesh, group, // drawable group activePbrIbl_ >= 0 ? pbrImageBasedLightings_[activePbrIbl_].get() : nullptr); // pbr image based lighting + // TODO: Carry skinData to PbrDrawable. break; + default: + CORRADE_INTERNAL_ASSERT_UNREACHABLE(); } + + drawableCountAndNumFaces_.first += 1; + if (mesh) { + drawableCountAndNumFaces_.second += mesh->count() / 3; + } + } // ResourceManager::createDrawable void ResourceManager::initDefaultLightSetups() { @@ -2933,20 +3397,6 @@ void ResourceManager::initPbrImageBasedLighting( activePbrIbl_ = 0; } -void ResourceManager::initDefaultMaterials() { - shaderManager_.set(DEFAULT_MATERIAL_KEY, - new gfx::PhongMaterialData{}); - auto* whiteMaterialData = new gfx::PhongMaterialData; - whiteMaterialData->ambientColor = Mn::Color4{1.0}; - shaderManager_.set(WHITE_MATERIAL_KEY, whiteMaterialData); - auto* perVertexObjectId = new gfx::PhongMaterialData{}; - perVertexObjectId->perVertexObjectId = true; - perVertexObjectId->ambientColor = Mn::Color4{1.0}; - shaderManager_.set(PER_VERTEX_OBJECT_ID_MATERIAL_KEY, - perVertexObjectId); - shaderManager_.setFallback(new gfx::PhongMaterialData{}); -} - bool ResourceManager::isLightSetupCompatible( const LoadedAssetData& loadedAssetData, const Mn::ResourceKey& lightSetupKey) const { @@ -2998,7 +3448,7 @@ void ResourceManager::joinSemanticHierarchy( const MeshMetaData& metaData, const MeshTransformNode& node, const Mn::Matrix4& transformFromParentToWorld) const { - Magnum::Matrix4 transformFromLocalToWorld = + Mn::Matrix4 transformFromLocalToWorld = transformFromParentToWorld * node.transformFromLocalToParent; // If the mesh local id is not -1, populate the mesh data. @@ -3024,7 +3474,7 @@ void ResourceManager::joinSemanticHierarchy( // Save the vertices for (const auto& pos : vertices) { - mesh.vbo.push_back(Magnum::EigenIntegration::cast( + mesh.vbo.push_back(Mn::EigenIntegration::cast( transformFromLocalToWorld.transformPoint(pos))); } @@ -3081,7 +3531,7 @@ std::unique_ptr ResourceManager::createJoinedSemanticCollisionMesh( const MeshMetaData& metaData = getMeshMetaData(filename); - Magnum::Matrix4 identity; + Mn::Matrix4 identity; joinSemanticHierarchy(*mesh, objectIds, metaData, metaData.root, identity); return mesh; diff --git a/src/esp/assets/ResourceManager.h b/src/esp/assets/ResourceManager.h index 964f9b476a..9da5ac71e0 100644 --- a/src/esp/assets/ResourceManager.h +++ b/src/esp/assets/ResourceManager.h @@ -12,7 +12,6 @@ #include #include #include -#include #include #include @@ -43,6 +42,8 @@ class VoxelGrid; namespace gfx { class Drawable; class PbrImageBasedLighting; +struct SkinData; +struct InstanceSkinData; namespace replay { class Recorder; } @@ -70,6 +71,7 @@ class CCSemanticObject; struct SceneConfiguration; } // namespace scene namespace physics { +class ArticulatedObject; class PhysicsManager; class RigidObject; } // namespace physics @@ -536,12 +538,14 @@ class ResourceManager { * gfx::Drawable. */ - void createDrawable(Mn::GL::Mesh* mesh, - gfx::Drawable::Flags& meshAttributeFlags, - scene::SceneNode& node, - const Mn::ResourceKey& lightSetupKey, - const Mn::ResourceKey& materialKey, - DrawableGroup* group = nullptr); + void createDrawable( + Mn::GL::Mesh* mesh, + gfx::Drawable::Flags& meshAttributeFlags, + scene::SceneNode& node, + const Mn::ResourceKey& lightSetupKey, + const Mn::ResourceKey& materialKey, + DrawableGroup* group = nullptr, + const std::shared_ptr& skinData = nullptr); /** * @brief Remove the specified primitive mesh. @@ -720,13 +724,26 @@ class ResourceManager { /** * @brief Build data for a report for vertex color mapping to semantic scene * objects - this list of strings will disclose which colors are found in - * vertices but not in semantic scene descirptors, and which semantic objects + * vertices but not in semantic scene descriptors, and which semantic objects * do not have their colors mapped in mesh verts. */ std::vector buildVertexColorMapReport( const std::shared_ptr& stageAttributes); + /** + * @brief Get the count of Drawables and the total face count across all + * Drawables in the scene. This is helpful for troubleshooting runtime perf. + * See also resetDrawableCountAndNumFaces. + */ + auto getDrawableCountAndNumFaces() { return drawableCountAndNumFaces_; } + + /** + * @brief Reset this count and this number. See also + * getDrawableCountAndNumFaces. + */ + void resetDrawableCountAndNumFaces() { drawableCountAndNumFaces_ = {0, 0}; } + private: /** * @brief Load the requested mesh info into @ref meshInfo corresponding to @@ -756,6 +773,35 @@ class ResourceManager { */ void buildPrimitiveAssetData(const std::string& primTemplateHandle); + /** + * @brief this will build a Phong @ref Magnum::Trade::MaterialData using + * default attributes from deprecated/removed esp::gfx::PhongMaterialData. + * @return The new phong color populated with default values + */ + Mn::Trade::MaterialData buildDefaultPhongMaterial(); + + /** + * @brief Define and set user-defined attributes for the passed + * @ref Magnum::Trade::MaterialData. + * @param material The material to initialize with the expected + * Habitat-specific user-defined attributes. + * @param shaderTypeToUse What shader to use to render the objects with this + * material. May not be the same as the material type. + * @param hasVertObjID Whether or not the material has vertex-based object ids + * for semantics. + * @param hasTxtrObjID Whether or not the material has texture-based object + * ids for semantics. + * @param txtrIdx The absolute index in the @ref textures_ store for the semantic + * annotation texture. + * @return the updated material + */ + Mn::Trade::MaterialData setMaterialDefaultUserAttributes( + const Mn::Trade::MaterialData& material, + ObjectInstanceShaderType shaderTypeToUse, + bool hasVertObjID = false, + bool hasTxtrObjID = false, + int txtrIdx = -1) const; + /** * @brief Configure the importerManager_ GL Extensions appropriately based on * compilation flags, before any general assets are imported. This should @@ -801,6 +847,7 @@ class ResourceManager { inline bool isRenderAssetGeneral(AssetType type) { return type == AssetType::MP3D_MESH || type == AssetType::UNKNOWN; } + /** * @brief Recursive construction of scene nodes for an asset. * @@ -822,15 +869,36 @@ class ResourceManager { * @param computeAbsoluteAABBs whether absolute bounding boxes should be * computed * @param staticDrawableInfo structure holding the drawable infos for aabbs + * @param skinData structure holding the skin and rig configuration for the + * instance + */ + void addComponent( + const MeshMetaData& metaData, + scene::SceneNode& parent, + const Mn::ResourceKey& lightSetupKey, + DrawableGroup* drawables, + const MeshTransformNode& meshTransformNode, + std::vector& visNodeCache, + bool computeAbsoluteAABBs, + std::vector& staticDrawableInfo, + const std::shared_ptr& skinData = nullptr); + + /** + * @brief Recursive construction of instance skinning data. + * + * Fills the fields of a @ref InstanceSkinData to enable skinned mesh rendering + * by associating each bone to a corresponding articulated object link. + * + * @param meshTransformNode The @ref MeshTransformNode being traversed. + * @param creationInfo Creation information for the instance which contains + * the rig. + * @param skinData Structure holding the skin and rig configuration for the + * instance. */ - void addComponent(const MeshMetaData& metaData, - scene::SceneNode& parent, - const Mn::ResourceKey& lightSetupKey, - DrawableGroup* drawables, - const MeshTransformNode& meshTransformNode, - std::vector& visNodeCache, - bool computeAbsoluteAABBs, - std::vector& staticDrawableInfo); + void mapSkinnedModelToArticulatedObject( + const MeshTransformNode& meshTransformNode, + const std::shared_ptr& rig, + const std::shared_ptr& skinData); /** * @brief Load textures from importer into assets, and update metaData for @@ -853,6 +921,15 @@ class ResourceManager { */ void loadMeshes(Importer& importer, LoadedAssetData& loadedAssetData); + /** + * @brief Load skins from importer into assets. + * + * @param importer The importer already loaded with information for the + * asset. + * @param loadedAssetData The asset's @ref LoadedAssetData object. + */ + void loadSkins(Importer& importer, LoadedAssetData& loadedAssetData); + /** * @brief Recursively build a unified @ref MeshData from loaded assets via a * tree of @ref MeshTransformNode. @@ -913,7 +990,7 @@ class ResourceManager { /** * @brief Boolean check if @p typeToCheck aligns with passed types explicitly * specified, or type in material - * @param typeToCheck The ObjectInstanceShaderType value beign queried for. + * @param typeToCheck The ObjectInstanceShaderType value being queried for. * @param materialData The material whose type we are verifying against * @param verificationType The ObjectInstanceShaderType we are verifying * against @@ -929,39 +1006,45 @@ class ResourceManager { Mn::Trade::MaterialType mnVerificationType) const; /** - * @brief Build a @ref PhongMaterialData for use with flat shading + * @brief Build a @ref Magnum::Trade::MaterialData for use with Flat shading + * that holds all custom attributes except texture pointers. + * Note : habitat-sim currently uses the Phong shader for Flat materials. * * Textures must already be loaded for the asset this material belongs to * * @param material Material data with texture IDs - * @param textureBaseIndex Base index of the assets textures in textures_ + * @param textureBaseIndex Base index of the assets textures in @ref textures_ + * store */ - gfx::PhongMaterialData::uptr buildFlatShadedMaterialData( + Mn::Trade::MaterialData buildCustomAttributeFlatMaterial( const Mn::Trade::MaterialData& materialData, int textureBaseIndex); /** - * @brief Build a @ref PhongMaterialData for use with phong shading + * @brief Build a @ref Magnum::Trade::MaterialData for use with Phong shading + * that holds all custom attributes except texture pointers. * * Textures must already be loaded for the asset this material belongs to * * @param material Material data with texture IDs - * @param textureBaseIndex Base index of the assets textures in textures_ - + * @param textureBaseIndex Base index of the assets textures in @ref textures_ + * store */ - gfx::PhongMaterialData::uptr buildPhongShadedMaterialData( + Mn::Trade::MaterialData buildCustomAttributePhongMaterial( const Mn::Trade::MaterialData& material, int textureBaseIndex) const; /** - * @brief Build a @ref PbrMaterialData for use with PBR shading + * @brief Build a @ref Magnum::Trade::MaterialData for use with PBR shading + * that holds all custom attributes except texture pointers. * * Textures must already be loaded for the asset this material belongs to * * @param material Material data with texture IDs - * @param textureBaseIndex Base index of the assets textures in textures_ + * @param textureBaseIndex Base index of the assets textures in @ref textures_ + * store */ - gfx::PbrMaterialData::uptr buildPbrShadedMaterialData( + Mn::Trade::MaterialData buildCustomAttributePbrMaterial( const Mn::Trade::MaterialData& material, int textureBaseIndex) const; @@ -1172,7 +1255,7 @@ class ResourceManager { const std::vector& staticDrawableInfo); /** - * @brief Compute absolute transformations of all drwables stored in + * @brief Compute absolute transformations of all drawables stored in * staticDrawableInfo_ */ std::vector computeAbsoluteTransformations( @@ -1196,6 +1279,7 @@ class ResourceManager { * @brief The mesh data for loaded assets. */ std::map> meshes_; + std::pair drawableCountAndNumFaces_{0, 0}; /** * @brief The next available unique ID for loaded textures @@ -1213,8 +1297,18 @@ class ResourceManager { int nextMaterialID_ = 0; /** - * @brief Storage for precomuted voxel grids. Useful for when multiple objects - * in a scene are using the same VoxelGrid. + * @brief The next available unique ID for loaded skins + */ + int nextSkinID_ = 0; + + /** + * @brief The skin data for loaded assets. + */ + std::map> skins_; + + /** + * @brief Storage for precomputed voxel grids. Useful for when multiple + * objects in a scene are using the same VoxelGrid. * * Maps absolute path keys to VoxelGrid. */ @@ -1309,7 +1403,7 @@ class ResourceManager { /** * @brief The imaged based lighting for PBR, each is a collection of * an environment map, an irradiance map, a BRDF lookup table (2D texture), - * and a pre-fitered map + * and a pre-filtered map */ std::vector> pbrImageBasedLightings_; diff --git a/src/esp/bindings/GfxReplayBindings.cpp b/src/esp/bindings/GfxReplayBindings.cpp index bbb9643287..e60e027952 100644 --- a/src/esp/bindings/GfxReplayBindings.cpp +++ b/src/esp/bindings/GfxReplayBindings.cpp @@ -110,6 +110,19 @@ void initGfxReplayBindings(py::module& m) { }, R"(Write all saved keyframes to a string, then discard the keyframes.)") + .def( + "write_incremental_saved_keyframes_to_string_array", + [](ReplayManager& self) { + if (!self.getRecorder()) { + throw std::runtime_error( + "replay save not enabled. See " + "SimulatorConfiguration.enable_gfx_replay_save."); + } + return self.getRecorder() + ->writeIncrementalSavedKeyframesToStringArray(); + }, + R"(Write all saved keyframes to individual strings. See Recorder.h for details.)") + .def("read_keyframes_from_file", &ReplayManager::readKeyframesFromFile, R"(Create a Player object from a replay file.)"); } diff --git a/src/esp/bindings/MetadataMediatorBindings.cpp b/src/esp/bindings/MetadataMediatorBindings.cpp index f05d898c3b..8ff81daa42 100644 --- a/src/esp/bindings/MetadataMediatorBindings.cpp +++ b/src/esp/bindings/MetadataMediatorBindings.cpp @@ -72,6 +72,11 @@ void initMetadataMediatorBindings(py::module& m) { .def( "get_scene_handles", &MetadataMediator::getAllSceneInstanceHandles, R"(Returns a list the names of all the available scene instances in the currently active dataset.)") + .def( + "get_scene_user_defined", + &MetadataMediator::getSceneInstanceUserConfiguration, + R"(Returns the user_defined attributes for the scene instance specified by scene_name)", + "scene_name"_a) .def_property_readonly( "summary", &MetadataMediator::getDatasetsOverview, R"(This provides a summary of the datasets currently loaded.)") diff --git a/src/esp/bindings/PhysicsWrapperManagerBindings.cpp b/src/esp/bindings/PhysicsWrapperManagerBindings.cpp index 2c47b9e325..39e6e82fd3 100644 --- a/src/esp/bindings/PhysicsWrapperManagerBindings.cpp +++ b/src/esp/bindings/PhysicsWrapperManagerBindings.cpp @@ -286,7 +286,7 @@ void initPhysicsWrapperManagerBindings(pybind11::module& m) { "filepath"_a, "fixed_base"_a = false, "global_scale"_a = 1.0, "mass_scale"_a = 1.0, "force_reload"_a = false, - "maintain_link_order"_a = false, + "maintain_link_order"_a = false, "intertia_from_urdf"_a = false, "light_setup_key"_a = DEFAULT_LIGHTING_KEY, R"(Load and parse a URDF file using the given 'filepath' into a model, then use this model to instantiate an Articulated Object in the world. diff --git a/src/esp/bindings/SimBindings.cpp b/src/esp/bindings/SimBindings.cpp index 3b9c0eeea8..70e96687f2 100644 --- a/src/esp/bindings/SimBindings.cpp +++ b/src/esp/bindings/SimBindings.cpp @@ -360,6 +360,12 @@ void initSimBindings(py::module& m) { R"(Get a copy of the settings for an existing rigid constraint.)") .def("remove_rigid_constraint", &Simulator::removeRigidConstraint, "constraint_id"_a, R"(Remove a rigid constraint by id.)") + .def( + "get_runtime_perf_stat_names", &Simulator::getRuntimePerfStatNames, + R"(Runtime perf stats are various scalars helpful for troubleshooting runtime perf. This can be called once at startup. See also get_runtime_perf_stat_values.)") + .def( + "get_runtime_perf_stat_values", &Simulator::getRuntimePerfStatValues, + R"(Runtime perf stats are various scalars helpful for troubleshooting runtime perf. These values generally change after every sim step. See also get_runtime_perf_stat_names.)") .def("get_debug_line_render", &Simulator::getDebugLineRender, pybind11::return_value_policy::reference, R"(Get visualization helper for rendering lines.)"); @@ -457,7 +463,11 @@ void initSimBindings(py::module& m) { [](AbstractReplayRenderer& self) { return py::capsule(self.getCudaColorBufferDevicePointer()); }, - R"(Retrieve the depth buffer as a CUDA device pointer.)"); + R"(Retrieve the depth buffer as a CUDA device pointer.)") + .def("debug_line_render", &AbstractReplayRenderer::getDebugLineRender, + R"(Get visualization helper for rendering lines.)") + .def("unproject", &AbstractReplayRenderer::unproject, + R"(Unproject a screen-space point to a world-space ray.)"); } } // namespace sim diff --git a/src/esp/gfx/CMakeLists.txt b/src/esp/gfx/CMakeLists.txt index bbfdd7d21c..5d64eb3508 100644 --- a/src/esp/gfx/CMakeLists.txt +++ b/src/esp/gfx/CMakeLists.txt @@ -14,13 +14,11 @@ set( DrawableGroup.h GenericDrawable.cpp GenericDrawable.h + SkinData.h MeshVisualizerDrawable.cpp MeshVisualizerDrawable.h LightSetup.cpp LightSetup.h - MaterialData.h - MaterialUtil.cpp - MaterialUtil.h magnum.h RenderCamera.cpp RenderCamera.h diff --git a/src/esp/gfx/DebugLineRender.cpp b/src/esp/gfx/DebugLineRender.cpp index f377d49bbf..ceb10ede6f 100644 --- a/src/esp/gfx/DebugLineRender.cpp +++ b/src/esp/gfx/DebugLineRender.cpp @@ -91,6 +91,11 @@ void DebugLineRender::setLineWidth(float lineWidth) { void DebugLineRender::flushLines(const Magnum::Matrix4& camMatrix, const Magnum::Matrix4& projMatrix, const Magnum::Vector2i& viewport) { + flushLines(projMatrix * camMatrix, viewport); +} + +void DebugLineRender::flushLines(const Magnum::Matrix4& projCamMatrix, + const Magnum::Vector2i& viewport) { CORRADE_ASSERT(_glResources, "DebugLineRender::flushLines: no GL resources; see " "also releaseGLResources", ); @@ -134,7 +139,7 @@ void DebugLineRender::flushLines(const Magnum::Matrix4& camMatrix, Mn::Vector3(0, x * sqrtOfTwo, 0), Mn::Vector3(0, -x * sqrtOfTwo, 0)}; - Mn::Matrix4 projCam = projMatrix * camMatrix; + const auto& projCam = projCamMatrix; auto submitLinesWithOffsets = [&]() { for (const auto& offset : offsets) { diff --git a/src/esp/gfx/DebugLineRender.h b/src/esp/gfx/DebugLineRender.h index f3526f5768..d3a1edf8ac 100644 --- a/src/esp/gfx/DebugLineRender.h +++ b/src/esp/gfx/DebugLineRender.h @@ -78,6 +78,9 @@ class DebugLineRender { const Magnum::Matrix4& projMatrix, const Magnum::Vector2i& viewport); + void flushLines(const Magnum::Matrix4& projCamMatrix, + const Magnum::Vector2i& viewport); + /** * @brief Push (multiply) a transform onto the transform stack, affecting all * line-drawing until popped. Must be paired with popTransform(). For example, diff --git a/src/esp/gfx/Drawable.h b/src/esp/gfx/Drawable.h index 0f18dbfd6c..5c7ea91d2f 100644 --- a/src/esp/gfx/Drawable.h +++ b/src/esp/gfx/Drawable.h @@ -9,7 +9,7 @@ #include #include #include - +#include #include "esp/core/Esp.h" namespace esp { @@ -127,6 +127,16 @@ class Drawable : public Magnum::SceneGraph::Drawable3D { return *mesh_; } + /** + * Set or change this drawable's @ref Magnum::Trade::MaterialData values from passed material. + * This is only pertinent for material-equipped drawables. + * @param material + */ + virtual void setMaterialValues( + CORRADE_UNUSED const Magnum::Resource& + material) {} + protected: /** * @brief Draw the object using given camera diff --git a/src/esp/gfx/GenericDrawable.cpp b/src/esp/gfx/GenericDrawable.cpp index 8390f144f5..5b01a48503 100644 --- a/src/esp/gfx/GenericDrawable.cpp +++ b/src/esp/gfx/GenericDrawable.cpp @@ -6,9 +6,16 @@ #include #include +#include #include #include +#include +#include +#include "Corrade/Containers/GrowableArray.h" +#include "Magnum/Types.h" +#include "esp/core/Check.h" +#include "esp/gfx/SkinData.h" #include "esp/scene/SceneNode.h" namespace Mn = Magnum; @@ -16,64 +23,97 @@ namespace Mn = Magnum; namespace esp { namespace gfx { -GenericDrawable::GenericDrawable(scene::SceneNode& node, - Mn::GL::Mesh* mesh, - Drawable::Flags& meshAttributeFlags, - ShaderManager& shaderManager, - const Mn::ResourceKey& lightSetupKey, - const Mn::ResourceKey& materialDataKey, - DrawableGroup* group /* = nullptr */) +GenericDrawable::GenericDrawable( + scene::SceneNode& node, + Mn::GL::Mesh* mesh, + Drawable::Flags& meshAttributeFlags, + ShaderManager& shaderManager, + const Mn::ResourceKey& lightSetupKey, + const Mn::ResourceKey& materialDataKey, + DrawableGroup* group /* = nullptr */, + const std::shared_ptr& skinData /* = nullptr */) : Drawable{node, mesh, DrawableType::Generic, group}, shaderManager_{shaderManager}, lightSetup_{shaderManager.get(lightSetupKey)}, - materialData_{ - shaderManager.get(materialDataKey)} { + skinData_(skinData), + jointTransformations_(), + meshAttributeFlags_{meshAttributeFlags} { + setMaterialValuesInternal( + shaderManager.get( + materialDataKey)); + + // update the shader early here to to avoid doing it during the render loop + if (glMeshExists()) { + updateShader(); + } +} + +void GenericDrawable::setMaterialValuesInternal( + const Mn::Resource& + material) { + materialData_ = material; + flags_ = Mn::Shaders::PhongGL::Flag::ObjectId; + const auto& tmpMaterial = materialData_->as(); + + matCache.ambientColor = tmpMaterial.ambientColor(); + matCache.diffuseColor = tmpMaterial.diffuseColor(); + matCache.specularColor = tmpMaterial.specularColor(); + matCache.shininess = tmpMaterial.shininess(); /* If texture transformation is specified, enable it only if the material is actually textured -- it's an error otherwise */ - if (materialData_->textureMatrix != Mn::Matrix3{} && - (materialData_->ambientTexture || materialData_->diffuseTexture || - materialData_->specularTexture || materialData_->specularTexture || - materialData_->textureObjectId)) { + if (tmpMaterial.commonTextureMatrix() != Mn::Matrix3{} && + (materialData_->hasAttribute("ambientTexturePointer") || + materialData_->hasAttribute("baseColorTexturePointer") || + materialData_->hasAttribute("diffuseTexturePointer") || + materialData_->hasAttribute("specularTexturePointer") || + materialData_->hasAttribute("objectIdTexturePointer"))) { flags_ |= Mn::Shaders::PhongGL::Flag::TextureTransformation; + matCache.textureMatrix = tmpMaterial.commonTextureMatrix(); } - if (materialData_->ambientTexture) { + if (materialData_->hasAttribute("ambientTexturePointer")) { flags_ |= Mn::Shaders::PhongGL::Flag::AmbientTexture; + matCache.ambientTexture = + materialData_->attribute("ambientTexturePointer"); } - if (materialData_->diffuseTexture) { + if (materialData_->hasAttribute("diffuseTexturePointer")) { flags_ |= Mn::Shaders::PhongGL::Flag::DiffuseTexture; + matCache.diffuseTexture = + materialData_->attribute("diffuseTexturePointer"); } - if (materialData_->specularTexture) { + if (materialData_->hasAttribute("specularTexturePointer")) { flags_ |= Mn::Shaders::PhongGL::Flag::SpecularTexture; + matCache.specularTexture = + materialData_->attribute("specularTexturePointer"); } - if (materialData_->normalTexture) { - if (meshAttributeFlags & Drawable::Flag::HasTangent) { + if (materialData_->hasAttribute("normalTexturePointer")) { + matCache.normalTexture = + materialData_->attribute("normalTexturePointer"); + if (meshAttributeFlags_ & Drawable::Flag::HasTangent) { flags_ |= Mn::Shaders::PhongGL::Flag::NormalTexture; - if (meshAttributeFlags & Drawable::Flag::HasSeparateBitangent) { + + if (meshAttributeFlags_ & Drawable::Flag::HasSeparateBitangent) { flags_ |= Mn::Shaders::PhongGL::Flag::Bitangent; } } else { - ESP_WARNING() << "Mesh does not have tangents and Magnum cannot generate " - "them yet, ignoring a normal map"; + ESP_DEBUG() << "Mesh does not have tangents and Magnum cannot generate " + "them yet, ignoring a normal map"; } } - if (materialData_->perVertexObjectId) { + if (materialData_->attribute("hasPerVertexObjectId")) { flags_ |= Mn::Shaders::PhongGL::Flag::InstancedObjectId; } - if (materialData_->textureObjectId) { + if (materialData_->hasAttribute("objectIdTexturePointer")) { flags_ |= Mn::Shaders::PhongGL::Flag::ObjectIdTexture; + matCache.objectIdTexture = + materialData_->attribute("objectIdTexturePointer"); } - if (meshAttributeFlags & Drawable::Flag::HasVertexColor) { + if (meshAttributeFlags_ & Drawable::Flag::HasVertexColor) { flags_ |= Mn::Shaders::PhongGL::Flag::VertexColor; } - - // update the shader early here to to avoid doing it during the render loop - if (glMeshExists()) { - updateShader(); - } -} +} // GenericDrawable::setMaterialValuesInternal void GenericDrawable::setLightSetup(const Mn::ResourceKey& resourceKey) { lightSetup_ = shaderManager_.get(resourceKey); @@ -99,8 +139,12 @@ void GenericDrawable::updateShaderLightingParameters( for (Mn::UnsignedInt i = 0; i < lightSetup_->size(); ++i) { const auto& lightInfo = (*lightSetup_)[i]; - lightPositions.emplace_back(getLightPositionRelativeToCamera( - lightInfo, transformationMatrix, cameraMatrix)); + Mn::Vector4 pos = getLightPositionRelativeToCamera( + lightInfo, transformationMatrix, cameraMatrix); + // flip directional lights to faciliate faster, non-forking calc in + // shader. Leave non-directional lights unchanged + pos *= (pos[3] * 2) - 1; + lightPositions.emplace_back(pos); const auto& lightColor = (*lightSetup_)[i].color; lightColors.emplace_back(lightColor); @@ -114,10 +158,10 @@ void GenericDrawable::updateShaderLightingParameters( // See documentation in src/deps/magnum/src/Magnum/Shaders/Phong.h (*shader_) - .setAmbientColor(materialData_->ambientColor * ambientLightColor) - .setDiffuseColor(materialData_->diffuseColor) - .setSpecularColor(materialData_->specularColor) - .setShininess(materialData_->shininess) + .setAmbientColor(matCache.ambientColor * ambientLightColor) + .setDiffuseColor(matCache.diffuseColor) + .setSpecularColor(matCache.specularColor) + .setShininess(matCache.shininess) .setLightPositions(lightPositions) .setLightColors(lightColors) .setLightRanges(lightRanges); @@ -132,45 +176,107 @@ void GenericDrawable::draw(const Mn::Matrix4& transformationMatrix, updateShaderLightingParameters(transformationMatrix, camera); + Mn::Matrix3x3 rotScale = transformationMatrix.rotationScaling(); + // Find determinant to calculate backface culling winding dir + const float normalDet = rotScale.determinant(); + // Normal matrix is calculated as `m.inverted().transposed()`, and + // `m.inverted()` is the same as `m.comatrix().transposed()/m.determinant()`. + // We need the determinant to figure out the winding direction as well, thus + // we calculate it separately and then do + // `(m.comatrix().transposed()/determinant).transposed()`, which is the same + // as `m.comatrix()/determinant`. + Mn::Matrix3x3 normalMatrix = rotScale.comatrix() / normalDet; + + // Flip winding direction to correct handle backface culling + if (normalDet < 0) { + Mn::GL::Renderer::setFrontFace(Mn::GL::Renderer::FrontFace::ClockWise); + } + (*shader_) // e.g., semantic mesh has its own per vertex annotation, which has been // uploaded to GPU so simply pass 0 to the uniform "objectId" in the // fragment shader .setObjectId( static_cast(camera).useDrawableIds() ? drawableId_ - : (materialData_->perVertexObjectId || materialData_->textureObjectId) + : ((((flags_ & Mn::Shaders::PhongGL::Flag::InstancedObjectId) == + Mn::Shaders::PhongGL::Flag::InstancedObjectId) || + ((flags_ & Mn::Shaders::PhongGL::Flag::ObjectIdTexture)) == + Mn::Shaders::PhongGL::Flag::ObjectIdTexture)) ? 0 : node_.getSemanticId()) .setTransformationMatrix(transformationMatrix) .setProjectionMatrix(camera.projectionMatrix()) - .setNormalMatrix(transformationMatrix.normalMatrix()); + .setNormalMatrix(normalMatrix); - if ((flags_ & Mn::Shaders::PhongGL::Flag::TextureTransformation) && - materialData_->textureMatrix != Mn::Matrix3{}) { - shader_->setTextureMatrix(materialData_->textureMatrix); + if (flags_ & Mn::Shaders::PhongGL::Flag::TextureTransformation) { + shader_->setTextureMatrix(matCache.textureMatrix); } if (flags_ & Mn::Shaders::PhongGL::Flag::AmbientTexture) { - shader_->bindAmbientTexture(*(materialData_->ambientTexture)); + shader_->bindAmbientTexture(*(matCache.ambientTexture)); } if (flags_ & Mn::Shaders::PhongGL::Flag::DiffuseTexture) { - shader_->bindDiffuseTexture(*(materialData_->diffuseTexture)); + shader_->bindDiffuseTexture(*(matCache.diffuseTexture)); } if (flags_ & Mn::Shaders::PhongGL::Flag::SpecularTexture) { - shader_->bindSpecularTexture(*(materialData_->specularTexture)); + shader_->bindSpecularTexture(*(matCache.specularTexture)); } if (flags_ & Mn::Shaders::PhongGL::Flag::NormalTexture) { - shader_->bindNormalTexture(*(materialData_->normalTexture)); + shader_->bindNormalTexture(*(matCache.normalTexture)); } if (flags_ >= Mn::Shaders::PhongGL::Flag::ObjectIdTexture) { - shader_->bindObjectIdTexture(*(materialData_->objectIdTexture)); + shader_->bindObjectIdTexture(*(matCache.objectIdTexture)); + } + + if (skinData_) { + // Gather joint transformations + const auto& skin = skinData_->skinData->skin; + const auto& transformNodes = skinData_->jointIdToTransformNode; + + ESP_CHECK(jointTransformations_.size() == skin->joints().size(), + "Joint transformation count doesn't match bone count."); + + // Undo root node transform so that the model origin matches the root + // articulated object link. + const auto invRootTransform = + skinData_->rootArticulatedObjectNode->absoluteTransformationMatrix() + .inverted(); + + for (std::size_t i = 0; i != jointTransformations_.size(); ++i) { + const auto jointNodeIt = transformNodes.find(skin->joints()[i]); + if (jointNodeIt != transformNodes.end()) { + jointTransformations_[i] = + invRootTransform * + jointNodeIt->second->absoluteTransformationMatrix() * + skin->inverseBindMatrices()[i]; + } else { + // Joint not found, use placeholder matrix. + jointTransformations_[i] = Mn::Matrix4{Mn::Math::IdentityInit}; + } + } + shader_->setJointMatrices(jointTransformations_); } shader_->draw(getMesh()); + + // Reset winding direction + if (normalDet < 0) { + Mn::GL::Renderer::setFrontFace( + Mn::GL::Renderer::FrontFace::CounterClockWise); + } } void GenericDrawable::updateShader() { - Mn::UnsignedInt lightCount = lightSetup_->size(); + const Mn::UnsignedInt lightCount = lightSetup_->size(); + const Mn::UnsignedInt jointCount = + skinData_ ? skinData_->skinData->skin->joints().size() : 0; + const Mn::UnsignedInt perVertexJointCount = + skinData_ ? skinData_->skinData->perVertexJointCount : 0; + + if (skinData_) { + Corrade::Containers::arrayResize( + jointTransformations_, skinData_->skinData->skin->joints().size()); + } if (!shader_ || shader_->lightCount() != lightCount || shader_->flags() != flags_) { @@ -178,15 +284,17 @@ void GenericDrawable::updateShader() { // compatible shader shader_ = shaderManager_.get( - getShaderKey(lightCount, flags_)); + getShaderKey(lightCount, flags_, jointCount)); // if no shader with desired number of lights and flags exists, create one if (!shader_) { shaderManager_.set( shader_.key(), - new Mn::Shaders::PhongGL{Mn::Shaders::PhongGL::Configuration{} - .setFlags(flags_) - .setLightCount(lightCount)}, + new Mn::Shaders::PhongGL{ + Mn::Shaders::PhongGL::Configuration{} + .setFlags(flags_) + .setLightCount(lightCount) + .setJointCount(jointCount, perVertexJointCount)}, Mn::ResourceDataState::Final, Mn::ResourcePolicy::ReferenceCounted); } @@ -197,10 +305,12 @@ void GenericDrawable::updateShader() { Mn::ResourceKey GenericDrawable::getShaderKey( Mn::UnsignedInt lightCount, - Mn::Shaders::PhongGL::Flags flags) const { + Mn::Shaders::PhongGL::Flags flags, + Mn::UnsignedInt jointCount) const { return Corrade::Utility::formatString( SHADER_KEY_TEMPLATE, lightCount, - static_cast(flags)); + static_cast(flags), + jointCount); } } // namespace gfx diff --git a/src/esp/gfx/GenericDrawable.h b/src/esp/gfx/GenericDrawable.h index e7fb109a4e..3483c8ac97 100644 --- a/src/esp/gfx/GenericDrawable.h +++ b/src/esp/gfx/GenericDrawable.h @@ -6,49 +6,105 @@ #define ESP_GFX_GENERICDRAWABLE_H_ #include +#include #include "esp/gfx/Drawable.h" #include "esp/gfx/ShaderManager.h" namespace esp { namespace gfx { +struct InstanceSkinData; class GenericDrawable : public Drawable { public: + /** + * This config holds all the material quantities and attributes from the + * Magnum MaterialData to speed up access in draw. + */ + struct GenericMaterialCache { + Mn::Color4 ambientColor{}; + Mn::Color4 diffuseColor{}; + Mn::Color4 specularColor{}; + float shininess = 80.0f; + Mn::Matrix3 textureMatrix{}; + Mn::GL::Texture2D* ambientTexture = nullptr; + Mn::GL::Texture2D* diffuseTexture = nullptr; + Mn::GL::Texture2D* specularTexture = nullptr; + Mn::GL::Texture2D* normalTexture = nullptr; + Mn::GL::Texture2D* objectIdTexture = nullptr; + + }; // GenericMaterialCache + //! Create a GenericDrawable for the given object using shader and mesh. //! Adds drawable to given group and uses provided texture, and //! color for textured buffer and color shader output respectively - explicit GenericDrawable(scene::SceneNode& node, - Magnum::GL::Mesh* mesh, - Drawable::Flags& meshAttributeFlags, - ShaderManager& shaderManager, - const Magnum::ResourceKey& lightSetupKey, - const Magnum::ResourceKey& materialDataKey, - DrawableGroup* group = nullptr); + explicit GenericDrawable( + scene::SceneNode& node, + Mn::GL::Mesh* mesh, + Drawable::Flags& meshAttributeFlags, + ShaderManager& shaderManager, + const Mn::ResourceKey& lightSetupKey, + const Mn::ResourceKey& materialDataKey, + DrawableGroup* group = nullptr, + const std::shared_ptr& skinData = nullptr); + + void setLightSetup(const Mn::ResourceKey& lightSetupKey) override; + static constexpr const char* SHADER_KEY_TEMPLATE = + "Phong-lights={}-flags={}-joints={}"; - void setLightSetup(const Magnum::ResourceKey& lightSetupKey) override; - static constexpr const char* SHADER_KEY_TEMPLATE = "Phong-lights={}-flags={}"; + /** + * Set or change this drawable's @ref Magnum::Trade::MaterialData values from passed material. + * This is only pertinent for material-equipped drawables. + * @param material + */ + void setMaterialValues( + const Mn::Resource& + material) override { + setMaterialValuesInternal(material); + } + + private: + /** + * @brief Internal implementation of material setting, so that it can be + * called from constructor without virtual dispatch issues + */ + void setMaterialValuesInternal( + const Mn::Resource& + material); protected: - void draw(const Magnum::Matrix4& transformationMatrix, - Magnum::SceneGraph::Camera3D& camera) override; + void draw(const Mn::Matrix4& transformationMatrix, + Mn::SceneGraph::Camera3D& camera) override; void updateShader(); - void updateShaderLightingParameters( - const Magnum::Matrix4& transformationMatrix, - Magnum::SceneGraph::Camera3D& camera); + void updateShaderLightingParameters(const Mn::Matrix4& transformationMatrix, + Mn::SceneGraph::Camera3D& camera); - Magnum::ResourceKey getShaderKey(Magnum::UnsignedInt lightCount, - Magnum::Shaders::PhongGL::Flags flags) const; + Mn::ResourceKey getShaderKey(Mn::UnsignedInt lightCount, + Mn::Shaders::PhongGL::Flags flags, + Mn::UnsignedInt jointCount) const; // shader parameters + + Mn::Shaders::PhongGL::Flags flags_; ShaderManager& shaderManager_; - Magnum::Resource - shader_; - Magnum::Resource materialData_; - Magnum::Resource lightSetup_; + Mn::Resource shader_; + Mn::Resource lightSetup_; + std::shared_ptr skinData_; + Cr::Containers::Array jointTransformations_; - Magnum::Shaders::PhongGL::Flags flags_; + /** + * Local cache of material quantities to speed up access in draw + */ + GenericMaterialCache matCache{}; + /** + * Creation attributes of this drawable + */ + const gfx::Drawable::Flags meshAttributeFlags_; + /** + * Material to use to render this Phong drawawble + */ + Mn::Resource materialData_; }; } // namespace gfx diff --git a/src/esp/gfx/LightSetup.cpp b/src/esp/gfx/LightSetup.cpp index 9030cd769a..ff6b1a30b8 100644 --- a/src/esp/gfx/LightSetup.cpp +++ b/src/esp/gfx/LightSetup.cpp @@ -76,15 +76,12 @@ LightSetup getLightsAtBoxCorners(const Magnum::Range3D& box, {{box.backBottomLeft(), w}, lightColor}, {{box.backBottomRight(), w}, lightColor}}; } - LightSetup getDefaultLights() { - return LightSetup{ - {{0.0, 0.5, 1.0, 0.0}, {2.4, 2.4, 2.4}}, // forward - {{0.0, 0.5, -1.0, 0.0}, {2.4, 2.4, 2.4}}, // backward - {{-1.0, 1.0, 0.0, 0.0}, {2.0, 2.0, 2.0}}, // top left - {{1.0, 1.0, 1.0, 0.0}, {2.0, 2.0, 2.0}}, // top right - {{0.0, -1.0, 0.0, 0.0}, {0.64, 0.64, 0.64}} // bottom - }; + return LightSetup{{{0.0, -0.5, -1.0, 0.0}, {2.4, 2.4, 2.4}}, // forward + {{0.0, -0.5, 1.0, 0.0}, {2.4, 2.4, 2.4}}, // backward + {{1.0, -1.0, 0.0, 0.0}, {2.0, 2.0, 2.0}}, // top left + {{-1.0, -1.0, -1.0, 0.0}, {2.0, 2.0, 2.0}}, // top right + {{0.0, 1.0, 0.0, 0.0}, {0.64, 0.64, 0.64}}}; // bottom } Magnum::Color3 getAmbientLightColor(const LightSetup& lightSetup) { diff --git a/src/esp/gfx/LightSetup.h b/src/esp/gfx/LightSetup.h index 45b68c06bb..e740ed7cfe 100644 --- a/src/esp/gfx/LightSetup.h +++ b/src/esp/gfx/LightSetup.h @@ -91,7 +91,7 @@ LightSetup getLightsAtBoxCorners( LightSetup getDefaultLights(); /** - * @brief Get get a single, combined ambient light color for use with the Phong + * @brief Get a single, combined ambient light color for use with the Phong * lighting model. */ Magnum::Color3 getAmbientLightColor(const LightSetup& lightSetup); diff --git a/src/esp/gfx/MaterialData.h b/src/esp/gfx/MaterialData.h deleted file mode 100644 index ae23049867..0000000000 --- a/src/esp/gfx/MaterialData.h +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and its affiliates. -// This source code is licensed under the MIT license found in the -// LICENSE file in the root directory of this source tree. - -#ifndef ESP_GFX_MATERIALDATA_H_ -#define ESP_GFX_MATERIALDATA_H_ - -#include -#include -#include -#include - -#include "esp/core/Esp.h" - -namespace esp { -namespace gfx { - -enum class MaterialDataType { - /** @brief default, does not represent any material */ - None = 0, - /** @brief phong material data */ - Phong = 1, - /** @brief Physically based rendering (PBR) material data */ - Pbr = 2, -}; - -struct MaterialData { - explicit MaterialData(MaterialDataType _type = MaterialDataType::None) - : type(_type){}; - - MaterialDataType type{MaterialDataType::None}; - - bool perVertexObjectId = false; - - // This denotes a material with texture-based annotations - bool textureObjectId = false; - - // This references the texture of ObjectID values - Magnum::GL::Texture2D* objectIdTexture = nullptr; - - // construct it using the default constructor. NO initial values, such as - // identity matrix - Magnum::Matrix3 textureMatrix; - - bool doubleSided = false; - - // Shader type specified for this material upon load/creation - int shaderTypeSpec = -1; -}; - -struct PhongMaterialData : public MaterialData { - PhongMaterialData() : MaterialData(MaterialDataType::Phong){}; - Magnum::Float shininess = 80.f; - Magnum::Color4 ambientColor{0.1}; - // NOTE: This multiplication is a hack to roughly balance the Phong and PBR - // light intensity reactions. - Magnum::Color4 diffuseColor{0.7 * 0.175}; - Magnum::Color4 specularColor{0.2 * 0.175}; - Magnum::GL::Texture2D *ambientTexture = nullptr, *diffuseTexture = nullptr, - *specularTexture = nullptr, *normalTexture = nullptr; - - ESP_SMART_POINTERS(PhongMaterialData) -}; - -struct PbrMaterialData : public MaterialData { - PbrMaterialData() : MaterialData(MaterialDataType::Pbr){}; - - // TODO: Migrate the the Magnum built-in PBR material - // material with default values: - // when both the material property and the texture (e.g., roughness and - // roughness texture) are NOT presented, use these default values; - // accoding to glTF 2.0: - // "If a texture is not given, all respective texture components within this - // material model are assumed to have a value of 1.0. " - Magnum::Color4 baseColor{1.0f}; - Magnum::Float roughness = 1.0f; - Magnum::Float metallic = 1.0f; - Magnum::Color3 emissiveColor{0.0f}; - Magnum::Float normalTextureScale = 1.0f; - - Magnum::GL::Texture2D* baseColorTexture = nullptr; - Magnum::GL::Texture2D* normalTexture = nullptr; - Magnum::GL::Texture2D* emissiveTexture = nullptr; - - // Question: - // Why not just specify a MetallicRoughnessTexture as the glTF 2.0 spec - // suggested? - // - // Answer: - // We need a mechanism to identify the case where either metallic or roughness - // texture exists, but not both. - - Magnum::GL::Texture2D* metallicTexture = nullptr; - Magnum::GL::Texture2D* roughnessTexture = nullptr; - // TODO: - // AO texture - ESP_SMART_POINTERS(PbrMaterialData) -}; - -// TODO: Ptex material data - -} // namespace gfx -} // namespace esp - -#endif // ESP_GFX_MATERIALDATA_H_ diff --git a/src/esp/gfx/MaterialUtil.cpp b/src/esp/gfx/MaterialUtil.cpp deleted file mode 100644 index d9e6d1b550..0000000000 --- a/src/esp/gfx/MaterialUtil.cpp +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and its affiliates. -// This source code is licensed under the MIT license found in the -// LICENSE file in the root directory of this source tree. - -#include "MaterialUtil.h" - -#include -#include -#include -#include -#include - -namespace Mn = Magnum; -namespace Cr = Corrade; - -namespace esp { -namespace gfx { - -using Mn::Trade::MaterialAttribute; - -namespace { - -/** - * @brief Make sure we do not add an attribute that already exists in source - * material - doing this will cause an assertion. - * @tparam the type of the value to be added - * @param material Source material for new attributes array - * @param newAttributes The array of attributes to be used to build a new - * Mn::Trade::MaterialData. - * @param matAttr The Mn::Trade::MaterialAttribute tag describing the attribute - * to add - * @param value A reference to the value to add. - */ -template -void appendIfNotPresent( - const Mn::Trade::MaterialData& material, - Cr::Containers::Array& newAttributes, - const MaterialAttribute& matAttr, - const T& value) { - if (!material.hasAttribute(matAttr)) { - arrayAppend(newAttributes, {matAttr, value}); - } -} - -} // namespace - -Mn::Trade::MaterialData createUniversalMaterial( - const Mn::Trade::MaterialData& origMaterialData) { - // create a material, based on the passed material, that will have reasonable - // attributes to support any possible shader type. should only be called if - // we are not using the Material's natively specified shaderType - - // NOLINTNEXTLINE(google-build-using-namespace) - using namespace Mn::Math::Literals; - - // get source attributes from original material - Cr::Containers::Array newAttributes; - arrayAppend(newAttributes, origMaterialData.attributeData()); - - const auto origMatTypes = origMaterialData.types(); - // add appropriate attributes based on what is missing - // flat material already recognizes Phong and pbr, so don't have to do - // anything - - // multiplicative magic number for scaling from PBR to phong, - // hacky method to attempt to balance Phong and PBR light intensity reactions - const float magicPhongScaling = 1.5f; - - // whether the MaterialAttribute::TextureMatrix has been set already - bool setTexMatrix = false; - if (!(origMatTypes & Mn::Trade::MaterialType::Phong)) { - // add appropriate values for expected attributes to support Phong from - // PbrMetallicRoughnessMaterialData - - const auto& pbrMaterial = - origMaterialData.as(); - - ///////////////// - // calculate Phong values from PBR material values - //////////////// - - // derive ambient color from pbr baseColor - const Mn::Color4 ambientColor = pbrMaterial.baseColor(); - - // If there's a roughness texture, we have no way to use it here. The safest - // fallback is to assume roughness == 1, thus producing no spec highlights. - // If pbrMaterial does not have a roughness value, it returns a 1 by - // default. - const float roughness = - pbrMaterial.hasRoughnessTexture() ? 1.0f : pbrMaterial.roughness(); - - // If there's a metalness texture, we have no way to use it here. - // The safest fallback is to assume non-metal. - const float metalness = - pbrMaterial.hasMetalnessTexture() ? 0.0f : pbrMaterial.metalness(); - - // Heuristic to map roughness to spec power. - // Higher exponent makes the spec highlight larger (lower power) - // example calc : - // https://www.wolframalpha.com/input/?i=1.1+%2B+%281+-+x%29%5E4.5+*+180.0+for+x+from+0+to+1 - // lower power for metal - const float maxShininess = Mn::Math::lerp(250.0f, 120.0f, metalness); - const float shininess = - 1.1f + Mn::Math::pow(1.0f - roughness, 4.5f) * maxShininess; - - // increase spec intensity for metal - // example calc : - // https://www.wolframalpha.com/input/?i=1.0+%2B+10*%28x%5E1.7%29++from+0+to+1 - const float specIntensityScale = - (metalness > 0.0f - ? (metalness < 1.0f - ? (1.0f + 10.0f * Mn::Math::pow(metalness, 1.7f)) - : 11.0f) - : 1.0f); - // Heuristic to map roughness to spec intensity. - // higher exponent decreases intensity. - // example calc : - // https://www.wolframalpha.com/input/?i=1.4+*+%281-x%29%5E2.5++from+0+to+1 - const float specIntensity = - Mn::Math::pow(1.0f - roughness, 2.5f) * 1.4f * specIntensityScale; - - // NOTE: The magic-number multiplication at the end is a hack to - // roughly balance the Phong and PBR light intensity reactions - const Mn::Color4 diffuseColor = ambientColor * magicPhongScaling; - - // Set spec base color to white or material base color, depending on - // metalness. - const Mn::Color4 specBaseColor = - (metalness > 0.0f - ? (metalness < 1.0f - ? Mn::Math::lerp(0xffffffff_rgbaf, ambientColor, - Mn::Math::pow(metalness, 0.5f)) - : ambientColor) - : 0xffffffff_rgbaf); - // NOTE: The magic-number multiplication at the end is a hack to - // roughly balance the Phong and PBR light intensity reactions - const Mn::Color4 specColor = - specBaseColor * specIntensity * magicPhongScaling; - - ///////////////// - // set Phong attributes appropriately from precalculated value - //////////////// - // normal mapping is already present in copied array if present in - // original material. - - // make sure not to re-add values that already exist in array, or - // new color creation will assert. - appendIfNotPresent(origMaterialData, newAttributes, - MaterialAttribute::Shininess, shininess); - appendIfNotPresent(origMaterialData, newAttributes, - MaterialAttribute::AmbientColor, ambientColor); - appendIfNotPresent(origMaterialData, newAttributes, - MaterialAttribute::DiffuseColor, diffuseColor); - appendIfNotPresent(origMaterialData, newAttributes, - MaterialAttribute::SpecularColor, specColor); - - // texture transforms, if there's none the returned matrix is an - // identity only copy if we don't already have TextureMatrix - // attribute (if original pbrMaterial does not have that specific - // matrix) - if (!setTexMatrix && - (!pbrMaterial.hasAttribute(MaterialAttribute::TextureMatrix))) { - arrayAppend(newAttributes, {MaterialAttribute::TextureMatrix, - pbrMaterial.commonTextureMatrix()}); - setTexMatrix = true; - } - - if (pbrMaterial.hasAttribute(MaterialAttribute::BaseColorTexture)) { - // only provide texture indices if BaseColorTexture attribute - // exists - const Mn::UnsignedInt BCTexture = pbrMaterial.baseColorTexture(); - appendIfNotPresent(origMaterialData, newAttributes, - MaterialAttribute::AmbientTexture, BCTexture); - appendIfNotPresent(origMaterialData, newAttributes, - MaterialAttribute::DiffuseTexture, BCTexture); - if (metalness >= 0.5) { - appendIfNotPresent(origMaterialData, newAttributes, - MaterialAttribute::SpecularTexture, BCTexture); - } - } - } // if no phong material support exists in material - - if (!(origMatTypes & Mn::Trade::MaterialType::PbrMetallicRoughness)) { - // add appropriate values for expected attributes for PbrMetallicRoughness - // derived from Phong attributes - const auto& phongMaterial = - origMaterialData.as(); - - ///////////////// - // calculate PBR values from Phong material values - //////////////// - - // derive base color from Phong diffuse or ambient color, depending on which - // is present. set to white if neither is present - const Mn::Color4 baseColor = phongMaterial.diffuseColor(); - - // Experimental metalness heuristic using saturation of spec color - // to derive approximation of metalness - const Mn::Color4 specColor = phongMaterial.specularColor(); - - // if specColor alpha == 0 then no metalness - float metalness = 0.0f; - // otherwise, this hacky heuristic will derive a value for metalness based - // on how non-grayscale the specular color is (HSV Saturation). - if (specColor.a() != 0.0f) { - metalness = specColor.saturation(); - } - - ///////////////// - // set PbrMetallicRoughness attributes appropriately from precalculated - // values - //////////////// - - // normal mapping is already present in copied array if present in - // original material. - appendIfNotPresent(origMaterialData, newAttributes, - MaterialAttribute::BaseColor, baseColor); - appendIfNotPresent(origMaterialData, newAttributes, - MaterialAttribute::Metalness, metalness); - - // if diffuse texture is present, use as base color texture in pbr. - if (phongMaterial.hasAttribute(MaterialAttribute::DiffuseTexture)) { - uint32_t bcTextureVal = phongMaterial.diffuseTexture(); - appendIfNotPresent(origMaterialData, newAttributes, - MaterialAttribute::BaseColorTexture, bcTextureVal); - } - // texture transforms, if there's none the returned matrix is an - // identity Only copy if we don't already have TextureMatrix attribute - // (if original phongMaterial does not have that specific attribute) - if (!setTexMatrix && - (!phongMaterial.hasAttribute(MaterialAttribute::TextureMatrix))) { - arrayAppend(newAttributes, {MaterialAttribute::TextureMatrix, - phongMaterial.commonTextureMatrix()}); - // setTexMatrix = true; - } - - // base texture - - } // if no PbrMetallicRoughness material support exists in material - - // build flags to support all materials - constexpr auto flags = Mn::Trade::MaterialType::Flat | - Mn::Trade::MaterialType::Phong | - Mn::Trade::MaterialType::PbrMetallicRoughness; - - // create new material from attributes array - Mn::Trade::MaterialData newMaterialData{flags, std::move(newAttributes)}; - - return newMaterialData; -} // ResourceManager::createUniversalMaterial - -} // namespace gfx -} // namespace esp diff --git a/src/esp/gfx/MaterialUtil.h b/src/esp/gfx/MaterialUtil.h deleted file mode 100644 index b91e1d9d17..0000000000 --- a/src/esp/gfx/MaterialUtil.h +++ /dev/null @@ -1,33 +0,0 @@ - -// Copyright (c) Meta Platforms, Inc. and its affiliates. -// This source code is licensed under the MIT license found in the -// LICENSE file in the root directory of this source tree. - -#ifndef ESP_GFX_MATERIALUTIL_H_ -#define ESP_GFX_MATERIALUTIL_H_ - -#include -#include "esp/gfx/MaterialData.h" - -namespace Mn = Magnum; - -namespace esp { -namespace gfx { - -/** - * @brief This function will take an existing Mn::Trade::MaterialData and add - * the missing attributes for the types it does not support, so that it will - * have attributes for all habitat-supported types. This should only be called - * if the user has specified a desired shader type that the material does not - * natively support. - * @param origMaterialData The original material from the importer - * @return The new material with attribute support for all supported shader - * types. - */ -Mn::Trade::MaterialData createUniversalMaterial( - const Mn::Trade::MaterialData& origMaterialData); - -} // namespace gfx -} // namespace esp - -#endif // ESP_GFX_MATERIALUTIL_H_ diff --git a/src/esp/gfx/PbrDrawable.cpp b/src/esp/gfx/PbrDrawable.cpp index 69743637d9..a07a609b2a 100644 --- a/src/esp/gfx/PbrDrawable.cpp +++ b/src/esp/gfx/PbrDrawable.cpp @@ -6,6 +6,9 @@ #include #include +#include +#include + #include namespace Mn = Magnum; @@ -24,61 +27,122 @@ PbrDrawable::PbrDrawable(scene::SceneNode& node, : Drawable{node, mesh, DrawableType::Pbr, group}, shaderManager_{shaderManager}, lightSetup_{shaderManager.get(lightSetupKey)}, - materialData_{ - shaderManager.get(materialDataKey)}, + meshAttributeFlags_{meshAttributeFlags}, pbrIbl_(pbrIbl) { - if (materialData_->metallicTexture && materialData_->roughnessTexture) { + setMaterialValuesInternal( + shaderManager.get(materialDataKey)); + + if (pbrIbl_) { + flags_ |= PbrShader::Flag::ImageBasedLighting; + } + + // Defer the shader initialization because at this point, the lightSetup may + // not be done in the Simulator. Simulator itself is currently under + // construction in this case. + // updateShader().updateShaderLightParameters(); +} + +void PbrDrawable::setMaterialValuesInternal( + const Mn::Resource& + material) { + materialData_ = material; + + const auto& tmpMaterialData = + materialData_->as(); + flags_ = PbrShader::Flag::ObjectId; + + matCache.baseColor = tmpMaterialData.baseColor(); + matCache.roughness = tmpMaterialData.roughness(); + matCache.metalness = tmpMaterialData.metalness(); + matCache.emissiveColor = tmpMaterialData.emissiveColor(); + + if (materialData_->hasAttribute("metallicTexturePointer") && + materialData_->hasAttribute("roughnessTexturePointer")) { CORRADE_ASSERT( - materialData_->metallicTexture == materialData_->roughnessTexture, - "PbrDrawable::PbrDrawable(): if both the metallic and roughness " + materialData_->attribute( + "metallicTexturePointer") == + materialData_->attribute( + "roughnessTexturePointer"), + "PbrDrawable::setMaterialValuesInternal(): if both the metallic and " + "roughness " "texture exist, they must be packed in the same texture based on glTF " "2.0 Spec.", ); } - - flags_ = PbrShader::Flag::ObjectId; - if (materialData_->textureMatrix != Mn::Matrix3{}) { + if (tmpMaterialData.commonTextureMatrix() != Mn::Matrix3{}) { flags_ |= PbrShader::Flag::TextureTransformation; + matCache.textureMatrix = tmpMaterialData.commonTextureMatrix(); } - if (materialData_->baseColorTexture) { + if (materialData_->hasAttribute("baseColorTexturePointer")) { flags_ |= PbrShader::Flag::BaseColorTexture; + matCache.baseColorTexture = + materialData_->attribute("baseColorTexturePointer"); } - if (materialData_->roughnessTexture) { + + matCache.hasAnyMetallicRoughnessTexture = false; + matCache.useMetallicRoughnessTexture = nullptr; + // noneRoughnessMetallic takes precedence, but currently all are + // treated the same way + if (materialData_->hasAttribute("noneRoughnessMetallicTexturePointer")) { + flags_ |= PbrShader::Flag::NoneRoughnessMetallicTexture; + matCache.noneRoughnessMetallicTexture = + materialData_->attribute( + "noneRoughnessMetallicTexturePointer"); + matCache.hasAnyMetallicRoughnessTexture = true; + matCache.useMetallicRoughnessTexture = + matCache.noneRoughnessMetallicTexture; + } + if (materialData_->hasAttribute("roughnessTexturePointer")) { flags_ |= PbrShader::Flag::RoughnessTexture; + matCache.roughnessTexture = + materialData_->attribute("roughnessTexturePointer"); + if (!matCache.hasAnyMetallicRoughnessTexture) { + matCache.useMetallicRoughnessTexture = matCache.roughnessTexture; + } + matCache.hasAnyMetallicRoughnessTexture = true; } - if (materialData_->metallicTexture) { + if (materialData_->hasAttribute("metallicTexturePointer")) { flags_ |= PbrShader::Flag::MetallicTexture; + matCache.metallicTexture = + materialData_->attribute("metallicTexturePointer"); + + if (!matCache.hasAnyMetallicRoughnessTexture) { + matCache.useMetallicRoughnessTexture = matCache.metallicTexture; + } + matCache.hasAnyMetallicRoughnessTexture = true; } - if (materialData_->normalTexture) { + + CORRADE_ASSERT(((matCache.useMetallicRoughnessTexture != nullptr) == + matCache.hasAnyMetallicRoughnessTexture), + "PbrDrawable::setMaterialValuesInternal(): Error assigning " + "proper Metallic/Roughness texture pointers - either a " + "texture is expected but not present or vice versa.", ); + + if (materialData_->hasAttribute("normalTexturePointer")) { flags_ |= PbrShader::Flag::NormalTexture; - if (meshAttributeFlags & gfx::Drawable::Flag::HasTangent) { + matCache.normalTexture = + materialData_->attribute("normalTexturePointer"); + if (meshAttributeFlags_ & gfx::Drawable::Flag::HasTangent) { flags_ |= PbrShader::Flag::PrecomputedTangent; } - if (materialData_->normalTextureScale != 1.0f) { + if (tmpMaterialData.normalTextureScale() != 1.0f) { flags_ |= PbrShader::Flag::NormalTextureScale; - CORRADE_ASSERT(materialData_->normalTextureScale > 0.0f, + CORRADE_ASSERT(tmpMaterialData.normalTextureScale() > 0.0f, "PbrDrawable::PbrDrawable(): the normal texture scale " "must be positive.", ); } } - if (materialData_->emissiveTexture) { + if (materialData_->hasAttribute("emissiveTexturePointer")) { flags_ |= PbrShader::Flag::EmissiveTexture; + matCache.emissiveTexture = + materialData_->attribute("emissiveTexturePointer"); } - if (materialData_->perVertexObjectId) { - // TODO: may be supported in the future + if (materialData_->attribute("hasPerVertexObjectId")) { + flags_ |= PbrShader::Flag::InstancedObjectId; } - if (materialData_->doubleSided) { + if (materialData_->isDoubleSided()) { flags_ |= PbrShader::Flag::DoubleSided; } - - if (pbrIbl_) { - flags_ |= PbrShader::Flag::ImageBasedLighting; - } - - // Defer the shader initialization because at this point, the lightSetup may - // not be done in the Simulator. Simulator itself is currently under - // construction in this case. - // updateShader().updateShaderLightParameters(); -} +} // PbrDrawable::setMaterialValuesInternal void PbrDrawable::setLightSetup(const Mn::ResourceKey& lightSetupKey) { lightSetup_ = shaderManager_.get(lightSetupKey); @@ -95,17 +159,17 @@ void PbrDrawable::draw(const Mn::Matrix4& transformationMatrix, // ABOUT PbrShader::Flag::DoubleSided: // - // "Specifies whether the material is double sided. When this value is false, - // back-face culling is enabled. When this value is true, back-face culling is - // disabled and double sided lighting is enabled. The back-face must have its - // normals reversed before the lighting equation is evaluated." - // See here: + // "Specifies whether the material is double sided. When this value is + // false, back-face culling is enabled. When this value is true, back-face + // culling is disabled and double sided lighting is enabled. The back-face + // must have its normals reversed before the lighting equation is + // evaluated." See here: // https://github.com/KhronosGroup/glTF/blob/master/specification/2.0/schema/material.schema.json // HOWEVER, WE CANNOT DISABLE BACK FACE CULLING (that is why the following - // code is commented out) since it causes lighting artifacts ("dashed lines") - // on hard edges. (maybe due to potential numerical issues? we do not know - // yet.) + // code is commented out) since it causes lighting artifacts ("dashed + // lines") on hard edges. (maybe due to potential numerical issues? we do + // not know yet.) /* if ((flags_ & PbrShader::Flag::DoubleSided) && glIsEnabled(GL_CULL_FACE)) { Mn::GL::Renderer::disable(Mn::GL::Renderer::Feature::FaceCulling); @@ -114,61 +178,68 @@ void PbrDrawable::draw(const Mn::Matrix4& transformationMatrix, Mn::Matrix4 modelMatrix = camera.cameraMatrix().inverted() * transformationMatrix; + Mn::Matrix3x3 rotScale = modelMatrix.rotationScaling(); + // Find determinant to calculate backface culling winding dir + const float normalDet = rotScale.determinant(); + // Normal matrix is calculated as `m.inverted().transposed()`, and + // `m.inverted()` is the same as + // `m.comatrix().transposed()/m.determinant()`. We need the determinant to + // figure out the winding direction as well, thus we calculate it separately + // and then do + // `(m.comatrix().transposed()/determinant).transposed()`, which is the same + // as `m.comatrix()/determinant`. + Mn::Matrix3x3 normalMatrix = rotScale.comatrix() / normalDet; + + // Flip winding direction to correct handle backface culling + if (normalDet < 0) { + Mn::GL::Renderer::setFrontFace(Mn::GL::Renderer::FrontFace::ClockWise); + } + (*shader_) - // e.g., semantic mesh has its own per vertex annotation, which has been - // uploaded to GPU so simply pass 0 to the uniform "objectId" in the - // fragment shader - .setObjectId( - static_cast(camera).useDrawableIds() - ? drawableId_ - : (materialData_->perVertexObjectId ? 0 : node_.getSemanticId())) + // e.g., semantic mesh has its own per vertex annotation, which has + // been uploaded to GPU so simply pass 0 to the uniform "objectId" in + // the fragment shader + .setObjectId(static_cast(camera).useDrawableIds() + ? drawableId_ + : ((flags_ & PbrShader::Flag::InstancedObjectId) == + PbrShader::Flag::InstancedObjectId + ? 0 + : node_.getSemanticId())) .setProjectionMatrix(camera.projectionMatrix()) .setViewMatrix(camera.cameraMatrix()) .setModelMatrix(modelMatrix) // NOT modelview matrix! - .setNormalMatrix(modelMatrix.normalMatrix()) + .setNormalMatrix(normalMatrix) .setCameraWorldPosition( camera.object().absoluteTransformationMatrix().translation()) - .setBaseColor(materialData_->baseColor) - .setRoughness(materialData_->roughness) - .setMetallic(materialData_->metallic) - .setEmissiveColor(materialData_->emissiveColor); + .setBaseColor(matCache.baseColor) + .setRoughness(matCache.roughness) + .setMetallic(matCache.metalness) + .setEmissiveColor(matCache.emissiveColor); // TODO: // IN PbrShader class, we set the resonable defaults for the // PbrShader::PbrEquationScales. Here we need a smart way to reset it // just in case user would like to do so during the run-time. - if ((flags_ & PbrShader::Flag::BaseColorTexture) && - (materialData_->baseColorTexture != nullptr)) { - shader_->bindBaseColorTexture(*materialData_->baseColorTexture); + if (flags_ & PbrShader::Flag::BaseColorTexture) { + shader_->bindBaseColorTexture(*matCache.baseColorTexture); } - if (flags_ & - (PbrShader::Flag::RoughnessTexture | PbrShader::Flag::MetallicTexture)) { - Magnum::GL::Texture2D* metallicRoughnessTexture = - materialData_->roughnessTexture; - if (!metallicRoughnessTexture) { - metallicRoughnessTexture = materialData_->metallicTexture; - } - CORRADE_ASSERT(metallicRoughnessTexture, - "PbrDrawable::draw(): texture pointer cannot be nullptr if " - "RoughnessTexture or MetallicTexture is enabled.", ); - shader_->bindMetallicRoughnessTexture(*metallicRoughnessTexture); + if (matCache.hasAnyMetallicRoughnessTexture) { + shader_->bindMetallicRoughnessTexture( + *matCache.useMetallicRoughnessTexture); } - if ((flags_ & PbrShader::Flag::NormalTexture) && - (materialData_->normalTexture != nullptr)) { - shader_->bindNormalTexture(*materialData_->normalTexture); + if (flags_ & PbrShader::Flag::NormalTexture) { + shader_->bindNormalTexture(*matCache.normalTexture); } - if ((flags_ & PbrShader::Flag::EmissiveTexture) && - (materialData_->emissiveTexture != nullptr)) { - shader_->bindEmissiveTexture(*materialData_->emissiveTexture); + if (flags_ & PbrShader::Flag::EmissiveTexture) { + shader_->bindEmissiveTexture(*matCache.emissiveTexture); } - if ((flags_ & PbrShader::Flag::TextureTransformation) && - (materialData_->textureMatrix != Mn::Matrix3{})) { - shader_->setTextureMatrix(materialData_->textureMatrix); + if (flags_ & PbrShader::Flag::TextureTransformation) { + shader_->setTextureMatrix(matCache.textureMatrix); } // setup image based lighting for the shader @@ -205,6 +276,12 @@ void PbrDrawable::draw(const Mn::Matrix4& transformationMatrix, shader_->draw(getMesh()); + // Reset winding direction + if (normalDet < 0) { + Mn::GL::Renderer::setFrontFace( + Mn::GL::Renderer::FrontFace::CounterClockWise); + } + // WE stopped supporting doubleSided material due to lighting artifacts on // hard edges. See comments at the beginning of this function. /* @@ -212,7 +289,7 @@ void PbrDrawable::draw(const Mn::Matrix4& transformationMatrix, Mn::GL::Renderer::enable(Mn::GL::Renderer::Feature::FaceCulling); } */ -} +} // namespace gfx Mn::ResourceKey PbrDrawable::getShaderKey(Mn::UnsignedInt lightCount, PbrShader::Flags flags) const { @@ -262,8 +339,8 @@ PbrDrawable& PbrDrawable::updateShaderLightParameters() { // update light direction (or position) in *world* space to the shader PbrDrawable& PbrDrawable::updateShaderLightDirectionParameters( - const Magnum::Matrix4& transformationMatrix, - Magnum::SceneGraph::Camera3D& camera) { + const Mn::Matrix4& transformationMatrix, + Mn::SceneGraph::Camera3D& camera) { std::vector lightPositions; lightPositions.reserve(lightSetup_->size()); @@ -272,6 +349,9 @@ PbrDrawable& PbrDrawable::updateShaderLightDirectionParameters( const auto& lightInfo = (*lightSetup_)[iLight]; Mn::Vector4 pos = getLightPositionRelativeToWorld( lightInfo, transformationMatrix, cameraMatrix); + // flip directional lights to faciliate faster, non-forking calc in + // shader. Leave non-directional lights unchanged + pos *= (pos[3] * 2) - 1; lightPositions.emplace_back(pos); } diff --git a/src/esp/gfx/PbrDrawable.h b/src/esp/gfx/PbrDrawable.h index 374faf5d40..a1d6974f70 100644 --- a/src/esp/gfx/PbrDrawable.h +++ b/src/esp/gfx/PbrDrawable.h @@ -17,17 +17,46 @@ namespace gfx { class PbrDrawable : public Drawable { public: + /** + * This cache holds all the material quantities and attributes from the + * Magnum MaterialData to speed up access in draw. + */ + struct PBRMaterialCache { + Mn::Color4 baseColor{1.0f}; + float roughness = 1.0f; + float metalness = 1.0f; + Mn::Color3 emissiveColor{}; + Mn::Matrix3 textureMatrix{}; + + Mn::GL::Texture2D* baseColorTexture = nullptr; + + bool hasAnyMetallicRoughnessTexture = false; + + Mn::GL::Texture2D* useMetallicRoughnessTexture = nullptr; + /** + * Currently we only support a single NoneMetalnessRoughness texture for + * both metalness and roughness. Separate textures will be stored, but only + * the useMetallicRoughnessTexture should be sent to the shader + */ + Mn::GL::Texture2D* noneRoughnessMetallicTexture = nullptr; + Mn::GL::Texture2D* roughnessTexture = nullptr; + Mn::GL::Texture2D* metallicTexture = nullptr; + + Mn::GL::Texture2D* normalTexture = nullptr; + Mn::GL::Texture2D* emissiveTexture = nullptr; + }; + /** * @brief Constructor, to create a PbrDrawable for the given object using * shader and mesh. Adds drawable to given group and uses provided texture, * and color for textured buffer and color shader output respectively */ explicit PbrDrawable(scene::SceneNode& node, - Magnum::GL::Mesh* mesh, + Mn::GL::Mesh* mesh, gfx::Drawable::Flags& meshAttributeFlags, ShaderManager& shaderManager, - const Magnum::ResourceKey& lightSetupKey, - const Magnum::ResourceKey& materialDataKey, + const Mn::ResourceKey& lightSetupKey, + const Mn::ResourceKey& materialDataKey, DrawableGroup* group = nullptr, PbrImageBasedLighting* pbrIbl = nullptr); @@ -35,7 +64,7 @@ class PbrDrawable : public Drawable { * @brief Set the light info * @param lightSetupKey the key value for the light resource */ - void setLightSetup(const Magnum::ResourceKey& lightSetupKey) override; + void setLightSetup(const Mn::ResourceKey& lightSetupKey) override; /** * @brief Set the shadow map info @@ -49,6 +78,26 @@ class PbrDrawable : public Drawable { static constexpr const char* SHADER_KEY_TEMPLATE = "PBR-lights={}-flags={}"; + /** + * Set or change this drawable's @ref Magnum::Trade::MaterialData values from passed material. + * This is only pertinent for material-equipped drawables. + * @param material + */ + void setMaterialValues( + const Mn::Resource& + material) override { + setMaterialValuesInternal(material); + } + + private: + /** + * @brief Internal implementation of material setting, so that it can be + * called from constructor without virtual dispatch issues + */ + void setMaterialValuesInternal( + const Mn::Resource& + material); + protected: /** * @brief overload draw function, see here for more details: @@ -57,8 +106,8 @@ class PbrDrawable : public Drawable { * the drawable is attached) relative to camera * @param camera the camera that views and renders the world */ - void draw(const Magnum::Matrix4& transformationMatrix, - Magnum::SceneGraph::Camera3D& camera) override; + void draw(const Mn::Matrix4& transformationMatrix, + Mn::SceneGraph::Camera3D& camera) override; /** * @brief Update the shader so it can correcly handle the current material, @@ -82,24 +131,36 @@ class PbrDrawable : public Drawable { * @return Reference to self (for method chaining) */ PbrDrawable& updateShaderLightDirectionParameters( - const Magnum::Matrix4& transformationMatrix, - Magnum::SceneGraph::Camera3D& camera); + const Mn::Matrix4& transformationMatrix, + Mn::SceneGraph::Camera3D& camera); /** * @brief get the key for the shader * @param lightCount the number of the lights; * @param flags flags that defines the shader features */ - Magnum::ResourceKey getShaderKey(Magnum::UnsignedInt lightCount, - PbrShader::Flags flags) const; + Mn::ResourceKey getShaderKey(Mn::UnsignedInt lightCount, + PbrShader::Flags flags) const; // shader parameters PbrShader::Flags flags_; ShaderManager& shaderManager_; - Magnum::Resource shader_; - Magnum::Resource materialData_; - Magnum::Resource lightSetup_; + Mn::Resource shader_; + Mn::Resource lightSetup_; PbrImageBasedLighting* pbrIbl_ = nullptr; + + /** + * Local cache of material quantities to speed up access in draw + */ + PBRMaterialCache matCache{}; + /** + * Creation attributes of this drawable + */ + const gfx::Drawable::Flags meshAttributeFlags_; + /** + * Material to use to render this PBR drawawble + */ + Mn::Resource materialData_; ShadowMapManager* shadowMapManger_ = nullptr; ShadowMapKeys* shadowMapKeys_ = nullptr; }; diff --git a/src/esp/gfx/PbrShader.cpp b/src/esp/gfx/PbrShader.cpp index f4ab58268e..03d776d990 100644 --- a/src/esp/gfx/PbrShader.cpp +++ b/src/esp/gfx/PbrShader.cpp @@ -75,10 +75,10 @@ PbrShader::PbrShader(Flags originalFlags, unsigned int lightCount) "#define ATTRIBUTE_LOCATION_TANGENT4 {}\n", Tangent4::Location); } // TODO: Occlusion texture to be added. - const bool isTextured = - bool(flags_ & (Flag::BaseColorTexture | Flag::RoughnessTexture | - Flag::MetallicTexture | Flag::NormalTexture | - Flag::EmissiveTexture)); + const bool isTextured = bool( + flags_ & (Flag::BaseColorTexture | Flag::RoughnessTexture | + Flag::NoneRoughnessMetallicTexture | Flag::MetallicTexture | + Flag::NormalTexture | Flag::EmissiveTexture)); if (isTextured) { attributeLocationsStream @@ -115,6 +115,9 @@ PbrShader::PbrShader(Flags originalFlags, unsigned int lightCount) : "") .addSource(flags_ & Flag::MetallicTexture ? "#define METALLIC_TEXTURE\n" : "") + .addSource(flags_ & Flag::NoneRoughnessMetallicTexture + ? "#define NONE_ROUGHNESS_METALLIC_TEXTURE\n" + : "") .addSource(flags_ & Flag::NormalTexture ? "#define NORMAL_TEXTURE\n" : "") .addSource(flags_ & Flag::NormalTextureScale ? "#define NORMAL_TEXTURE_SCALE\n" @@ -150,7 +153,8 @@ PbrShader::PbrShader(Flags originalFlags, unsigned int lightCount) setUniform(uniformLocation("BaseColorTexture"), pbrTextureUnitSpace::TextureUnit::BaseColor); } - if (flags_ & (Flag::RoughnessTexture | Flag::MetallicTexture)) { + if (flags_ & (Flag::RoughnessTexture | Flag::MetallicTexture | + Flag::NoneRoughnessMetallicTexture)) { setUniform(uniformLocation("MetallicRoughnessTexture"), pbrTextureUnitSpace::TextureUnit::MetallicRoughness); } @@ -256,7 +260,7 @@ PbrShader::PbrShader(Flags originalFlags, unsigned int lightCount) Cr::DirectInit, lightCount_, // a single directional "fill" light, coming from the center of the // camera. - Mn::Vector4{0.0f, 0.0f, 1.0f, 0.0f}}); + Mn::Vector4{0.0f, 0.0f, -1.0f, 0.0f}}); Cr::Containers::Array colors{Cr::DirectInit, lightCount_, Mn::Color3{1.0f}}; setLightColors(colors); @@ -296,7 +300,8 @@ PbrShader& PbrShader::bindBaseColorTexture(Mn::GL::Texture2D& texture) { PbrShader& PbrShader::bindMetallicRoughnessTexture(Mn::GL::Texture2D& texture) { CORRADE_ASSERT( - flags_ & (Flag::RoughnessTexture | Flag::MetallicTexture), + flags_ & (Flag::RoughnessTexture | Flag::MetallicTexture | + Flag::NoneRoughnessMetallicTexture), "PbrShader::bindMetallicRoughnessTexture(): the shader was not " "created with metallicRoughness texture enabled.", *this); diff --git a/src/esp/gfx/PbrShader.h b/src/esp/gfx/PbrShader.h index 53e7ed8e2b..5bce3fc516 100644 --- a/src/esp/gfx/PbrShader.h +++ b/src/esp/gfx/PbrShader.h @@ -95,7 +95,7 @@ class PbrShader : public Magnum::GL::AbstractShaderProgram { /** * Multiply metallic with the metallic texture. * This flag term means the metallic texture is independent, and "metalness" - * is stored in the R channel of it. + * is stored in the B channel of it. * NOTE: * if NoneRoughnessMetallicTexture or OcclusionRoughnessMetallicTexture are * presented, this texture will be ignored. @@ -103,25 +103,33 @@ class PbrShader : public Magnum::GL::AbstractShaderProgram { */ MetallicTexture = 1 << 2, + /** + * This flag term means the NoneRoughnessMetallic texture is present, with + * the Roughness in G channel and metalness in B channel (R and Alpha + * channels are not used). + * @see @ref setMetallic(), @ref bindMetallicTexture() + */ + NoneRoughnessMetallicTexture = 1 << 3, + /* * The occlusion map texture. * The occlusion, Roughness and Metalness are packed together in one * texture, with Occlusion in R channel, Roughness in G channel and * metalness in B channel (Alpha channels is not used). */ - PackedOcclusionTexture = 1 << 3, + PackedOcclusionTexture = 1 << 4, /* * The occlusion map texture. * The occlusion map texture is separate from the metallicRoughness texture. * The values are sampled from the R channel. */ - SeparateOcclusionTexture = 1 << 4, + SeparateOcclusionTexture = 1 << 5, /** * Modify normals according to a texture. */ - NormalTexture = 1 << 5, + NormalTexture = 1 << 6, /** * Enable normal texture scale @@ -129,7 +137,12 @@ class PbrShader : public Magnum::GL::AbstractShaderProgram { * @ref Flag::NormalTexture is enabled as well. * @see @ref setNormalTextureScale */ - NormalTextureScale = 1 << 6, + NormalTextureScale = 1 << 7, + + /** + * emissive texture + */ + EmissiveTexture = 1 << 8, /** * Enable texture coordinate transformation. If this flag is set, @@ -141,12 +154,7 @@ class PbrShader : public Magnum::GL::AbstractShaderProgram { * @ref Flag::OcclusionRoughnessMetallicTexture is enabled as well. * @see @ref setTextureMatrix() */ - TextureTransformation = 1 << 7, - - /** - * emissive texture - */ - EmissiveTexture = 1 << 8, + TextureTransformation = 1 << 9, /** * TODO: Do we need instanced object? (instanced texture, istanced id etc.) @@ -167,32 +175,38 @@ class PbrShader : public Magnum::GL::AbstractShaderProgram { PrecomputedTangent = 1 << 10, /** - * Enable object ID output. + * Enable object ID output for this shader. */ ObjectId = 1 << 11, + /** + * Support Instanced object ID. Retrieves a per-instance / per-vertex + * object ID from the @ref ObjectId attribute. If this is false, the shader + * will use the node's semantic ID + */ + InstancedObjectId = (1 << 12) | ObjectId, /** * Enable double-sided rendering. * (Temporarily STOP supporting this functionality. See comments in * the PbrDrawable::draw() function) */ - DoubleSided = 1 << 12, + DoubleSided = 1 << 13, /** * Enable image based lighting */ - ImageBasedLighting = 1 << 13, + ImageBasedLighting = 1 << 14, /** * render point light shadows using variance shadow map (VSM) */ - ShadowsVSM = 1 << 14, + ShadowsVSM = 1 << 15, /** * Enable shader debug mode. Then developer can set the uniform * PbrDebugDisplay in the fragment shader for debugging */ - DebugDisplay = 1 << 15, + DebugDisplay = 1 << 16, /* * TODO: alphaMask */ diff --git a/src/esp/gfx/ShaderManager.h b/src/esp/gfx/ShaderManager.h index 53f2a16bca..a80a4bd50e 100644 --- a/src/esp/gfx/ShaderManager.h +++ b/src/esp/gfx/ShaderManager.h @@ -7,16 +7,16 @@ #include #include +#include #include "esp/gfx/LightSetup.h" -#include "esp/gfx/MaterialData.h" namespace esp { namespace gfx { using ShaderManager = Magnum::ResourceManager; + Magnum::Trade::MaterialData>; /** * @brief Set the light setup for a subtree diff --git a/src/esp/gfx/SkinData.h b/src/esp/gfx/SkinData.h new file mode 100644 index 0000000000..84cbae01e0 --- /dev/null +++ b/src/esp/gfx/SkinData.h @@ -0,0 +1,49 @@ +// Copyright (c) Meta Platforms, Inc. and its affiliates. +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +#ifndef ESP_GFX_SKINDATA_H_ +#define ESP_GFX_SKINDATA_H_ + +#include +#include +#include +#include +#include + +namespace esp { +namespace gfx { + +/** + * @brief Stores skinning data for an asset. + */ +struct SkinData { + /** @brief Pointer to loaded skin data for the instance. */ + std::shared_ptr skin{}; + /** @brief Map of bone names and skin joint IDs. */ + std::unordered_map boneNameJointIdMap{}; + /** @brief Number of bones that can influence each vertex. */ + int perVertexJointCount{4}; +}; + +/** + * @brief Stores skinning data for an instance. + * Contains association information of graphics bones and articulated object + * links. + */ +struct InstanceSkinData { + /** @brief Pointer to loaded skin data for the instance. */ + const std::shared_ptr& skinData = nullptr; + /** @brief Root articulated object node. */ + scene::SceneNode* rootArticulatedObjectNode = nullptr; + /** @brief Map between skin joint IDs and scaled articulated object transform + * nodes. */ + std::unordered_map jointIdToTransformNode{}; + + explicit InstanceSkinData(const std::shared_ptr& skinData) + : skinData(skinData){}; +}; +} // namespace gfx +} // namespace esp + +#endif diff --git a/src/esp/gfx/replay/Player.cpp b/src/esp/gfx/replay/Player.cpp index ea9900ab85..08e150ee3d 100644 --- a/src/esp/gfx/replay/Player.cpp +++ b/src/esp/gfx/replay/Player.cpp @@ -18,6 +18,27 @@ namespace esp { namespace gfx { namespace replay { +namespace { + +// At recording time, material overrides gets stringified and appended to the +// filepath. See ResourceManager::createModifiedAssetName. +// AbstractSceneGraphPlayerImplementation doesn't support parsing this material +// info. More info at +// https://docs.google.com/document/d/1ngA73cXl3YRaPfFyICSUHONZN44C-XvieS7kwyQDbkI/edit#bookmark=id.aoe7xgsro2r7 +std::string removeMaterialOverrideFromFilepathAndWarn(const std::string& src) { + auto pos = src.find('?'); + if (pos != std::string::npos) { + ESP_WARNING(Mn::Debug::Flag::NoSpace) + << "Ignoring material-override for [" << src << "]"; + + return src.substr(0, pos); + } else { + return src; + } +} + +} // namespace + static_assert(std::is_nothrow_move_constructible::value, ""); void AbstractPlayerImplementation::setNodeSemanticId(NodeHandle, unsigned) {} @@ -173,22 +194,28 @@ void Player::applyKeyframe(const Keyframe& keyframe) { for (const auto& pair : keyframe.creations) { const auto& creation = pair.second; - if (assetInfos_.count(creation.filepath) == 0u) { - if (failedFilepaths_.count(creation.filepath) == 0u) { + + auto adjustedFilepath = + removeMaterialOverrideFromFilepathAndWarn(creation.filepath); + + if (assetInfos_.count(adjustedFilepath) == 0u) { + if (failedFilepaths_.count(adjustedFilepath) == 0u) { ESP_WARNING(Mn::Debug::Flag::NoSpace) - << "Missing asset info for [" << creation.filepath << "]"; - failedFilepaths_.insert(creation.filepath); + << "Missing asset info for [" << adjustedFilepath << "]"; + failedFilepaths_.insert(adjustedFilepath); } continue; } - CORRADE_INTERNAL_ASSERT(assetInfos_.count(creation.filepath)); + CORRADE_INTERNAL_ASSERT(assetInfos_.count(adjustedFilepath)); + auto adjustedCreation = creation; + adjustedCreation.filepath = adjustedFilepath; auto* node = implementation_->loadAndCreateRenderAssetInstance( - assetInfos_[creation.filepath], creation); + assetInfos_[adjustedFilepath], adjustedCreation); if (!node) { - if (failedFilepaths_.count(creation.filepath) == 0u) { + if (failedFilepaths_.count(adjustedFilepath) == 0u) { ESP_WARNING(Mn::Debug::Flag::NoSpace) - << "Load failed for asset [" << creation.filepath << "]"; - failedFilepaths_.insert(creation.filepath); + << "Load failed for asset [" << adjustedFilepath << "]"; + failedFilepaths_.insert(adjustedFilepath); } continue; } diff --git a/src/esp/gfx/replay/Recorder.cpp b/src/esp/gfx/replay/Recorder.cpp index 9b2489e0b0..88a3521daf 100644 --- a/src/esp/gfx/replay/Recorder.cpp +++ b/src/esp/gfx/replay/Recorder.cpp @@ -58,8 +58,11 @@ void Recorder::onCreateRenderAssetInstance( auto adjustedCreation = creation; - // bake node scale into creation + // We assume constant node scaling over the node's lifetime. Bake node scale + // into creation. auto nodeScale = node->absoluteTransformation().scaling(); + // todo: check for reflection (rotationShear.determinant() < 0.0f) and bake + // into scaling (negate X scale). if (nodeScale != Mn::Vector3(1.f, 1.f, 1.f)) { adjustedCreation.scale = adjustedCreation.scale ? *adjustedCreation.scale * nodeScale @@ -189,9 +192,17 @@ int Recorder::findInstance(const scene::SceneNode* queryNode) { RenderAssetInstanceState Recorder::getInstanceState( const scene::SceneNode* node) { const auto absTransformMat = node->absoluteTransformation(); - Transform absTransform{ - absTransformMat.translation(), - Magnum::Quaternion::fromMatrix(absTransformMat.rotationShear())}; + auto rotationShear = absTransformMat.rotationShear(); + // Remove reflection (negative scaling) from the matrix. We assume constant + // node scaling for the node's lifetime. It is baked into instance-creation so + // it doesn't need to be saved into RenderAssetInstanceState. See also + // onCreateRenderAssetInstance. + if (rotationShear.determinant() < 0.0f) { + rotationShear[0] *= -1.f; + } + + Transform absTransform{absTransformMat.translation(), + Magnum::Quaternion::fromMatrix(rotationShear)}; return RenderAssetInstanceState{absTransform, node->getSemanticId()}; } @@ -232,6 +243,25 @@ std::string Recorder::writeSavedKeyframesToString() { return esp::io::jsonToString(document); } +std::vector +Recorder::writeIncrementalSavedKeyframesToStringArray() { + std::vector results; + results.reserve(savedKeyframes_.size()); + + for (const auto& keyframe : savedKeyframes_) { + results.emplace_back(keyframeToString(keyframe)); + } + + // note we don't call consolidateSavedKeyframes. Use this function if you are + // using keyframes incrementally, e.g. repeated calls to this function and + // feeding them to a renderer. Contrast with writeSavedKeyframesToFile, which + // "consolidates" before discarding old keyframes to avoid losing state + // information. + savedKeyframes_.clear(); + + return results; +} + std::string Recorder::keyframeToString(const Keyframe& keyframe) { rapidjson::Document d(rapidjson::kObjectType); rapidjson::Document::AllocatorType& allocator = d.GetAllocator(); diff --git a/src/esp/gfx/replay/Recorder.h b/src/esp/gfx/replay/Recorder.h index 89b5f47afe..d695afed71 100644 --- a/src/esp/gfx/replay/Recorder.h +++ b/src/esp/gfx/replay/Recorder.h @@ -126,10 +126,21 @@ class Recorder { bool usePrettyWriter = false); /** - * @brief write saved keyframes to string. + * @brief write saved keyframes to string. '{"keyframes": [{...},{...},...]}' */ std::string writeSavedKeyframesToString(); + /** + * @brief write saved keyframes as individual strings ['{"keyframe": ...}', + * '{"keyframe": ...}', ...] + * + * Use this function if you are using keyframes incrementally, e.g. + * repeated calls to this function and feeding them to a renderer. Contrast + * with writeSavedKeyframesToFile, which "consolidates" before discarding old + * keyframes to avoid losing state information. + */ + std::vector writeIncrementalSavedKeyframesToStringArray(); + /** * @brief returns JSONized version of given keyframe. */ diff --git a/src/esp/io/URDFParser.cpp b/src/esp/io/URDFParser.cpp index 6502d581bd..7d684bf639 100644 --- a/src/esp/io/URDFParser.cpp +++ b/src/esp/io/URDFParser.cpp @@ -11,6 +11,7 @@ #include #include +#include "Corrade/Containers/Containers.h" #include "URDFParser.h" #include "esp/core/Logging.h" #include "esp/io/Json.h" @@ -120,15 +121,68 @@ bool Model::loadJsonAttributes(const std::string& filename) { ESP_ERROR() << " : Failed to parse" << jsonName << "as JSON."; return false; } + // JSON file exists and has been loaded; use to build this model's // configuration attributes // convert doc to const Json Generic val const io::JsonGenericValue jsonConfig = docConfig->GetObject(); + // check for render asset + const char* attrRenderAsset = "render_asset"; + const char* attrSemanticId = "semantic_id"; + const char* attrDebugRenderPrimitives = "debug_render_primitives"; + + // Parse render_asset + if (jsonConfig.HasMember(attrRenderAsset)) { + if (!jsonConfig[attrRenderAsset].IsString()) { + ESP_WARNING() << " : Json Config file specifies a render_asset " + "attribute but " + "it is not a string. Skipping render_asset config load."; + return false; + } else { + const std::string renderAssetRelativePath{ + jsonConfig[attrRenderAsset].GetString()}; + // render_asset path is relative to the urdf path. + auto fileDir = CrUt::Path::split(filename).first(); + auto renderAssetPath = CrUt::Path::join(fileDir, renderAssetRelativePath); + if (!CrUt::Path::exists(renderAssetPath)) { + ESP_WARNING() << " : render_asset defined in \"" << jsonName + << "\" could not be found. The path is relative to the " + "ao_config.json file."; + } + m_renderAsset = std::string(renderAssetPath.data()); + } + } + + // Parse semantic_id + if (jsonConfig.HasMember(attrSemanticId)) { + if (!jsonConfig[attrSemanticId].IsInt()) { + ESP_WARNING() << " : Json Config file specifies semantic_id " + "attribute but it is not an int. Skipping semantic_id " + "config load."; + return false; + } else { + m_semanticId = jsonConfig[attrSemanticId].GetInt(); + } + } + + // Parse debug_render_primitives + if (jsonConfig.HasMember(attrDebugRenderPrimitives)) { + if (!jsonConfig[attrDebugRenderPrimitives].IsBool()) { + ESP_WARNING() + << " : Json Config file specifies debug_render_primitives " + "attribute but it is not a bool. Skipping debug_render_primitives " + "config load."; + return false; + } else { + m_debugRenderPrimitives = jsonConfig[attrDebugRenderPrimitives].GetBool(); + } + } + + // check for user defined attributes and verify it is an object const std::string subGroupName = "user_defined"; const char* sg_cstr = subGroupName.c_str(); - // check for user defined attributes and verify it is an object if (jsonConfig.HasMember(sg_cstr)) { if (!jsonConfig[sg_cstr].IsObject()) { ESP_WARNING() @@ -153,8 +207,7 @@ bool Model::loadJsonAttributes(const std::string& filename) { return (numConfigSettings > 0); } // if has user_defined tag - ESP_WARNING() << " : Json Config file exists but \"" << subGroupName - << "\" tag not found within file."; + return false; } // Model::loadJsonAttributes diff --git a/src/esp/io/URDFParser.h b/src/esp/io/URDFParser.h index 30c3fdd187..40b9b7b501 100644 --- a/src/esp/io/URDFParser.h +++ b/src/esp/io/URDFParser.h @@ -14,6 +14,7 @@ #include #include #include +#include "Corrade/Containers/Optional.h" #include "esp/core/Configuration.h" /** @file @@ -246,7 +247,7 @@ struct Link { //! list of all link children of this link std::vector> m_childJoints; //! joints attaching this link to each of its children (with index - //! correspondance to m_childJoints) + //! correspondence to m_childJoints) std::vector> m_childLinks; //! the index of this link in the overall articulated object @@ -361,16 +362,55 @@ class Model { */ float getMassScaling() const { return m_massScaling; } + /** + * @brief Set the path to the render asset to attach to this URDF model. + */ + void setRenderAsset(Cr::Containers::Optional renderAsset) { + m_renderAsset = std::move(renderAsset); + } + + /** + * @brief Get the path to the render asset to attach to this URDF model. + */ + Cr::Containers::Optional getRenderAsset() const { + return m_renderAsset; + } + + /** + * @brief Set the semantic ID of the URDF model. + */ + void setSemanticId(int semanticId) { m_semanticId = semanticId; } + + /** + * @brief Get the semantic ID of this URDF model. + */ + int getSemanticId() const { return m_semanticId; } + + /** + * @brief Set hint to render articulated object primitives even if a render + * asset is present. + */ + void setDebugRenderPrimitives(bool debugRenderPrimitives) { + m_debugRenderPrimitives = debugRenderPrimitives; + } + + /** + * @brief Get hint to render articulated object primitives even if a render + * asset is present. + */ + bool getDebugRenderPrimitives() const { return m_debugRenderPrimitives; } + /** * @brief This function conditionally loads configuration data from a Json * file for this Articulated Model, should an appropriate file exist. This * configuration file must meet the following criteria to be loaded : - * --Exist in the same directory as thsi model's source URDF/XML file. + * - Exist in the same directory as this model's source URDF/XML file. * - Have the same root name as this model's source URDF/XML file. * - Have '.ao_config.json' as an extension. * * This method will construct a name candidate from the passed @p filename and * attempt to load this file into this model's @ref jsonAttributes_ variable. + * This also loads the render asset path from the json file. * @param filename The filename for the URDF?XML file describing this model. * @return Whether successful or not. */ @@ -390,7 +430,7 @@ class Model { protected: /** * @brief Json-based attributes defining characteristics of this model not - * specified in the source XML/URDF. Primarly to support defaultt user-defined + * specified in the source XML/URDF. Primarily to support default user-defined * attributes. This data is read in from a json file with the same base name * as the source XML/URDF for this model but the extension ".ao_config.json". */ @@ -405,6 +445,15 @@ class Model { //! Mass scaling of the model's Link inertias. float m_massScaling = 1.0; + //! Path to a render asset associated with this articulated object. + Cr::Containers::Optional m_renderAsset = Cr::Containers::NullOpt; + + //! Semantic ID of this model. + int m_semanticId = 0; + + //! Forces link primitives to be rendered even if a render asset is present. + bool m_debugRenderPrimitives = false; + //! Scale the transformation and parameters of a Shape void scaleShape(Shape& shape, float scale); }; // class model diff --git a/src/esp/metadata/MetadataMediator.cpp b/src/esp/metadata/MetadataMediator.cpp index 358dd6ffd6..323adf39bb 100644 --- a/src/esp/metadata/MetadataMediator.cpp +++ b/src/esp/metadata/MetadataMediator.cpp @@ -376,6 +376,40 @@ MetadataMediator::makeSceneAndReferenceStage( return sceneInstanceAttributes; } // MetadataMediator::makeSceneAndReferenceStage +std::shared_ptr +MetadataMediator::getSceneInstanceUserConfiguration( + const std::string& curSceneName) { + // get current dataset attributes + attributes::SceneDatasetAttributes::ptr datasetAttr = getActiveDSAttribs(); + // this should never happen + if (datasetAttr == nullptr) { + ESP_ERROR() << "No dataset specified/exists. Aborting."; + return nullptr; + } + // get scene instance attribute manager + managers::SceneInstanceAttributesManager::ptr dsSceneAttrMgr = + datasetAttr->getSceneInstanceAttributesManager(); + // get list of scene attributes handles that contain sceneName as a substring + auto sceneList = dsSceneAttrMgr->getObjectHandlesBySubstring(curSceneName); + // returned list of scene names must not be empty, otherwise display error + // message and return nullptr + if (!sceneList.empty()) { + // Scene instance exists with given name, registered SceneInstanceAttributes + // in current active dataset. + // In this case the SceneInstanceAttributes is returned. + ESP_DEBUG() << "Query dataset :" << activeSceneDataset_ + << "for SceneInstanceAttributes named :" << curSceneName + << "yields" << sceneList.size() << "candidates. Using" + << sceneList[0] << Mn::Debug::nospace << "."; + return dsSceneAttrMgr->getObjectCopyByHandle(sceneList[0]) + ->getUserConfiguration(); + } + ESP_ERROR() << "No scene instance specified/exists with name" << curSceneName + << ", so Aborting."; + return nullptr; + +} // MetadataMediator::getSceneInstanceUserConfiguration + std::string MetadataMediator::getFilePathForHandle( const std::string& assetHandle, const std::map& assetMapping, diff --git a/src/esp/metadata/MetadataMediator.h b/src/esp/metadata/MetadataMediator.h index 801f1ecd18..a1a9d0bbe5 100644 --- a/src/esp/metadata/MetadataMediator.h +++ b/src/esp/metadata/MetadataMediator.h @@ -21,6 +21,10 @@ namespace esp { +namespace core { +class Configuration; +} + namespace sim { struct SimulatorConfiguration; } @@ -417,6 +421,15 @@ class MetadataMediator { */ std::string createDatasetReport(const std::string& sceneDataset = "") const; + /** + * @brief Return the root-level user defined attributes configuration for the + * specified scene instance. + * @param sceneName The scene name in the currently loaded SceneDataset. + * @return The scene instance user-defined configuration. + */ + std::shared_ptr + getSceneInstanceUserConfiguration(const std::string& curSceneName); + protected: /** * @brief Return the file path corresponding to the passed handle in the diff --git a/src/esp/metadata/attributes/SceneDatasetAttributes.h b/src/esp/metadata/attributes/SceneDatasetAttributes.h index e6b6ebcb7c..35d8a20e2a 100644 --- a/src/esp/metadata/attributes/SceneDatasetAttributes.h +++ b/src/esp/metadata/attributes/SceneDatasetAttributes.h @@ -284,7 +284,7 @@ class SceneDatasetAttributes : public AbstractAttributes { void setArticulatedObjectModelFilename(const std::string& key, const std::string& val) { auto artObjPathIter = articulatedObjPaths.find(key); - if (artObjPathIter == articulatedObjPaths.end()) { + if (artObjPathIter != articulatedObjPaths.end()) { ESP_WARNING() << "Articulated model filepath named" << key << "already exists (" << artObjPathIter->second << "), so this is being overwritten by" << val << "."; diff --git a/src/esp/metadata/managers/LightLayoutAttributesManager.cpp b/src/esp/metadata/managers/LightLayoutAttributesManager.cpp index 2cb482613e..05b4b69604 100644 --- a/src/esp/metadata/managers/LightLayoutAttributesManager.cpp +++ b/src/esp/metadata/managers/LightLayoutAttributesManager.cpp @@ -279,10 +279,10 @@ gfx::LightSetup LightLayoutAttributesManager::createLightSetupFromAttributes( if (numLightInstances == 0) { // setup default LightInfo instances - lifted from LightSetup.cpp. // TODO create default attributes describing these lights? - return gfx::LightSetup{{.vector = {1.0, 1.0, 0.0, 0.0}, + return gfx::LightSetup{{.vector = {-1.0, -1.0, 0.0, 0.0}, .color = {0.75, 0.75, 0.75}, .model = gfx::LightPositionModel::Global}, - {.vector = {-0.5, 0.0, 1.0, 0.0}, + {.vector = {0.5, 0.0, -1.0, 0.0}, .color = {0.4, 0.4, 0.4}, .model = gfx::LightPositionModel::Global}}; } else { diff --git a/src/esp/physics/ArticulatedObject.h b/src/esp/physics/ArticulatedObject.h index b5b9bc8221..f11470cfb3 100644 --- a/src/esp/physics/ArticulatedObject.h +++ b/src/esp/physics/ArticulatedObject.h @@ -425,6 +425,21 @@ class ArticulatedObject : public esp::physics::PhysicsObjectBase { return ids; } + /** + * @brief Get a list of link ids including the base (-1). + * + * @return A list of link ids for this object. + */ + std::vector getLinkIdsWithBase() const { + std::vector ids; + ids.reserve(links_.size() + 1); + ids.push_back(-1); + for (auto it = links_.begin(); it != links_.end(); ++it) { + ids.push_back(it->first); + } + return ids; + } + /** * @brief Get a map of object ids to link ids. * diff --git a/src/esp/physics/PhysicsManager.cpp b/src/esp/physics/PhysicsManager.cpp index 33ab8a619e..7bf8da403f 100644 --- a/src/esp/physics/PhysicsManager.cpp +++ b/src/esp/physics/PhysicsManager.cpp @@ -329,7 +329,7 @@ int PhysicsManager::addArticulatedObjectInstance( filepath, &drawables, aObjInstAttributes->getFixedBase(), aObjInstAttributes->getUniformScale(), static_cast(aObjInstAttributes->getMassScale()), false, false, - lightSetup); + false, lightSetup); if (aObjID == ID_UNDEFINED) { // instancing failed for some reason. ESP_ERROR() << "Articulated Object create failed for model filepath" diff --git a/src/esp/physics/PhysicsManager.h b/src/esp/physics/PhysicsManager.h index 13f76e7d02..4df1ba5885 100644 --- a/src/esp/physics/PhysicsManager.h +++ b/src/esp/physics/PhysicsManager.h @@ -472,6 +472,8 @@ class PhysicsManager : public std::enable_shared_from_this { * cached model. * @param maintainLinkOrder If true, maintain the order of link definitions * from the URDF file as the link indices. + * @param intertiaFromURDF If true, load the link inertia matrices from the + * URDF file instead of computing automatically from collision shapes. * @param lightSetup The string name of the desired lighting setup to use. * * @return A unique id for the @ref BulletArticulatedObject, allocated from @@ -484,6 +486,7 @@ class PhysicsManager : public std::enable_shared_from_this { CORRADE_UNUSED float massScale = 1.0, CORRADE_UNUSED bool forceReload = false, CORRADE_UNUSED bool maintainLinkOrder = false, + CORRADE_UNUSED bool intertiaFromURDF = false, CORRADE_UNUSED const std::string& lightSetup = DEFAULT_LIGHTING_KEY) { ESP_DEBUG() << "Not implemented in base PhysicsManager."; return ID_UNDEFINED; @@ -508,6 +511,8 @@ class PhysicsManager : public std::enable_shared_from_this { * cached model. * @param maintainLinkOrder If true, maintain the order of link definitions * from the URDF file as the link indices. + * @param intertiaFromURDF If true, load the link inertia matrices from the + * URDF file instead of computing automatically from collision shapes. * @param lightSetup The string name of the desired lighting setup to use. * * @return A unique id for the @ref ArticulatedObject, allocated from the same @@ -521,6 +526,7 @@ class PhysicsManager : public std::enable_shared_from_this { CORRADE_UNUSED float massScale = 1.0, CORRADE_UNUSED bool forceReload = false, CORRADE_UNUSED bool maintainLinkOrder = false, + CORRADE_UNUSED bool intertiaFromURDF = false, CORRADE_UNUSED const std::string& lightSetup = DEFAULT_LIGHTING_KEY) { ESP_DEBUG() << "Not implemented in base PhysicsManager."; return ID_UNDEFINED; diff --git a/src/esp/physics/bullet/BulletArticulatedObject.cpp b/src/esp/physics/bullet/BulletArticulatedObject.cpp index 2150d0c2f2..b64492d355 100644 --- a/src/esp/physics/bullet/BulletArticulatedObject.cpp +++ b/src/esp/physics/bullet/BulletArticulatedObject.cpp @@ -79,11 +79,12 @@ void BulletArticulatedObject::initializeFromURDF( auto urdfModel = u2b.getModel(); + node().setSemanticId(urdfModel->getSemanticId()); + // cache the global scaling from the source model globalScale_ = urdfModel->getGlobalScaling(); int urdfLinkIndex = u2b.getRootLinkIndex(); - // int rootIndex = u2b.getRootLinkIndex(); bool recursive = (u2b.flags & CUF_MAINTAIN_LINK_ORDER) == 0; diff --git a/src/esp/physics/bullet/BulletArticulatedObject.h b/src/esp/physics/bullet/BulletArticulatedObject.h index 7800307576..f965ddc2a2 100644 --- a/src/esp/physics/bullet/BulletArticulatedObject.h +++ b/src/esp/physics/bullet/BulletArticulatedObject.h @@ -95,11 +95,11 @@ class BulletArticulatedObject : public ArticulatedObject { scene::SceneNode* physicsNode) override; /** - * @brief Cosntruct a Static btRigidObject to act as a proxy collision object + * @brief Construct a Static btRigidObject to act as a proxy collision object * for the fixed base. * * This optimization reduces the collision island size for articulated objects - * with heavy branching (e.g. a counter with many drawers) resuling in better + * with heavy branching (e.g. a counter with many drawers) resulting in better * sleeping behavior (e.g. contact with the countertop should not activate all * drawers and contained objects). * @@ -409,7 +409,7 @@ class BulletArticulatedObject : public ArticulatedObject { * * By default, state is interpreted as position targets unless `velocities` is * specified. Expected input is the full length position or velocity array for - * this object. This function will safely skip states for jointa which don't + * this object. This function will safely skip states for joints which don't * support JointMotors. * * Note: No base implementation. See @ref bullet::BulletArticulatedObject. @@ -457,7 +457,7 @@ class BulletArticulatedObject : public ArticulatedObject { //! maps local link id to parent joint's limit constraint std::unordered_map jointLimitConstraints; - // scratch datastrcutures for updateKinematicState + // scratch data structures for updateKinematicState btAlignedObjectArray scratch_q_; btAlignedObjectArray scratch_m_; diff --git a/src/esp/physics/bullet/BulletPhysicsManager.cpp b/src/esp/physics/bullet/BulletPhysicsManager.cpp index 768b4505f5..9fa76c03a1 100644 --- a/src/esp/physics/bullet/BulletPhysicsManager.cpp +++ b/src/esp/physics/bullet/BulletPhysicsManager.cpp @@ -12,10 +12,12 @@ #include "BulletDynamics/Featherstone/btMultiBodyLinkCollider.h" #include "BulletRigidObject.h" #include "BulletURDFImporter.h" +#include "esp/assets/RenderAssetInstanceCreationInfo.h" #include "esp/assets/ResourceManager.h" #include "esp/metadata/attributes/PhysicsManagerAttributes.h" #include "esp/physics/objectManagers/ArticulatedObjectManager.h" #include "esp/physics/objectManagers/RigidObjectManager.h" +#include "esp/scene/SceneNode.h" #include "esp/sim/Simulator.h" namespace esp { @@ -114,11 +116,12 @@ int BulletPhysicsManager::addArticulatedObjectFromURDF( float massScale, bool forceReload, bool maintainLinkOrder, + bool intertiaFromURDF, const std::string& lightSetup) { auto& drawables = simulator_->getDrawableGroup(); - return addArticulatedObjectFromURDF(filepath, &drawables, fixedBase, - globalScale, massScale, forceReload, - maintainLinkOrder, lightSetup); + return addArticulatedObjectFromURDF( + filepath, &drawables, fixedBase, globalScale, massScale, forceReload, + maintainLinkOrder, intertiaFromURDF, lightSetup); } int BulletPhysicsManager::addArticulatedObjectFromURDF( @@ -129,9 +132,10 @@ int BulletPhysicsManager::addArticulatedObjectFromURDF( float massScale, bool forceReload, bool maintainLinkOrder, + bool inertiaFromURDF, const std::string& lightSetup) { if (simulator_ != nullptr) { - // aquire context if available + // acquire context if available simulator_->getRenderGLContext(); } ESP_CHECK( @@ -160,9 +164,34 @@ int BulletPhysicsManager::addArticulatedObjectFromURDF( if (maintainLinkOrder) { u2b->flags |= CUF_MAINTAIN_LINK_ORDER; } + if (inertiaFromURDF) { + u2b->flags |= CUF_USE_URDF_INERTIA; + } u2b->initURDF2BulletCache(); articulatedObject->initializeFromURDF(*urdfImporter_, {}, physicsNode_); + auto model = u2b->getModel(); + + // if the URDF model specifies a render asset, load and link it + const auto renderAssetPath = model->getRenderAsset(); + if (renderAssetPath) { + // load associated skinned mesh + assets::AssetInfo assetInfo = assets::AssetInfo::fromPath(*renderAssetPath); + assets::RenderAssetInstanceCreationInfo creationInfo; + creationInfo.filepath = *renderAssetPath; + creationInfo.lightSetupKey = lightSetup; + creationInfo.scale = globalScale * Mn::Vector3(1.f, 1.f, 1.f); + creationInfo.rig = articulatedObject; + esp::assets::RenderAssetInstanceCreationInfo::Flags flags; + flags |= esp::assets::RenderAssetInstanceCreationInfo::Flag::IsRGBD; + flags |= esp::assets::RenderAssetInstanceCreationInfo::Flag::IsSemantic; + creationInfo.flags = flags; + auto* gfxNode = resourceManager_.loadAndCreateRenderAssetInstance( + assetInfo, creationInfo, objectNode, drawables); + // Propagate the semantic ID to the graphics subtree + esp::scene::setSemanticIdForSubtree( + gfxNode, articulatedObject->node().getSemanticId()); + } // allocate ids for links for (int linkIx = 0; linkIx < articulatedObject->btMultiBody_->getNumLinks(); @@ -173,20 +202,27 @@ int BulletPhysicsManager::addArticulatedObjectFromURDF( articulatedObject->btMultiBody_->getLinkCollider(linkIx), linkObjectId); } - // attach link visual shapes - for (size_t urdfLinkIx = 0; urdfLinkIx < u2b->getModel()->m_links.size(); - ++urdfLinkIx) { - auto urdfLink = u2b->getModel()->getLink(urdfLinkIx); - if (!urdfLink->m_visualArray.empty()) { - int bulletLinkIx = - u2b->cache->m_urdfLinkIndices2BulletLinkIndices[urdfLinkIx]; - ArticulatedLink& linkObject = articulatedObject->getLink(bulletLinkIx); - ESP_CHECK( - attachLinkGeometry(&linkObject, urdfLink, drawables, lightSetup), - "BulletPhysicsManager::addArticulatedObjectFromURDF(): Failed to " - "instance render asset (attachGeometry) for link" - << urdfLinkIx << "."); - linkObject.node().computeCumulativeBB(); + // render visual shapes if either no skinned mesh is present or if the debug + // flag is enabled + bool renderVisualShapes = + !renderAssetPath || model->getDebugRenderPrimitives(); + if (renderVisualShapes) { + // attach link visual shapes + for (size_t urdfLinkIx = 0; urdfLinkIx < model->m_links.size(); + ++urdfLinkIx) { + auto urdfLink = model->getLink(urdfLinkIx); + if (!urdfLink->m_visualArray.empty()) { + int bulletLinkIx = + u2b->cache->m_urdfLinkIndices2BulletLinkIndices[urdfLinkIx]; + ArticulatedLink& linkObject = articulatedObject->getLink(bulletLinkIx); + ESP_CHECK( + attachLinkGeometry(&linkObject, urdfLink, drawables, lightSetup, + articulatedObject->node().getSemanticId()), + "BulletPhysicsManager::addArticulatedObjectFromURDF(): Failed to " + "instance render asset (attachGeometry) for link" + << urdfLinkIx << "."); + linkObject.node().computeCumulativeBB(); + } } } @@ -248,9 +284,9 @@ bool BulletPhysicsManager::attachLinkGeometry( ArticulatedLink* linkObject, const std::shared_ptr& link, gfx::DrawableGroup* drawables, - const std::string& lightSetup) { + const std::string& lightSetup, + int semanticId) { const bool forceFlatShading = (lightSetup == esp::NO_LIGHT_KEY); - bool geomSuccess = false; for (auto& visual : link->m_visualArray) { bool visualSetupSuccess = true; @@ -344,19 +380,22 @@ bool BulletPhysicsManager::attachLinkGeometry( assets::RenderAssetInstanceCreationInfo creation( visualMeshInfo.filepath, Mn::Vector3{1}, flags, lightSetup); - geomSuccess = resourceManager_.loadAndCreateRenderAssetInstance( - visualMeshInfo, creation, &visualGeomComponent, - drawables, &linkObject->visualNodes_) != nullptr; + auto* geomNode = resourceManager_.loadAndCreateRenderAssetInstance( + visualMeshInfo, creation, &visualGeomComponent, drawables, + &linkObject->visualNodes_); - // cache the visual component for later query - if (geomSuccess) { + if (geomNode) { + // Propagate the semantic ID to the graphics subtree + esp::scene::setSemanticIdForSubtree(&visualGeomComponent, semanticId); + // cache the visual component for later query linkObject->visualAttachments_.emplace_back( &visualGeomComponent, visual.m_geometry.m_meshFileName); + return true; } } } - return geomSuccess; + return false; } void BulletPhysicsManager::setGravity(const Magnum::Vector3& gravity) { @@ -579,7 +618,7 @@ std::vector BulletPhysicsManager::getContactPoints() const { pt.positionOnAInWS = Mn::Vector3(srcPt.getPositionWorldOnA()); pt.positionOnBInWS = Mn::Vector3(srcPt.getPositionWorldOnB()); - // convert impulses to forces w/ recent physics timstep + // convert impulses to forces w/ recent physics timestep pt.normalForce = static_cast(srcPt.getAppliedImpulse()) / recentTimeStep_; @@ -732,7 +771,7 @@ int BulletPhysicsManager::createRigidConstraint( } } - // link objects to their consraints for later deactivation/removal logic + // link objects to their constraints for later deactivation/removal logic objectConstraints_[settings.objectIdA].push_back(nextConstraintId_); if (settings.objectIdB != ID_UNDEFINED) { objectConstraints_[settings.objectIdB].push_back(nextConstraintId_); diff --git a/src/esp/physics/bullet/BulletPhysicsManager.h b/src/esp/physics/bullet/BulletPhysicsManager.h index aff6db0d8f..66bf737f7f 100644 --- a/src/esp/physics/bullet/BulletPhysicsManager.h +++ b/src/esp/physics/bullet/BulletPhysicsManager.h @@ -82,6 +82,8 @@ class BulletPhysicsManager : public PhysicsManager { * cached model. * @param maintainLinkOrder If true, maintain the order of link definitions * from the URDF file as the link indices. + * @param intertiaFromURDF If true, load the link inertia matrices from the + * URDF file instead of computing automatically from collision shapes. * @param lightSetup The string name of the desired lighting setup to use. * * @return A unique id for the @ref ArticulatedObject, allocated from the same @@ -94,6 +96,7 @@ class BulletPhysicsManager : public PhysicsManager { float massScale = 1.0, bool forceReload = false, bool maintainLinkOrder = false, + bool intertiaFromURDF = false, const std::string& lightSetup = DEFAULT_LIGHTING_KEY) override; /** @@ -113,6 +116,8 @@ class BulletPhysicsManager : public PhysicsManager { * cached model. * @param maintainLinkOrder If true, maintain the order of link definitions * from the URDF file as the link indices. + * @param intertiaFromURDF If true, load the link inertia matrices from the + * URDF file instead of computing automatically from collision shapes. * @param lightSetup The string name of the desired lighting setup to use. * * @return A unique id for the @ref ArticulatedObject, allocated from the same @@ -126,6 +131,7 @@ class BulletPhysicsManager : public PhysicsManager { float massScale = 1.0, bool forceReload = false, bool maintainLinkOrder = false, + bool intertiaFromURDF = false, const std::string& lightSetup = DEFAULT_LIGHTING_KEY) override; /** @@ -145,7 +151,8 @@ class BulletPhysicsManager : public PhysicsManager { bool attachLinkGeometry(ArticulatedLink* linkObject, const std::shared_ptr& link, gfx::DrawableGroup* drawables, - const std::string& lightSetup); + const std::string& lightSetup, + int semanticId); /** * @brief Override of @ref PhysicsManager::removeObject to also remove any @@ -209,7 +216,7 @@ class BulletPhysicsManager : public PhysicsManager { /** @brief Get the current coefficient of restitution for the stage * collision geometry. This determines the ratio of initial to final relative - * velocity between the stage and collidiing object. See @ref + * velocity between the stage and colliding object. See @ref * staticStageObject_ and BulletRigidObject::getRestitutionCoefficient. * @return The scalar coefficient of restitution for the stage geometry. */ diff --git a/src/esp/physics/bullet/BulletRigidObject.cpp b/src/esp/physics/bullet/BulletRigidObject.cpp index aa5338231a..7141f2c333 100644 --- a/src/esp/physics/bullet/BulletRigidObject.cpp +++ b/src/esp/physics/bullet/BulletRigidObject.cpp @@ -205,7 +205,7 @@ BulletRigidObject::buildPrimitiveCollisionObject(int primTypeVal, case metadata::PrimObjTypes::CYLINDER_WF: { // use bullet cylinder shape :btCylinderShape (const btVector3& // halfExtents); - btVector3 dim(1.0, 1.0, 1.0); + btVector3 dim(1.0, halfLength, 1.0); obj = std::make_unique(dim); break; } diff --git a/src/esp/physics/objectManagers/ArticulatedObjectManager.cpp b/src/esp/physics/objectManagers/ArticulatedObjectManager.cpp index 01323e65fa..8aa9bdaf3b 100644 --- a/src/esp/physics/objectManagers/ArticulatedObjectManager.cpp +++ b/src/esp/physics/objectManagers/ArticulatedObjectManager.cpp @@ -37,11 +37,12 @@ ArticulatedObjectManager::addArticulatedObjectFromURDF( float massScale, bool forceReload, bool maintainLinkOrder, + bool intertiaFromURDF, const std::string& lightSetup) { if (auto physMgr = this->getPhysicsManager()) { int newAObjID = physMgr->addArticulatedObjectFromURDF( filepath, fixedBase, globalScale, massScale, forceReload, - maintainLinkOrder, lightSetup); + maintainLinkOrder, intertiaFromURDF, lightSetup); return this->getObjectCopyByID(newAObjID); } return nullptr; @@ -56,11 +57,12 @@ ArticulatedObjectManager::addArticulatedObjectFromURDFWithDrawables( float massScale, bool forceReload, bool maintainLinkOrder, + bool intertiaFromURDF, const std::string& lightSetup) { if (auto physMgr = this->getPhysicsManager()) { int newAObjID = physMgr->addArticulatedObjectFromURDF( filepath, drawables, fixedBase, globalScale, massScale, forceReload, - maintainLinkOrder, lightSetup); + maintainLinkOrder, intertiaFromURDF, lightSetup); return this->getObjectCopyByID(newAObjID); } return nullptr; diff --git a/src/esp/physics/objectManagers/ArticulatedObjectManager.h b/src/esp/physics/objectManagers/ArticulatedObjectManager.h index 71b939b6dc..620caf9f86 100644 --- a/src/esp/physics/objectManagers/ArticulatedObjectManager.h +++ b/src/esp/physics/objectManagers/ArticulatedObjectManager.h @@ -38,6 +38,8 @@ class ArticulatedObjectManager * cached model. * @param maintainLinkOrder If true, maintain the order of link definitions * from the URDF file as the link indices. + * @param intertiaFromURDF If true, load the link inertia matrices from the + * URDF file instead of computing automatically from collision shapes. * @param lightSetup The string name of the desired lighting setup to use. * * @return A reference to the created ArticulatedObject @@ -49,6 +51,7 @@ class ArticulatedObjectManager float massScale = 1.0, bool forceReload = false, bool maintainLinkOrder = false, + bool intertiaFromURDF = false, const std::string& lightSetup = DEFAULT_LIGHTING_KEY); /** @@ -67,6 +70,8 @@ class ArticulatedObjectManager * cached model. * @param maintainLinkOrder If true, maintain the order of link definitions * from the URDF file as the link indices. + * @param intertiaFromURDF If true, load the link inertia matrices from the + * URDF file instead of computing automatically from collision shapes. * @param lightSetup The string name of the desired lighting setup to use. * * @return A reference to the created ArticulatedObject @@ -78,11 +83,12 @@ class ArticulatedObjectManager float massScale = 1.0, bool forceReload = false, bool maintainLinkOrder = false, + bool intertiaFromURDF = false, const std::string& lightSetup = DEFAULT_LIGHTING_KEY) { std::shared_ptr objPtr = addArticulatedObjectFromURDF(filepath, fixedBase, globalScale, massScale, forceReload, maintainLinkOrder, - lightSetup); + intertiaFromURDF, lightSetup); if (std::shared_ptr castObjPtr = std::dynamic_pointer_cast(objPtr)) { @@ -108,6 +114,8 @@ class ArticulatedObjectManager * the cached model. * @param maintainLinkOrder If true, maintain the order of link definitions * from the URDF file as the link indices. + * @param intertiaFromURDF If true, load the link inertia matrices from the + * URDF file instead of computing automatically from collision shapes. * @param lightSetup The string name of the desired lighting setup to use. * * @return A reference to the created ArticulatedObject @@ -121,6 +129,7 @@ class ArticulatedObjectManager float massScale = 1.0, bool forceReload = false, bool maintainLinkOrder = false, + bool intertiaFromURDF = false, const std::string& lightSetup = DEFAULT_LIGHTING_KEY); protected: diff --git a/src/esp/physics/objectWrappers/ManagedArticulatedObject.h b/src/esp/physics/objectWrappers/ManagedArticulatedObject.h index a7f54bd55c..8b7aa309a6 100644 --- a/src/esp/physics/objectWrappers/ManagedArticulatedObject.h +++ b/src/esp/physics/objectWrappers/ManagedArticulatedObject.h @@ -66,6 +66,13 @@ class ManagedArticulatedObject return {}; } + std::vector getLinkIdsWithBase() const { + if (auto sp = getObjectReference()) { + return sp->getLinkIdsWithBase(); + } + return {}; + } + std::unordered_map getLinkObjectIds() const { if (auto sp = getObjectReference()) { return sp->getLinkObjectIds(); diff --git a/src/esp/sim/AbstractReplayRenderer.cpp b/src/esp/sim/AbstractReplayRenderer.cpp index a09c05c01a..8effc32d2f 100644 --- a/src/esp/sim/AbstractReplayRenderer.cpp +++ b/src/esp/sim/AbstractReplayRenderer.cpp @@ -114,5 +114,27 @@ const void* AbstractReplayRenderer::getCudaDepthBufferDevicePointer() { return nullptr; } +std::shared_ptr +AbstractReplayRenderer::getDebugLineRender(unsigned envIndex) { + ESP_CHECK(envIndex == 0, "getDebugLineRender is only available for env 0"); + // We only create this if/when used (lazy creation) + if (!debugLineRender_) { + debugLineRender_ = std::make_shared(); + } + return debugLineRender_; +} + +esp::geo::Ray AbstractReplayRenderer::unproject( + unsigned envIndex, + const Mn::Vector2i& viewportPosition) { + checkEnvIndex(envIndex); + return doUnproject(envIndex, viewportPosition); +} + +void AbstractReplayRenderer::checkEnvIndex(unsigned envIndex) { + ESP_CHECK(envIndex < doEnvironmentCount(), + "envIndex " << envIndex << " is out of range"); +} + } // namespace sim } // namespace esp diff --git a/src/esp/sim/AbstractReplayRenderer.h b/src/esp/sim/AbstractReplayRenderer.h index ef24a8f27f..3f584738b9 100644 --- a/src/esp/sim/AbstractReplayRenderer.h +++ b/src/esp/sim/AbstractReplayRenderer.h @@ -8,7 +8,12 @@ #include #include +#include "esp/core/Check.h" #include "esp/core/Esp.h" +#include "esp/geo/Geo.h" +#include "esp/gfx/DebugLineRender.h" + +#include namespace esp { @@ -108,6 +113,25 @@ class AbstractReplayRenderer { // Retrieve the depth buffer as a CUDA device pointer. */ virtual const void* getCudaDepthBufferDevicePointer(); + std::shared_ptr getDebugLineRender( + unsigned envIndex); + + /** + * @brief Unproject a 2D viewport point to a 3D ray with origin at camera + * position. Ray direction is normalized. + * + * @param envIndex + * @param viewportPosition The 2D point on the viewport to unproject + * ([0,width], [0,height]). + */ + esp::geo::Ray unproject(unsigned envIndex, + const Magnum::Vector2i& viewportPosition); + + protected: + void checkEnvIndex(unsigned envIndex); + + std::shared_ptr debugLineRender_; + private: /* Implementation of all public API is in the private do*() functions, similarly to how e.g. Magnum plugin interfaces work. The public API does @@ -144,6 +168,9 @@ class AbstractReplayRenderer { virtual void doRender(Magnum::GL::AbstractFramebuffer& framebuffer) = 0; + virtual esp::geo::Ray doUnproject(unsigned envIndex, + const Mn::Vector2i& viewportPosition) = 0; + ESP_SMART_POINTERS(AbstractReplayRenderer) }; diff --git a/src/esp/sim/BatchReplayRenderer.cpp b/src/esp/sim/BatchReplayRenderer.cpp index a0195e4197..a5346294fa 100644 --- a/src/esp/sim/BatchReplayRenderer.cpp +++ b/src/esp/sim/BatchReplayRenderer.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -55,6 +56,33 @@ BatchReplayRenderer::BatchReplayRenderer( : renderer_{renderer}, sceneId_{sceneId} {} private: + bool isSupportedRenderAsset( + const Corrade::Containers::StringView& filepath) { + // Primitives aren't directly supported in the Magnum batch renderer. See + // https://docs.google.com/document/d/1ngA73cXl3YRaPfFyICSUHONZN44C-XvieS7kwyQDbkI/edit#bookmark=id.yq39718gqbwz + + const std::array primNamePrefixes = { + "capsule3DSolid", "capsule3DWireframe", "coneSolid", + "coneWireframe", "cubeSolid", "cubeWireframe", + "cylinderSolid", "cylinderWireframe", "icosphereSolid", + "icosphereWireframe", "uvSphereSolid", "uvSphereWireframe"}; + + // primitive render asset filepaths start with one of the above prefixes. + // Examples: icosphereSolid_subdivs_1 + // capsule3DSolid_hemiRings_4_cylRings_1_segments_12_halfLen_3.25_useTexCoords_false_useTangents_false + for (const auto& primNamePrefix : primNamePrefixes) { + if (filepath.size() < primNamePrefix.size()) { + continue; + } + + if (filepath.prefix(primNamePrefix.size()) == primNamePrefix) { + return false; + } + } + + return true; + } + gfx::replay::NodeHandle loadAndCreateRenderAssetInstance( const esp::assets::AssetInfo& assetInfo, const esp::assets::RenderAssetInstanceCreationInfo& creation) override { @@ -62,16 +90,22 @@ BatchReplayRenderer::BatchReplayRenderer( // TODO is creation.lightSetupKey actually mapping to anything in the // replay file? + if (!isSupportedRenderAsset(creation.filepath)) { + ESP_WARNING() << "Unsupported render asset: " << creation.filepath; + return nullptr; + } + /* If no such name is known yet, add as a file */ if (!renderer_.hasNodeHierarchy(creation.filepath)) { ESP_WARNING() << creation.filepath << "not found in any composite file, loading from the filesystem"; - // TODO asserts might be TOO BRUTAL? - CORRADE_INTERNAL_ASSERT_OUTPUT( + + ESP_CHECK( renderer_.addFile(creation.filepath, gfx_batch::RendererFileFlag::Whole | - gfx_batch::RendererFileFlag::GenerateMipmap)); + gfx_batch::RendererFileFlag::GenerateMipmap), + "addFile failed for " << creation.filepath); CORRADE_INTERNAL_ASSERT(renderer_.hasNodeHierarchy(creation.filepath)); } @@ -224,6 +258,9 @@ void BatchReplayRenderer::doRender( "with a standalone renderer", ); static_cast(*renderer_).draw(); + // todo: integrate debugLineRender_->flushLines + CORRADE_INTERNAL_ASSERT(!debugLineRender_); + for (int envIndex = 0; envIndex != envs_.size(); ++envIndex) { const auto rectangle = Mn::Range2Di::fromSize( renderer_->tileSize() * @@ -242,6 +279,25 @@ void BatchReplayRenderer::doRender( "a standalone renderer", ); renderer_->draw(framebuffer); + + if (debugLineRender_) { + framebuffer.bind(); + constexpr unsigned envIndex = 0; + auto projCamMatrix = renderer_->camera(envIndex); + debugLineRender_->flushLines(projCamMatrix, renderer_->tileSize()); + } +} + +esp::geo::Ray BatchReplayRenderer::doUnproject( + CORRADE_UNUSED unsigned envIndex, + const Mn::Vector2i& viewportPosition) { + // temp stub implementation: produce a placeholder ray that varies with + // viewportPosition + return esp::geo::Ray( + {static_cast(viewportPosition.x()) / renderer_->tileSize().x(), + 0.5f, + static_cast(viewportPosition.y()) / renderer_->tileSize().y()}, + {0.f, -1.f, 0.f}); } const void* BatchReplayRenderer::getCudaColorBufferDevicePointer() { diff --git a/src/esp/sim/BatchReplayRenderer.h b/src/esp/sim/BatchReplayRenderer.h index 7f113b591b..d7af77e9ea 100644 --- a/src/esp/sim/BatchReplayRenderer.h +++ b/src/esp/sim/BatchReplayRenderer.h @@ -49,6 +49,9 @@ class BatchReplayRenderer : public AbstractReplayRenderer { void doRender(Magnum::GL::AbstractFramebuffer& framebuffer) override; + esp::geo::Ray doUnproject(unsigned envIndex, + const Mn::Vector2i& viewportPosition) override; + /* If standalone_ is true, renderer_ contains a RendererStandalone. Has to be before the EnvironmentRecord array because Player calls gfx_batch::Renderer::clear() on destruction. */ diff --git a/src/esp/sim/ClassicReplayRenderer.cpp b/src/esp/sim/ClassicReplayRenderer.cpp index 67fa483e5b..d7bd5c6db3 100644 --- a/src/esp/sim/ClassicReplayRenderer.cpp +++ b/src/esp/sim/ClassicReplayRenderer.cpp @@ -31,6 +31,10 @@ ClassicReplayRenderer::ClassicReplayRenderer( resourceManager_ = std::make_unique(metadataMediator, flags); + // hack to get ReplicCAD non-baked stages to render correctly + resourceManager_->getShaderManager().setFallback( + esp::gfx::getDefaultLights()); + sceneManager_ = scene::SceneManager::create_unique(); class SceneGraphPlayerImplementation @@ -209,10 +213,10 @@ void ClassicReplayRenderer::doRender( auto& sceneGraph = getSceneGraph(envIndex); #ifdef ESP_BUILD_WITH_BACKGROUND_RENDERER - // todo: investigate flags (frustum culling?) - renderer_->enqueueAsyncDrawJob(visualSensor, sceneGraph, - imageViews[envIndex], - esp::gfx::RenderCamera::Flags{}); + renderer_->enqueueAsyncDrawJob( + visualSensor, sceneGraph, imageViews[envIndex], + esp::gfx::RenderCamera::Flags{ + gfx::RenderCamera::Flag::FrustumCulling}); #else // TODO what am I supposed to do here? CORRADE_ASSERT_UNREACHABLE("Not implemented yet, sorry.", ); @@ -240,8 +244,16 @@ void ClassicReplayRenderer::doRender( visualSensor.renderTarget().renderEnter(); auto& sceneGraph = getSceneGraph(envIndex); - renderer_->draw(*visualSensor.getRenderCamera(), sceneGraph, - esp::gfx::RenderCamera::Flags{}); + renderer_->draw( + *visualSensor.getRenderCamera(), sceneGraph, + esp::gfx::RenderCamera::Flags{gfx::RenderCamera::Flag::FrustumCulling}); + + if (envIndex == 0 && debugLineRender_) { + auto* camera = visualSensor.getRenderCamera(); + debugLineRender_->flushLines(camera->cameraMatrix(), + camera->projectionMatrix(), + camera->viewport()); + } visualSensor.renderTarget().renderExit(); @@ -270,6 +282,19 @@ ClassicReplayRenderer::getEnvironmentSensors(unsigned envIndex) { return env.sensorMap_; } +esp::geo::Ray ClassicReplayRenderer::doUnproject( + unsigned envIndex, + const Mn::Vector2i& viewportPosition) { + auto& sensorMap = getEnvironmentSensors(envIndex); + CORRADE_INTERNAL_ASSERT(sensorMap.size() == 1); + CORRADE_INTERNAL_ASSERT(sensorMap.begin()->second.get().isVisualSensor()); + auto& visualSensor = + static_cast(sensorMap.begin()->second.get()); + + return visualSensor.getRenderCamera()->unproject(viewportPosition, + /*normalized*/ true); +} + esp::scene::SceneGraph& ClassicReplayRenderer::getSceneGraph( unsigned envIndex) { CORRADE_INTERNAL_ASSERT(envIndex < envs_.size()); diff --git a/src/esp/sim/ClassicReplayRenderer.h b/src/esp/sim/ClassicReplayRenderer.h index c2955337e5..bb166f016a 100644 --- a/src/esp/sim/ClassicReplayRenderer.h +++ b/src/esp/sim/ClassicReplayRenderer.h @@ -56,6 +56,9 @@ class ClassicReplayRenderer : public AbstractReplayRenderer { std::map>& getEnvironmentSensors(unsigned envIndex); + esp::geo::Ray doUnproject(unsigned envIndex, + const Mn::Vector2i& viewportPosition) override; + private: unsigned doEnvironmentCount() const override; diff --git a/src/esp/sim/Simulator.cpp b/src/esp/sim/Simulator.cpp index 47b29e4d05..16337d860f 100644 --- a/src/esp/sim/Simulator.cpp +++ b/src/esp/sim/Simulator.cpp @@ -133,6 +133,10 @@ void Simulator::reconfigure(const SimulatorConfiguration& cfg) { resourceManager_->setMetadataMediator(metadataMediator_); } + // reset this count and number here because we are resetting (clearing) the + // scene + resourceManager_->resetDrawableCountAndNumFaces(); + if (!sceneManager_) { sceneManager_ = scene::SceneManager::create_unique(); } @@ -1345,5 +1349,36 @@ int Simulator::getAgentObservationSpaces( } return spaces.size(); } + +std::vector Simulator::getRuntimePerfStatNames() { + return {"num rigid", + "num active rigid", + "num artic", + "num active overlaps", + "num active contacts", + "num drawables", + "num faces"}; +} + +std::vector Simulator::getRuntimePerfStatValues() { + int drawableCount = 0; + int drawableNumFaces = 0; + std::tie(drawableCount, drawableNumFaces) = + resourceManager_->getDrawableCountAndNumFaces(); + + runtimePerfStatValues_.clear(); + runtimePerfStatValues_.push_back(physicsManager_->getNumRigidObjects()); + runtimePerfStatValues_.push_back(physicsManager_->checkActiveObjects()); + runtimePerfStatValues_.push_back(physicsManager_->getNumArticulatedObjects()); + runtimePerfStatValues_.push_back( + physicsManager_->getNumActiveOverlappingPairs()); + runtimePerfStatValues_.push_back( + physicsManager_->getNumActiveContactPoints()); + runtimePerfStatValues_.push_back(drawableCount); + runtimePerfStatValues_.push_back(drawableNumFaces); + + return runtimePerfStatValues_; +} + } // namespace sim } // namespace esp diff --git a/src/esp/sim/Simulator.h b/src/esp/sim/Simulator.h index b02fb51055..714a5db0bb 100644 --- a/src/esp/sim/Simulator.h +++ b/src/esp/sim/Simulator.h @@ -959,6 +959,24 @@ class Simulator { */ void setShadowMapsToDrawables(); + /** + * @brief Runtime perf stats are various scalars helpful for troubleshooting + * runtime perf. + * + * @return A vector of stat names; currently, this is constant so it can be + * called once at startup. See also getRuntimePerfStatValues. + */ + std::vector getRuntimePerfStatNames(); + + /** + * @brief Runtime perf stats are various scalars helpful for troubleshooting + * runtime perf. + * + * @return a vector of stat values. Stat values generally change after every + * physics step. See also getRuntimePerfStatNames. + */ + std::vector getRuntimePerfStatValues(); + protected: Simulator() = default; @@ -1138,6 +1156,8 @@ class Simulator { std::shared_ptr debugLineRender_; + std::vector runtimePerfStatValues_; + ESP_SMART_POINTERS(Simulator) }; diff --git a/src/shaders/pbr.frag b/src/shaders/pbr.frag index 7c16de0412..8c5e858cae 100644 --- a/src/shaders/pbr.frag +++ b/src/shaders/pbr.frag @@ -57,7 +57,8 @@ uniform MaterialData Material; #if defined(BASECOLOR_TEXTURE) uniform sampler2D BaseColorTexture; #endif -#if defined(METALLIC_TEXTURE) || defined(ROUGHNESS_TEXTURE) +#if defined(METALLIC_TEXTURE) || defined(ROUGHNESS_TEXTURE) || \ + defined(NONE_ROUGHNESS_METALLIC_TEXTURE) uniform sampler2D MetallicRoughnessTexture; #endif #if defined(NORMAL_TEXTURE) @@ -311,12 +312,12 @@ void main() { #endif float roughness = Material.roughness; -#if defined(ROUGHNESS_TEXTURE) +#if defined(ROUGHNESS_TEXTURE) || defined(NONE_ROUGHNESS_METALLIC_TEXTURE) roughness *= texture(MetallicRoughnessTexture, texCoord).g; #endif float metallic = Material.metallic; -#if defined(METALLIC_TEXTURE) +#if defined(METALLIC_TEXTURE) || defined(NONE_ROUGHNESS_METALLIC_TEXTURE) metallic *= texture(MetallicRoughnessTexture, texCoord).b; #endif diff --git a/src/tests/AttributesConfigsTest.cpp b/src/tests/AttributesConfigsTest.cpp index 88d153d835..6f03eada9f 100644 --- a/src/tests/AttributesConfigsTest.cpp +++ b/src/tests/AttributesConfigsTest.cpp @@ -135,6 +135,13 @@ struct AttributesConfigsTest : Cr::TestSuite::Tester { void testSceneInstanceAttrVals( std::shared_ptr sceneInstAttr); + + /** + * @brief This test will verify that the root-level scene instance user + * defined attribute values are as expected. + */ + void testSceneInstanceRootUserDefinedAttrVals( + std::shared_ptr userAttrs); /** * @brief This test will verify that the Stage attributes' managers' JSON * loading process is working as expected. @@ -165,6 +172,7 @@ struct AttributesConfigsTest : Cr::TestSuite::Tester { // test member vars + esp::metadata::MetadataMediator::uptr MM = nullptr; esp::logging::LoggingContext loggingContext_; AttrMgrs::LightLayoutAttributesManager::ptr lightLayoutAttributesManager_ = nullptr; @@ -179,7 +187,7 @@ struct AttributesConfigsTest : Cr::TestSuite::Tester { AttributesConfigsTest::AttributesConfigsTest() { // set up a default simulation config to initialize MM auto cfg = esp::sim::SimulatorConfiguration{}; - auto MM = MetadataMediator::create(cfg); + MM = MetadataMediator::create_unique(cfg); // get attributes managers for default dataset lightLayoutAttributesManager_ = MM->getLightLayoutAttributesManager(); objectAttributesManager_ = MM->getObjectAttributesManager(); @@ -518,6 +526,16 @@ void AttributesConfigsTest::testLightJSONLoad() { } // AttributesManagers_LightJSONLoadTest +void AttributesConfigsTest::testSceneInstanceRootUserDefinedAttrVals( + std::shared_ptr userAttrs) { + // test scene instance attributes-level user config vals + testUserDefinedConfigVals(userAttrs, 4, "scene instance defined string", true, + 99, 9.1, Mn::Vector2(1.3f, 2.4f), + Mn::Vector3(12.3, 32.5, 25.07), + Mn::Quaternion({3.2f, 2.6f, 5.1f}, 0.3f), + Mn::Vector4(13.5f, 14.6f, 15.7f, 16.9f)); +} // AttributesConfigsTest::testSceneInstanceRootUserDefinedAttrVals + void AttributesConfigsTest::testSceneInstanceAttrVals( std::shared_ptr sceneAttr) { @@ -532,11 +550,12 @@ void AttributesConfigsTest::testSceneInstanceAttrVals( CORRADE_COMPARE(sceneAttr->getSemanticSceneHandle(), "test_semantic_descriptor_path1"); // test scene instance attributes-level user config vals - testUserDefinedConfigVals( - sceneAttr->getUserConfiguration(), 4, "scene instance defined string", - true, 99, 9.1, Mn::Vector2(1.3f, 2.4f), Mn::Vector3(12.3, 32.5, 25.07), - Mn::Quaternion({3.2f, 2.6f, 5.1f}, 0.3f), - Mn::Vector4(13.5f, 14.6f, 15.7f, 16.9f)); + testSceneInstanceRootUserDefinedAttrVals(sceneAttr->getUserConfiguration()); + + // test scene instanct attributes-level user config vals retrieved from MM + // directly + testSceneInstanceRootUserDefinedAttrVals( + MM->getSceneInstanceUserConfiguration(sceneAttr->getHandle())); // verify objects auto objectInstanceList = sceneAttr->getObjectInstances(); diff --git a/src/tests/BatchReplayRendererTest.cpp b/src/tests/BatchReplayRendererTest.cpp index 6f5cd4095c..9af01274e7 100644 --- a/src/tests/BatchReplayRendererTest.cpp +++ b/src/tests/BatchReplayRendererTest.cpp @@ -44,6 +44,7 @@ struct BatchReplayRendererTest : Cr::TestSuite::Tester { explicit BatchReplayRendererTest(); void testIntegration(); + void testUnproject(); const Magnum::Float maxThreshold = 255.f; const Magnum::Float meanThreshold = 0.75f; @@ -65,6 +66,19 @@ Mn::MutableImageView2D getRGBView(int width, return view; } +std::vector getDefaultSensorSpecs( + const std::string& sensorName = "my_rgb") { + auto pinholeCameraSpec = esp::sensor::CameraSensorSpec::create(); + pinholeCameraSpec->sensorSubType = esp::sensor::SensorSubType::Pinhole; + pinholeCameraSpec->sensorType = esp::sensor::SensorType::Color; + pinholeCameraSpec->position = {0.0f, 0.f, 0.0f}; + pinholeCameraSpec->resolution = {512, 384}; + pinholeCameraSpec->uuid = sensorName; + std::vector sensorSpecifications = { + pinholeCameraSpec}; + return sensorSpecifications; +} + const struct { const char* name; Cr::Containers::Pointer (*create)( @@ -83,8 +97,63 @@ const struct { BatchReplayRendererTest::BatchReplayRendererTest() { addInstancedTests({&BatchReplayRendererTest::testIntegration}, Cr::Containers::arraySize(TestIntegrationData)); + + // temp only enable testUnproject for classic + addInstancedTests({&BatchReplayRendererTest::testUnproject}, 1); + // addInstancedTests({&BatchReplayRendererTest::testUnproject}, + // Cr::Containers::arraySize(TestIntegrationData)); } // ctor +// test recording and playback through the simulator interface +void BatchReplayRendererTest::testUnproject() { + auto&& data = TestIntegrationData[testCaseInstanceId()]; + setTestCaseDescription(data.name); + + std::vector sensorSpecifications; + auto pinholeCameraSpec = esp::sensor::CameraSensorSpec::create(); + pinholeCameraSpec->sensorSubType = esp::sensor::SensorSubType::Pinhole; + pinholeCameraSpec->sensorType = esp::sensor::SensorType::Color; + pinholeCameraSpec->position = {1.0f, 2.f, 3.0f}; + pinholeCameraSpec->resolution = {512, 384}; + pinholeCameraSpec->uuid = "my_rgb"; + sensorSpecifications = {pinholeCameraSpec}; + + ReplayRendererConfiguration batchRendererConfig; + batchRendererConfig.sensorSpecifications = std::move(sensorSpecifications); + batchRendererConfig.numEnvironments = 1; + { + Cr::Containers::Pointer renderer = + data.create(batchRendererConfig); + + const int h = pinholeCameraSpec->resolution.x(); + const int w = pinholeCameraSpec->resolution.y(); + constexpr int envIndex = 0; + auto ray = renderer->unproject(envIndex, {0, 0}); + CORRADE_COMPARE(Mn::Vector3(ray.origin), + Mn::Vector3(pinholeCameraSpec->position)); + CORRADE_COMPARE(Mn::Vector3(ray.direction), + Mn::Vector3(-0.51544, 0.68457, -0.51544)); + + // Ug, these tests reveal an off-by-one bug in our implementation in + // RenderCamera::unproject. Depending on your convention, we would expect + // some of these various corners to have exactly mirrored results, but they + // don't. + ray = renderer->unproject(envIndex, {w, 0}); + CORRADE_COMPARE(Mn::Vector3(ray.direction), + Mn::Vector3(0.51544, 0.68457, -0.51544)); + ray = renderer->unproject(envIndex, {w - 1, 0}); + CORRADE_COMPARE(Mn::Vector3(ray.direction), + Mn::Vector3(0.513467, 0.68551, -0.51615)); + + ray = renderer->unproject(envIndex, {0, h}); + CORRADE_COMPARE(Mn::Vector3(ray.direction), + Mn::Vector3(-0.51355, -0.68740, -0.51355)); + ray = renderer->unproject(envIndex, {0, h - 1}); + CORRADE_COMPARE(Mn::Vector3(ray.direction), + Mn::Vector3(-0.51449, -0.68599, -0.51450)); + } +} + // test recording and playback through the simulator interface void BatchReplayRendererTest::testIntegration() { auto&& data = TestIntegrationData[testCaseInstanceId()]; @@ -144,19 +213,8 @@ void BatchReplayRendererTest::testIntegration() { serKeyframes.emplace_back(std::move(serKeyframe)); } - std::vector sensorSpecifications; - { - auto pinholeCameraSpec = esp::sensor::CameraSensorSpec::create(); - pinholeCameraSpec->sensorSubType = esp::sensor::SensorSubType::Pinhole; - pinholeCameraSpec->sensorType = esp::sensor::SensorType::Color; - pinholeCameraSpec->position = {0.0f, 0.f, 0.0f}; - pinholeCameraSpec->resolution = {512, 384}; - pinholeCameraSpec->uuid = sensorName; - sensorSpecifications = {pinholeCameraSpec}; - } - ReplayRendererConfiguration batchRendererConfig; - batchRendererConfig.sensorSpecifications = std::move(sensorSpecifications); + batchRendererConfig.sensorSpecifications = getDefaultSensorSpecs(sensorName); batchRendererConfig.numEnvironments = numEnvs; { Cr::Containers::Pointer renderer = diff --git a/src/tests/GfxReplayTest.cpp b/src/tests/GfxReplayTest.cpp index 2908386de8..f2a113c566 100644 --- a/src/tests/GfxReplayTest.cpp +++ b/src/tests/GfxReplayTest.cpp @@ -522,7 +522,7 @@ void GfxReplayTest::testLightIntegration() { const LightInfo pointLight2{ {0.0f, 1.2f, -4.0f, 1.0f}, {4.0, 4.0, 4.0}, LightPositionModel::Object}; const LightInfo dirLight{ - {0.1f, 0.2f, -0.3f, 0.0f}, {0.0, 0.0, 1.0}, LightPositionModel::Global}; + {-0.1f, -0.2f, 0.3f, 0.0f}, {0.0, 0.0, 1.0}, LightPositionModel::Global}; const LightSetup lightSetup0{pointLight0, pointLight1}; const LightSetup lightSetup1{pointLight2}; const LightSetup lightSetup2{dirLight}; diff --git a/src/tests/MetadataMediatorTest.cpp b/src/tests/MetadataMediatorTest.cpp index 07d40a9516..0ba6a8640f 100644 --- a/src/tests/MetadataMediatorTest.cpp +++ b/src/tests/MetadataMediatorTest.cpp @@ -454,7 +454,7 @@ void MetadataMediatorTest::testDataset1() { ESP_WARNING() << "Starting testDataset1 : test LoadStages"; const auto& stageAttributesMgr = MM_->getStageAttributesManager(); int numStageHandles = stageAttributesMgr->getNumObjects(); - // shoudld be 6 : one for default NONE stage, glob lookup yields 2 stages + + // should be 6 : one for default NONE stage, glob lookup yields 2 stages + // 2 modified and 1 new stage in scene dataset config CORRADE_COMPARE(numStageHandles, 6); // end test LoadStages @@ -493,16 +493,16 @@ void MetadataMediatorTest::testDataset1() { ESP_WARNING() << "Starting test LoadArticulatedObjects"; namespace Dir = Cr::Utility::Path; - // verify # of urdf filepaths loaded - should be 6; + // verify # of urdf filepaths loaded - should be 7; const std::map& urdfTestFilenames = MM_->getArticulatedObjectModelFilenames(); - CORRADE_COMPARE(urdfTestFilenames.size(), 6); + CORRADE_COMPARE(urdfTestFilenames.size(), 7); // test that each stub name key corresponds to the actual file name passed // through the key making process for (std::map::const_iterator iter = urdfTestFilenames.begin(); iter != urdfTestFilenames.end(); ++iter) { - // TODO replace when model intherits from AbstractManagedObject and + // TODO replace when model inherits from AbstractManagedObject and // instances proper key synth methods. const std::string shortHandle = Dir::splitExtension( @@ -553,7 +553,7 @@ void MetadataMediatorTest::testDatasetDelete() { // verify not nullptr CORRADE_VERIFY(stageAttrMgr_DS1); - // load datsaet 0 and make active + // load dataset 0 and make active initDataset0(); // get new active dataset const std::string nameDS0 = MM_->getActiveSceneDatasetName(); diff --git a/src/tests/ResourceManagerTest.cpp b/src/tests/ResourceManagerTest.cpp index c286ecb682..9f2bd8c4f3 100644 --- a/src/tests/ResourceManagerTest.cpp +++ b/src/tests/ResourceManagerTest.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "esp/assets/MeshData.h" @@ -229,7 +230,7 @@ void buildMaterialIDs(const esp::assets::MeshTransformNode& root, * ResourceManager. We need to rebuild every test so that we force reload of * assets. */ -void testAssetTypeMatch(int specifiedAssetType, +void testAssetTypeMatch(ObjectInstanceShaderType specifiedAssetType, const esp::assets::AssetInfo& info, std::shared_ptr MM) { ResourceManager resourceManager(MM); @@ -247,16 +248,18 @@ void testAssetTypeMatch(int specifiedAssetType, ESP_DEBUG() << "# Materials specified in asset:" << matIDs.size(); CORRADE_COMPARE(matIDs.size(), 3); + int specifiedAssetTypeInt = static_cast(specifiedAssetType); // get shaderManager from RM and check material @ material ID auto& shaderManager = resourceManager.getShaderManager(); // all materials specified in the hierarchy should match the material for the // desired shadertype. for (const std::string id : matIDs) { int shaderTypeSpec = - shaderManager.get(id)->shaderTypeSpec; + shaderManager.get(id)->attribute( + "shaderTypeToUse"); ESP_DEBUG() << "mat ID : " << id << "type spec in mat :" << shaderTypeSpec - << " | spec in asset :" << specifiedAssetType; - CORRADE_COMPARE(shaderTypeSpec, specifiedAssetType); + << " | spec in asset :" << specifiedAssetTypeInt; + CORRADE_COMPARE(shaderTypeSpec, specifiedAssetTypeInt); } } // testAssetTypeMatch @@ -278,32 +281,29 @@ void ResourceManagerTest::testShaderTypeSpecification() { // force flat shading info.forceFlatShading = true; // object's material type is flat - testAssetTypeMatch(static_cast(ObjectInstanceShaderType::Flat), info, - MM); + testAssetTypeMatch(ObjectInstanceShaderType::Flat, info, MM); ESP_DEBUG() << "Testing Material type, which is PBR for this asset."; // enable lightig and use material type info.forceFlatShading = false; info.shaderTypeToUse = ObjectInstanceShaderType::Material; // object's material type is PBR - testAssetTypeMatch(static_cast(ObjectInstanceShaderType::PBR), info, MM); + testAssetTypeMatch(ObjectInstanceShaderType::PBR, info, MM); ESP_DEBUG() << "Testing PBR explicitly being set."; // force pbr info.shaderTypeToUse = ObjectInstanceShaderType::PBR; - testAssetTypeMatch(static_cast(ObjectInstanceShaderType::PBR), info, MM); + testAssetTypeMatch(ObjectInstanceShaderType::PBR, info, MM); ESP_DEBUG() << "Testing Phong explicitly being set."; // force phong info.shaderTypeToUse = ObjectInstanceShaderType::Phong; - testAssetTypeMatch(static_cast(ObjectInstanceShaderType::Phong), info, - MM); + testAssetTypeMatch(ObjectInstanceShaderType::Phong, info, MM); ESP_DEBUG() << "Testing Flat explicitly being set."; // force flat via shadertype info.shaderTypeToUse = ObjectInstanceShaderType::Flat; - testAssetTypeMatch(static_cast(ObjectInstanceShaderType::Flat), info, - MM); + testAssetTypeMatch(ObjectInstanceShaderType::Flat, info, MM); } // ResourceManagerTest::testFlatShaderTypeSpecification diff --git a/src/tests/SimTest.cpp b/src/tests/SimTest.cpp index feb28b7ca1..c0fe27d3fe 100644 --- a/src/tests/SimTest.cpp +++ b/src/tests/SimTest.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -12,10 +13,13 @@ #include #include #include +#include +#include "esp/assets/Asset.h" #include "esp/assets/ResourceManager.h" #include "esp/metadata/MetadataMediator.h" #include "esp/physics/RigidObject.h" +#include "esp/physics/objectManagers/ArticulatedObjectManager.h" #include "esp/physics/objectManagers/RigidObjectManager.h" #include "esp/sensor/CameraSensor.h" #include "esp/sim/Simulator.h" @@ -114,6 +118,11 @@ struct SimTest : Cr::TestSuite::Tester { Magnum::Float maxThreshold, Magnum::Float meanThreshold); + void addObjectsAndMakeObservation(Simulator& sim, + esp::sensor::CameraSensorSpec& cameraSpec, + const std::string& objTmpltHandle, + Observation& observation); + void basic(); void reconfigure(); void reset(); @@ -128,17 +137,20 @@ struct SimTest : Cr::TestSuite::Tester { void loadingObjectTemplates(); void buildingPrimAssetObjectTemplates(); void addObjectByHandle(); + void addObjectInvertedScale(); void addSensorToObject(); void createMagnumRenderingOff(); + void getRuntimePerfStats(); + void testArticulatedObjectSkinned(); esp::logging::LoggingContext loggingContext_; // TODO: remove outlier pixels from image and lower maxThreshold const Magnum::Float maxThreshold = 255.f; - LightSetup lightSetup1{{Magnum::Vector4{1.0f, 1.5f, 0.5f, 0.0f}, + LightSetup lightSetup1{{Magnum::Vector4{-1.0f, -1.5f, -0.5f, 0.0f}, {5.0, 5.0, 0.0}, LightPositionModel::Camera}}; - LightSetup lightSetup2{{Magnum::Vector4{0.0f, 0.5f, 1.0f, 0.0f}, + LightSetup lightSetup2{{Magnum::Vector4{0.0f, -0.5f, -1.0f, 0.0f}, {0.0, 5.0, 5.0}, LightPositionModel::Camera}}; }; // struct SimTest @@ -171,8 +183,14 @@ SimTest::SimTest() { &SimTest::loadingObjectTemplates, &SimTest::buildingPrimAssetObjectTemplates, &SimTest::addObjectByHandle, - &SimTest::addSensorToObject, - &SimTest::createMagnumRenderingOff}, Cr::Containers::arraySize(SimulatorBuilder) ); + &SimTest::addObjectInvertedScale, + &SimTest::addSensorToObject}, Cr::Containers::arraySize(SimulatorBuilder) ); + addTests({ + &SimTest::createMagnumRenderingOff, + &SimTest::getRuntimePerfStats}); +#ifdef ESP_BUILD_WITH_BULLET + addTests({&SimTest::testArticulatedObjectSkinned}); +#endif // clang-format on } void SimTest::basic() { @@ -271,11 +289,9 @@ void SimTest::checkPinholeCameraRGBAObservation( void SimTest::getSceneRGBAObservation() { ESP_DEBUG() << "Starting Test : getSceneRGBAObservation"; setTestCaseName(CORRADE_FUNCTION); - ESP_DEBUG() << "About to build simulator"; auto&& data = SimulatorBuilder[testCaseInstanceId()]; setTestCaseDescription(data.name); auto simulator = data.creator(*this, vangogh, esp::NO_LIGHT_KEY); - ESP_DEBUG() << "Built simulator"; checkPinholeCameraRGBAObservation(*simulator, "SimTestExpectedScene.png", maxThreshold, 0.75f); } @@ -654,7 +670,7 @@ void SimTest::buildingPrimAssetObjectTemplates() { CORRADE_COMPARE_AS(newHandle, origCylinderHandle, Cr::TestSuite::Compare::NotEqual); - // set bogus file directory, to validate that copy is reggistered + // set bogus file directory, to validate that copy is registered primAttr->setFileDirectory("test0"); // register new attributes int idx = assetAttribsMgr->registerObject(primAttr); @@ -727,6 +743,118 @@ void SimTest::addObjectByHandle() { CORRADE_VERIFY(obj->getID() != esp::ID_UNDEFINED); } +void SimTest::addObjectsAndMakeObservation( + Simulator& sim, + esp::sensor::CameraSensorSpec& cameraSpec, + const std::string& objTmpltHandle, + Observation& observation) { + auto rigidObjMgr = sim.getRigidObjectManager(); + // remove any existing objects + rigidObjMgr->removeAllObjects(); + // add and place first object + auto obj = rigidObjMgr->addObjectByHandle(objTmpltHandle, nullptr, + "custom_lighting_1"); + obj->setTranslation({-1.0f, 0.5f, -2.5f}); + + // add and place second object + auto otherObj = rigidObjMgr->addObjectByHandle(objTmpltHandle, nullptr, + "custom_lighting_2"); + otherObj->setTranslation({1.0f, 0.5f, -2.5f}); + + // Make Observation of constructed scene + CORRADE_VERIFY(sim.getAgentObservation(0, cameraSpec.uuid, observation)); + +} // SimTest::addObjectsAndMakeObservation + +void SimTest::addObjectInvertedScale() { + ESP_DEBUG() << "Starting Test : addObjectInvertedScale"; + auto&& data = SimulatorBuilder[testCaseInstanceId()]; + setTestCaseDescription(data.name); + auto simulator = data.creator(*this, planeStage, esp::NO_LIGHT_KEY); + auto rigidObjMgr = simulator->getRigidObjectManager(); + auto objAttrMgr = simulator->getObjectAttributesManager(); + // Add agent to take image + auto pinholeCameraSpec = CameraSensorSpec::create(); + pinholeCameraSpec->sensorSubType = esp::sensor::SensorSubType::Pinhole; + pinholeCameraSpec->sensorType = SensorType::Color; + pinholeCameraSpec->position = {0.0f, 1.5f, 0.0f}; + pinholeCameraSpec->resolution = {128, 128}; + + AgentConfiguration agentConfig{}; + agentConfig.sensorSpecifications = {pinholeCameraSpec}; + Agent::ptr agent = simulator->addAgent(agentConfig); + agent->setInitialState(AgentState{}); + + // Add 2 objects and take initial non-negative scaled observation + const auto objHandle = Cr::Utility::Path::join( + TEST_ASSETS, "objects/nested_box.object_config.json"); + + Observation expectedObservation; + addObjectsAndMakeObservation(*simulator, *pinholeCameraSpec, objHandle, + expectedObservation); + + // Make aa copy of observation buffer so future observations don't overwrite + // this one + Cr::Containers::Array obsCopy{ + Cr::NoInit, expectedObservation.buffer->data.size()}; + Cr::Utility::copy(expectedObservation.buffer->data, obsCopy); + + // File name of expected image for un-inverted and each axis-inverted image + const auto expectedScreenshotFile = Cr::Utility::Path::join( + screenshotDir, "SimTestInvertScaleImageExpected.png"); + // Make a ground truth image based on a copy of the observation buffer + const Mn::ImageView2D expectedImage{ + Mn::PixelFormat::RGBA8Unorm, + {pinholeCameraSpec->resolution[0], pinholeCameraSpec->resolution[1]}, + obsCopy}; + + // Verify non-negative scale scene is as expected + CORRADE_COMPARE_WITH( + expectedImage, expectedScreenshotFile, + (Mn::DebugTools::CompareImageToFile{maxThreshold, 0.01f})); + + // Create and test observations with scale negative in each of x, y and z + // directions + Cr::Containers::StringView testAxis[3]{"X_axis", "Y_axis", "Z_axis"}; + for (int i = 0; i < 3; ++i) { + CORRADE_ITERATION(testAxis[i]); + ObjectAttributes::ptr newObjAttr = + objAttrMgr->getObjectCopyByHandle(objHandle); + + Mn::Vector3 scale = newObjAttr->getScale(); + // change x, y, or z scale to be negative + scale[i] *= -1.0f; + + // Set modified scale + newObjAttr->setScale(scale); + // Register new object attributes with negative scale along a single axis + // using a new name + const std::string newObjHandle = + Cr::Utility::formatString("scale_{}_{}", i, objHandle); + objAttrMgr->registerObject(newObjAttr, newObjHandle); + + // Build object layout and retrieve observation + Observation newObservation; + addObjectsAndMakeObservation(*simulator, *pinholeCameraSpec, newObjHandle, + newObservation); + + const Mn::ImageView2D newImage{ + Mn::PixelFormat::RGBA8Unorm, + {pinholeCameraSpec->resolution[0], pinholeCameraSpec->resolution[1]}, + newObservation.buffer->data}; + + // Verify inverted scale scene is as expected compared to file. + CORRADE_COMPARE_WITH( + newImage, expectedScreenshotFile, + (Mn::DebugTools::CompareImageToFile{maxThreshold, 0.01f})); + + // Needed to make a buffer copy into the comparison image + CORRADE_COMPARE_WITH(newImage, expectedImage, + (Mn::DebugTools::CompareImage{maxThreshold, 0.01f})); + } + +} // SimTest::addObjectInvertedScale + void SimTest::addSensorToObject() { ESP_DEBUG() << "Starting Test : addSensorToObject"; auto&& data = SimulatorBuilder[testCaseInstanceId()]; @@ -892,6 +1020,121 @@ void SimTest::createMagnumRenderingOff() { CORRADE_VERIFY(!cameraSensor.getObservation(*simulator, observation)); } +void SimTest::getRuntimePerfStats() { + // create a simulator + SimulatorConfiguration simConfig{}; + simConfig.activeSceneName = vangogh; + simConfig.enablePhysics = true; + simConfig.physicsConfigFile = physicsConfigFile; + simConfig.overrideSceneLightDefaults = true; + auto simulator = Simulator::create_unique(simConfig); + + auto statNames = simulator->getRuntimePerfStatNames(); + + constexpr auto numRigidIdx = 0; + constexpr auto drawCountIdx = 5; + constexpr auto drawFacesIdx = 6; + CORRADE_COMPARE(statNames[numRigidIdx], "num rigid"); + CORRADE_COMPARE(statNames[drawCountIdx], "num drawables"); + CORRADE_COMPARE(statNames[drawFacesIdx], "num faces"); + + auto statValues = simulator->getRuntimePerfStatValues(); + + CORRADE_COMPARE(statValues[numRigidIdx], 0); + // magic numbers here correspond to the contents of the vangogh 3D asset + CORRADE_COMPARE(statValues[drawCountIdx], 15); + CORRADE_COMPARE(statValues[drawFacesIdx], 11272); + + { + auto objAttrMgr = simulator->getObjectAttributesManager(); + objAttrMgr->loadAllJSONConfigsFromPath( + Cr::Utility::Path::join(TEST_ASSETS, "objects/nested_box"), true); + auto rigidObjMgr = simulator->getRigidObjectManager(); + auto objs = objAttrMgr->getObjectHandlesBySubstring("nested_box"); + rigidObjMgr->addObjectByHandle(objs[0]); + } + + statNames = simulator->getRuntimePerfStatNames(); + statValues = simulator->getRuntimePerfStatValues(); + + CORRADE_COMPARE(statValues[numRigidIdx], 1); + // magic numbers here correspond to the contents of the vangogh and nested_box + // 3D assets + CORRADE_COMPARE(statValues[drawCountIdx], 17); + CORRADE_COMPARE(statValues[drawFacesIdx], 11296); + + simConfig.activeSceneName = esp::assets::EMPTY_SCENE; + simulator->reconfigure(simConfig); + + statValues = simulator->getRuntimePerfStatValues(); + + CORRADE_COMPARE(statValues[numRigidIdx], 0); + CORRADE_COMPARE(statValues[drawCountIdx], 0); + CORRADE_COMPARE(statValues[drawFacesIdx], 0); +} + } // namespace +void SimTest::testArticulatedObjectSkinned() { + ESP_DEBUG() << "Starting Test : testArticulatedObjectSkinned"; + + const std::string urdfFile = + Cr::Utility::Path::join(TEST_ASSETS, "urdf/skinned_prism.urdf"); + + // create a simulator + SimulatorConfiguration simConfig{}; + simConfig.activeSceneName = ""; + simConfig.enablePhysics = true; + simConfig.physicsConfigFile = physicsConfigFile; + simConfig.createRenderer = true; + auto simulator = Simulator::create_unique(simConfig); + auto aoManager = simulator->getArticulatedObjectManager(); + + CORRADE_COMPARE(aoManager->getNumObjects(), 0); + auto ao = aoManager->addArticulatedObjectFromURDF(urdfFile); + CORRADE_COMPARE(aoManager->getNumObjects(), 1); + + CORRADE_COMPARE(ao->getSceneNode()->getSemanticId(), 100); + + CORRADE_VERIFY(ao); + CORRADE_COMPARE(ao->getNumLinks(), 4); + + const auto linkIds = ao->getLinkIdsWithBase(); + + auto linkA = ao->getLink(linkIds[0]); + CORRADE_VERIFY(linkA->linkName == "A"); + auto linkB = ao->getLink(linkIds[1]); + CORRADE_VERIFY(linkB->linkName == "B"); + auto linkC = ao->getLink(linkIds[2]); + CORRADE_VERIFY(linkC->linkName == "C"); + auto linkD = ao->getLink(linkIds[3]); + CORRADE_VERIFY(linkD->linkName == "D"); + auto linkE = ao->getLink(linkIds[4]); + CORRADE_VERIFY(linkE->linkName == "E"); + + ao->setTranslation({1.f, -3.f, -6.f}); + + checkPinholeCameraRGBAObservation( + *simulator, "SimTestSkinnedAOInitialPose.png", maxThreshold, 0.71f); + + const auto rot = Mn::Quaternion::rotation( + Mn::Deg(25.f), Mn::Vector3(0.f, 1.f, 0.f).normalized()); + std::vector jointPos{}; + for (int i = 0; i < 4; ++i) { + const auto rotData = rot.data(); + jointPos.push_back(rotData[0]); // x + jointPos.push_back(rotData[1]); // y + jointPos.push_back(rotData[2]); // z + jointPos.push_back(rotData[3]); // w + } + ao->setJointPositions(jointPos); + + checkPinholeCameraRGBAObservation(*simulator, "SimTestSkinnedAOPose.png", + maxThreshold, 0.71f); + + aoManager->removeAllObjects(); + CORRADE_COMPARE(aoManager->getNumObjects(), 0); + +} // SimTest::testArticulatedObjectSkinned + CORRADE_TEST_MAIN(SimTest) diff --git a/src/utils/viewer/default_light_override.lighting_config.json b/src/utils/viewer/default_light_override.lighting_config.json index 0bde9e6950..8ce371abc1 100644 --- a/src/utils/viewer/default_light_override.lighting_config.json +++ b/src/utils/viewer/default_light_override.lighting_config.json @@ -2,32 +2,32 @@ "lights": { "forward": { "type": "directional", - "direction": [0, 0.5, 1], + "direction": [0, -0.5, -1], "intensity": 2.4, "color": [0.93, 0.98, 1] }, "backward": { "type": "directional", - "direction": [0, 0.5, -1], + "direction": [0, -0.5, 1], "intensity": 2.4, "color": [0.93, 0.98, 1] }, "top_left": { "type": "directional", "direction": [ - -1, 1, 0], + 1, -1, 0], "intensity": 2.0, "color": [0.93, 0.98, 1] }, "top_right": { "type": "directional", - "direction": [1, 1, 0], + "direction": [-1, -1, 0], "intensity": 2.0, "color": [0.93, 0.98, 1] }, "bottom": { "type": "directional", - "direction": [0, -1, 0], + "direction": [0, 1, 0], "intensity": 0.64, "color": [0.93, 0.98, 1] } diff --git a/src_python/habitat_sim/simulator.py b/src_python/habitat_sim/simulator.py index b2c8066ba7..d2f7943206 100755 --- a/src_python/habitat_sim/simulator.py +++ b/src_python/habitat_sim/simulator.py @@ -242,7 +242,6 @@ def _config_pathfinder(self, config: Configuration) -> None: or ( not self.pathfinder.is_loaded and config.sim_cfg.scene_id.lower() != "none" - and config.sim_cfg.create_renderer ) ): logger.info(