OpenXR-based VR integration

simulate/CMakeLists.txt

@@ -27,6 +27,16 @@ set(MUJOCO_DEP_VERSION_lodepng
     CACHE STRING "Version of `lodepng` to be fetched."
 )
 
+set(MUJOCO_URL_openxr
+    https://github.com/KhronosGroup/OpenXR-SDK-Source/archive/refs/tags/release-1.0.34.zip
+    CACHE STRING "Link to fetch `openxr`."
+)
+
+set(MUJOCO_URL_HASH_openxr
+    924a94a2da0b5ef8e82154c623d88644 # 1.0.34
+    CACHE STRING "URL hash to check fetched `openxr`."
+)
+
 project(
   mujoco_simulate
   VERSION 3.2.5
@@ -43,6 +53,7 @@ endif()
 
 option(SIMULATE_BUILD_EXECUTABLE "Build the simulate executable binary." ON)
 option(SIMULATE_GLFW_DYNAMIC_SYMBOLS "Whether to resolve GLFW symbols dynamically." OFF)
+option(SIMULATE_BUILD_XR "Build XR-enabled simulate executable binary." OFF)
 
 # Check if we are building as standalone project.
 set(SIMULATE_STANDALONE OFF)
@@ -79,6 +90,7 @@ if(MUJOCO_HARDEN)
   endif()
 endif()
 
+
 # Fetch lodepng dependency.
 if(NOT TARGET lodepng)
   FetchContent_Declare(
@@ -100,6 +112,19 @@ if(NOT TARGET lodepng)
   endif()
 endif()
 
+
+# Fetch OpenXR dependency.
+if(NOT TARGET openxr)
+  FetchContent_Declare(
+      openxr
+      URL_HASH MD5=${MUJOCO_URL_HASH_openxr}
+      URL ${MUJOCO_URL_openxr}
+          SOURCE_DIR
+          openxr
+  )
+  FetchContent_MakeAvailable(OpenXR)
+endif()
+
 # Simulate library
 add_library(platform_ui_adapter OBJECT)
 target_sources(
@@ -131,11 +156,35 @@ target_sources(
   PUBLIC simulate.h
   PRIVATE simulate.cc array_safety.h
 )
-target_include_directories(libsimulate PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_compile_options(libsimulate PRIVATE ${MUJOCO_SIMULATE_COMPILE_OPTIONS})
+target_include_directories(libsimulate PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_link_libraries(libsimulate PUBLIC lodepng mujoco::platform_ui_adapter mujoco::mujoco)
 target_link_options(libsimulate PRIVATE ${MUJOCO_SIMULATE_LINK_OPTIONS})
 
+if(SIMULATE_BUILD_XR)
+  set(SIMULATE_OPENXR_HEADERS
+      simulate_xr/simulate_xr.h)
+  set(SIMULATE_OPENXR_SOURCES
+      simulate_xr/simulate_xr.cc)
+  set(SIMULATE_OPENXR_GLAD
+      simulate_xr/glad/glad.h simulate_xr/glad/glad.c)
+  target_include_directories(libsimulate PUBLIC
+      ${SIMULATE_OPENXR_HEADERS} ${SIMULATE_OPENXR_SOURCES}
+      # From OpenXR repo
+      "${openxr_SOURCE_DIR}/src/common"
+      "${openxr_SOURCE_DIR}/external/include"
+  )
+  target_link_libraries(libsimulate PUBLIC openxr_loader)
+  target_sources(
+    libsimulate
+    PUBLIC ${SIMULATE_OPENXR_HEADERS}
+    PRIVATE ${SIMULATE_OPENXR_SOURCES} ${SIMULATE_OPENXR_GLAD}
+  )
+
+  # DEFINE to change the code
+  target_compile_definitions(mujoco PUBLIC mjBUILDSIMULATEXR)
+endif()
+
 if(APPLE)
   target_sources(libsimulate PRIVATE macos_gui.mm)
   target_link_libraries(libsimulate PUBLIC "-framework Cocoa")

simulate/cmake/SimulateDependencies.cmake

@@ -33,7 +33,20 @@ set(MUJOCO_DEP_VERSION_glfw3
     7482de6071d21db77a7236155da44c172a7f6c9e # 3.3.8
     CACHE STRING "Version of `glfw` to be fetched."
 )
+
+set(MUJOCO_URL_openxr
+    https://github.com/KhronosGroup/OpenXR-SDK-Source/archive/refs/tags/release-1.0.34.zip
+    CACHE STRING "Link to fetch `openxr`."
+)
+
+set(MUJOCO_URL_HASH_openxr
+    924a94a2da0b5ef8e82154c623d88644 # 1.0.34
+    CACHE STRING "URL hash to check fetched `openxr`."
+)
+
 mark_as_advanced(MUJOCO_DEP_VERSION_glfw3)
+mark_as_advanced(MUJOCO_URL_openxr)
+mark_as_advanced(MUJOCO_URL_HASH_openxr)
 
 find_package(Threads REQUIRED)
 

simulate/glfw_adapter.cc

@@ -1,3 +1,4 @@
+#include "glfw_adapter.h"
 // Copyright 2023 DeepMind Technologies Limited
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -50,7 +51,13 @@ GlfwAdapter::GlfwAdapter() {
   }
 
   // multisampling
+#ifdef mjBUILDSIMULATEXR
+  // window resampling must be disabled to allow different resolutions on monitor and VR headset
+  Glfw().glfwWindowHint(GLFW_SAMPLES, 0);
+#else
   Glfw().glfwWindowHint(GLFW_SAMPLES, 4);
+#endif // mjBUILDSIMULATEXR
+
   Glfw().glfwWindowHint(GLFW_VISIBLE, 1);
 
   // get video mode and save
@@ -161,7 +168,13 @@ void GlfwAdapter::SetVSync(bool enabled){
     core_video_.reset();
   }
 #else
+#ifdef mjBUILDSIMULATEXR
+  // might interfere with frame loop timing
+  // TODO(AS) consider disabling it somewhere else?
+  Glfw().glfwSwapInterval(0);
+#else // mjBUILDSIMULATEXR
   Glfw().glfwSwapInterval(enabled);
+#endif // mjBUILDSIMULATEXR
 #endif
 }
 

simulate/simulate.cc

@@ -2256,6 +2256,10 @@ void Simulate::LoadOnRenderThread() {
     }
   }
 
+#ifdef mjBUILDSIMULATEXR
+  if (simXr.is_initialized()) simXr.set_vis_params(this->m_);
+#endif  // mjBUILDSIMULATEXR
+
   // re-create scene and context
   mjv_makeScene(this->m_, &this->scn, kMaxGeom);
   if (this->is_passive_) {
@@ -2301,6 +2305,11 @@ void Simulate::LoadOnRenderThread() {
     mjv_updateSceneState(this->m_, this->d_, &this->opt, &this->scnstate_);
   }
 
+#ifdef mjBUILDSIMULATEXR
+  if (simXr.is_initialized())
+    simXr.set_scn_params(&this->scn);
+#endif // mjBUILDSIMULATEXR
+
   // set window title to model name
   if (this->m_->names) {
     char title[200] = "MuJoCo : ";
@@ -2476,7 +2485,23 @@ void Simulate::Render() {
   }
 
   // render scene
+#ifdef mjBUILDSIMULATEXR
+  if (simXr.is_initialized()) {
+    simXr.before_render(&this->scn, this->m_);
+    mjrRect rectXR = {0, 0, 0, 0};
+    rectXR.width = (int)simXr.width_render;
+    rectXR.height = (int)simXr.height;
+    // render in offscreen buffer
+    mjr_setBuffer(mjFB_OFFSCREEN, &this->platform_ui->mjr_context());
+    mjr_render(rectXR, &this->scn, &this->platform_ui->mjr_context());
+    simXr.after_render(&this->platform_ui->mjr_context());
+    mjr_setBuffer(mjFB_WINDOW, &this->platform_ui->mjr_context());
+  } else {
+    mjr_render(rect, &this->scn, &this->platform_ui->mjr_context());
+  }
+#else //mjBUILDSIMULATEXR
   mjr_render(rect, &this->scn, &this->platform_ui->mjr_context());
+#endif //mjBUILDSIMULATEXR
 
   // show last loading error
   if (this->load_error[0]) {
@@ -2595,6 +2620,14 @@ void Simulate::Render() {
 
 
 void Simulate::RenderLoop() {
+#ifdef mjBUILDSIMULATEXR
+  simXr.init();
+  if (simXr.is_initialized()) {
+    simXr.set_scn_params(&this->scn);
+    simXr.set_vis_params(this->m_);
+  }
+#endif  // mjBUILDSIMULATEXR
+
   // Set timer callback (milliseconds)
   mjcb_time = Timer;
 
@@ -2733,6 +2766,10 @@ void Simulate::RenderLoop() {
     mjv_freeSceneState(&scnstate_);
   }
 
+#ifdef mjBUILDSIMULATEXR
+  simXr.deinit();
+#endif  // mjBUILDSIMULATEXR
+
   this->exitrequest.store(2);
 }
 

simulate/simulate.h

@@ -30,6 +30,10 @@
 #include <mujoco/mujoco.h>
 #include "platform_ui_adapter.h"
 
+#ifdef mjBUILDSIMULATEXR
+  #include "simulate_xr/simulate_xr.h"
+#endif // mjBUILDSIMULATEXR
+
 namespace mujoco {
 
 // The viewer itself doesn't require a reentrant mutex, however we use it in
@@ -249,6 +253,10 @@ class Simulate {
   mjvFigure figsize = {};
   mjvFigure figsensor = {};
 
+#ifdef mjBUILDSIMULATEXR
+  SimulateXr simXr;
+#endif // mjBUILDSIMULATEXR
+
   // additional user-defined visualization geoms (used in passive mode)
   mjvScene* user_scn = nullptr;
   mjtByte user_scn_flags_prev_[mjNRNDFLAG];