In place switch to reversed Z rendering and offscreen 32 bit float depth buffers for improved camera depth accuracy

doc/changelog.rst

@@ -115,6 +115,9 @@ General
       attributes are specified. See the following
       `example model <https://github.com/deepmind/mujoco/blob/main/test/engine/testdata/vis_visualize/frustum.xml>`__.
     - Note that these attributes only take effect for offline rendering and do not affect interactive visualisation.
+21. Implemented reversed Z rendering and added enum ``mjtDepthMap`` with elements ``mjDEPTH_ZERONEAR`` and
+    ``mjDEPTH_ZEROFAR`` which can be used to set ``readDepthMap`` in ``mjrContext`` to control how the depth returned by
+    ``mjr_readPixels`` is mapped from ``znear`` to ``zfar``.
 
 Python bindings
 ^^^^^^^^^^^^^^^

include/mujoco/mjrender.h

@@ -39,6 +39,10 @@ typedef enum mjtFramebuffer_ {    // OpenGL framebuffer option
   mjFB_OFFSCREEN                  // offscreen buffer
 } mjtFramebuffer;
 
+typedef enum mjtDepthMap_ {       // depth mapping for `mjr_readPixels`
+  mjDEPTH_ZERONEAR    = 0,        // standard depth map; 0: znear, 1: zfar
+  mjDEPTH_ZEROFAR     = 1         // reversed depth map; 1: znear, 0: zfar
+} mjtDepthMap;
 
 typedef enum mjtFontScale_ {      // font scale, used at context creation
   mjFONTSCALE_50      = 50,       // 50% scale, suitable for low-res rendering
@@ -151,6 +155,9 @@ struct mjrContext_ {              // custom OpenGL context
 
   // pixel output format
   int     readPixelFormat;        // default color pixel format for mjr_readPixels
+
+  // depth output format
+  int     readDepthMap;           // depth mapping: mjDEPTH_ZERONEAR or mjDEPTH_ZEROFAR
 };
 typedef struct mjrContext_ mjrContext;
 

introspect/enums.py

@@ -621,6 +621,15 @@
              ('mjFB_OFFSCREEN', 1),
          ]),
      )),
+    ('mjtDepthMap',
+     EnumDecl(
+         name='mjtDepthMap',
+         declname='enum mjtDepthMap_',
+         values=dict([
+             ('mjDEPTH_ZERONEAR', 0),
+             ('mjDEPTH_ZEROFAR', 1),
+         ]),
+     )),
     ('mjtFontScale',
      EnumDecl(
          name='mjtFontScale',

introspect/structs.py

@@ -7126,6 +7126,11 @@
                  type=ValueType(name='int'),
                  doc='default color pixel format for mjr_readPixels',
              ),
+             StructFieldDecl(
+                 name='readDepthMap',
+                 type=ValueType(name='int'),
+                 doc='depth mapping: mjDEPTH_ZERONEAR or mjDEPTH_ZEROFAR',
+             ),
          ),
      )),
     ('mjuiState',

python/mujoco/render.cc

@@ -215,6 +215,7 @@ PYBIND11_MODULE(_render, pymodule) {
   X(windowDoublebuffer);
   X(currentBuffer);
   X(readPixelFormat);
+  X(readDepthMap);
 #undef X
 
 #define X(var)                      \

python/mujoco/renderer.py

@@ -86,6 +86,7 @@ def __init__(
     _render.mjr_setBuffer(
         _enums.mjtFramebuffer.mjFB_OFFSCREEN.value, self._mjr_context
     )
+    self._mjr_context.readDepthMap = _enums.mjtDepthMap.mjDEPTH_ZEROFAR
 
     # Default render flags.
     self._depth_rendering = False
@@ -138,7 +139,8 @@ def render(self, *, out: Optional[np.ndarray] = None) -> np.ndarray:
     """
     original_flags = self._scene.flags.copy()
 
-    if self._segmentation_rendering:
+    # Using segmented rendering for depth makes the calculated depth more accurate at far distances
+    if self._depth_rendering or self._segmentation_rendering:
       self._scene.flags[_enums.mjtRndFlag.mjRND_SEGMENT] = True
       self._scene.flags[_enums.mjtRndFlag.mjRND_IDCOLOR] = True
 
@@ -173,11 +175,33 @@ def render(self, *, out: Optional[np.ndarray] = None) -> np.ndarray:
       near = self._model.vis.map.znear * extent
       far = self._model.vis.map.zfar * extent
 
-      # Convert from [0 1] to depth in units of length, see links below:
-      # http://stackoverflow.com/a/6657284/1461210
-      # https://www.khronos.org/opengl/wiki/Depth_Buffer_Precision
-      out = near / (1 - out * (1 - near / far))
+      # Calculate OpenGL perspective matrix values in float32 precision
+      # so they are close to what glFrustum returns
+      # https://registry.khronos.org/OpenGL-Refpages/gl2.1/xhtml/glFrustum.xml
+      zfar  = np.float32(far)
+      znear = np.float32(near)
+      C = -(zfar + znear)/(zfar - znear)
+      D = -(np.float32(2)*zfar*znear)/(zfar - znear)
 
+      # In reverse Z mode the perspective matrix is transformed by the following
+      C = np.float32(-0.5)*C - np.float32(0.5)
+      D = np.float32(-0.5)*D
+
+      # We need 64 bits to convert Z from ndc to metric depth without noticeable losses in precision
+      out_64 = out.astype(np.float64)
+
+      # Undo OpenGL projection
+      # Note: We do not need to take action to convert from window coordinates to
+      # normalized device coordinates because in reversed Z mode the mapping
+      # is identity
+      out_64 = D / (out_64 + C)
+
+      # Cast result back to float32 for backwards compatibility
+      # This has a small accuracy cost
+      out[:] = out_64.astype(np.float32)
+
+      # Reset scene flags.
+      np.copyto(self._scene.flags, original_flags)
     elif self._segmentation_rendering:
       _render.mjr_readPixels(out, None, self._rect, self._mjr_context)
 

src/render/glad/glad.c

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 //
-// Based on the OpenGL loader generated by glad 0.1.35 on Mon Mar 21 12:42:33 2022.
+// Based on the OpenGL loader generated by glad 0.1.34 on Thu Jun 22 14:40:03 2023.
 //
 // The original generated code is distributed under CC0.
 // https://creativecommons.org/publicdomain/zero/1.0/
@@ -22,6 +22,8 @@
 // APIs: gl=1.5
 // Profile: compatibility
 // Extensions:
+//     GL_ARB_clip_control,
+//     GL_ARB_depth_buffer_float,
 //     GL_ARB_framebuffer_object,
 //     GL_ARB_seamless_cube_map,
 //     GL_ARB_vertex_buffer_object,
@@ -32,9 +34,9 @@
 // Reproducible: False
 //
 // Commandline:
-//     --profile="compatibility" --api="gl=1.5" --generator="c" --spec="gl" --extensions="GL_ARB_framebuffer_object,GL_ARB_seamless_cube_map,GL_ARB_vertex_buffer_object,GL_KHR_debug"
+//     --profile="compatibility" --api="gl=1.5" --generator="c" --spec="gl" --extensions="GL_ARB_clip_control,GL_ARB_depth_buffer_float,GL_ARB_framebuffer_object,GL_ARB_seamless_cube_map,GL_ARB_vertex_buffer_object,GL_KHR_debug"
 // Online:
-//     https://glad.dav1d.de/#profile=compatibility&language=c&specification=gl&loader=on&api=gl%3D1.5&extensions=GL_ARB_framebuffer_object&extensions=GL_ARB_seamless_cube_map&extensions=GL_ARB_vertex_buffer_object&extensions=GL_KHR_debug
+//     https://glad.dav1d.de/#profile=compatibility&language=c&specification=gl&loader=on&api=gl%3D1.5&extensions=GL_ARB_clip_control&extensions=GL_ARB_depth_buffer_float&extensions=GL_ARB_framebuffer_object&extensions=GL_ARB_seamless_cube_map&extensions=GL_ARB_vertex_buffer_object&extensions=GL_KHR_debug
 
 #if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic push
@@ -829,10 +831,13 @@ PFNGLWINDOWPOS3IPROC mjGlad_glWindowPos3i = NULL;
 PFNGLWINDOWPOS3IVPROC mjGlad_glWindowPos3iv = NULL;
 PFNGLWINDOWPOS3SPROC mjGlad_glWindowPos3s = NULL;
 PFNGLWINDOWPOS3SVPROC mjGlad_glWindowPos3sv = NULL;
+int mjGLAD_GL_ARB_clip_control = 0;
+int mjGLAD_GL_ARB_depth_buffer_float = 0;
 int mjGLAD_GL_ARB_framebuffer_object = 0;
 int mjGLAD_GL_ARB_seamless_cube_map = 0;
 int mjGLAD_GL_ARB_vertex_buffer_object = 0;
 int mjGLAD_GL_KHR_debug = 0;
+PFNGLCLIPCONTROLPROC mjGlad_glClipControl = NULL;
 PFNGLISRENDERBUFFERPROC mjGlad_glIsRenderbuffer = NULL;
 PFNGLBINDRENDERBUFFERPROC mjGlad_glBindRenderbuffer = NULL;
 PFNGLDELETERENDERBUFFERSPROC mjGlad_glDeleteRenderbuffers = NULL;
@@ -1355,6 +1360,10 @@ static void mjGlad_load_GL_VERSION_1_5(GLADloadproc load) {
   mjGlad_glGetBufferParameteriv = (PFNGLGETBUFFERPARAMETERIVPROC)load("glGetBufferParameteriv");
   mjGlad_glGetBufferPointerv = (PFNGLGETBUFFERPOINTERVPROC)load("glGetBufferPointerv");
 }
+static void mjGlad_load_GL_ARB_clip_control(GLADloadproc load) {
+  if(!mjGLAD_GL_ARB_clip_control) return;
+  mjGlad_glClipControl = (PFNGLCLIPCONTROLPROC)load("glClipControl");
+}
 static void mjGlad_load_GL_ARB_framebuffer_object(GLADloadproc load) {
   if(!mjGLAD_GL_ARB_framebuffer_object) return;
   mjGlad_glIsRenderbuffer = (PFNGLISRENDERBUFFERPROC)load("glIsRenderbuffer");
@@ -1419,6 +1428,8 @@ static void mjGlad_load_GL_KHR_debug(GLADloadproc load) {
 }
 static int mjGlad_find_extensionsGL(void) {
   if (!mjGlad_get_exts()) return 0;
+  mjGLAD_GL_ARB_clip_control = mjGlad_has_ext("GL_ARB_clip_control");
+  mjGLAD_GL_ARB_depth_buffer_float = mjGlad_has_ext("GL_ARB_depth_buffer_float");
   mjGLAD_GL_ARB_framebuffer_object = mjGlad_has_ext("GL_ARB_framebuffer_object");
   mjGLAD_GL_ARB_seamless_cube_map = mjGlad_has_ext("GL_ARB_seamless_cube_map");
   mjGLAD_GL_ARB_vertex_buffer_object = mjGlad_has_ext("GL_ARB_vertex_buffer_object");
@@ -1491,6 +1502,7 @@ int mjGladLoadGLUnsafe(void) {
     mjGlad_load_GL_VERSION_1_5(mjGlad_get_proc);
 
     if (!mjGlad_find_extensionsGL()) return 0;
+    mjGlad_load_GL_ARB_clip_control(mjGlad_get_proc);
     mjGlad_load_GL_ARB_framebuffer_object(mjGlad_get_proc);
     mjGlad_load_GL_ARB_vertex_buffer_object(mjGlad_get_proc);
     mjGlad_load_GL_KHR_debug(mjGlad_get_proc);

src/render/glad/glad.h

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 //
-// Based on the OpenGL loader generated by glad 0.1.35 on Mon Mar 21 12:42:33 2022.
+// Based on the OpenGL loader generated by glad 0.1.34 on Thu Jun 22 14:40:03 2023.
 //
 // The original generated code is distributed under CC0.
 // https://creativecommons.org/publicdomain/zero/1.0/
@@ -22,6 +22,8 @@
 // APIs: gl=1.5
 // Profile: compatibility
 // Extensions:
+//     GL_ARB_clip_control,
+//     GL_ARB_depth_buffer_float,
 //     GL_ARB_framebuffer_object,
 //     GL_ARB_seamless_cube_map,
 //     GL_ARB_vertex_buffer_object,
@@ -32,9 +34,9 @@
 // Reproducible: False
 //
 // Commandline:
-//     --profile="compatibility" --api="gl=1.5" --generator="c" --spec="gl" --extensions="GL_ARB_framebuffer_object,GL_ARB_seamless_cube_map,GL_ARB_vertex_buffer_object,GL_KHR_debug"
+//     --profile="compatibility" --api="gl=1.5" --generator="c" --spec="gl" --extensions="GL_ARB_clip_control,GL_ARB_depth_buffer_float,GL_ARB_framebuffer_object,GL_ARB_seamless_cube_map,GL_ARB_vertex_buffer_object,GL_KHR_debug"
 // Online:
-//     https://glad.dav1d.de/#profile=compatibility&language=c&specification=gl&loader=on&api=gl%3D1.5&extensions=GL_ARB_framebuffer_object&extensions=GL_ARB_seamless_cube_map&extensions=GL_ARB_vertex_buffer_object&extensions=GL_KHR_debug
+//     https://glad.dav1d.de/#profile=compatibility&language=c&specification=gl&loader=on&api=gl%3D1.5&extensions=GL_ARB_clip_control&extensions=GL_ARB_depth_buffer_float&extensions=GL_ARB_framebuffer_object&extensions=GL_ARB_seamless_cube_map&extensions=GL_ARB_vertex_buffer_object&extensions=GL_KHR_debug
 
 #ifndef MUJOCO_SRC_RENDER_GLAD_GLAD_H_
 #define MUJOCO_SRC_RENDER_GLAD_GLAD_H_
@@ -2304,6 +2306,15 @@ typedef void (APIENTRYP PFNGLGETBUFFERPOINTERVPROC)(GLenum target, GLenum pname,
 GLAPI PFNGLGETBUFFERPOINTERVPROC mjGlad_glGetBufferPointerv;
 #define glGetBufferPointerv mjGlad_glGetBufferPointerv
 #endif
+#define GL_LOWER_LEFT 0x8CA1
+#define GL_UPPER_LEFT 0x8CA2
+#define GL_NEGATIVE_ONE_TO_ONE 0x935E
+#define GL_ZERO_TO_ONE 0x935F
+#define GL_CLIP_ORIGIN 0x935C
+#define GL_CLIP_DEPTH_MODE 0x935D
+#define GL_DEPTH_COMPONENT32F 0x8CAC
+#define GL_DEPTH32F_STENCIL8 0x8CAD
+#define GL_FLOAT_32_UNSIGNED_INT_24_8_REV 0x8DAD
 #define GL_INVALID_FRAMEBUFFER_OPERATION 0x0506
 #define GL_FRAMEBUFFER_ATTACHMENT_COLOR_ENCODING 0x8210
 #define GL_FRAMEBUFFER_ATTACHMENT_COMPONENT_TYPE 0x8211
@@ -2487,6 +2498,17 @@ GLAPI PFNGLGETBUFFERPOINTERVPROC mjGlad_glGetBufferPointerv;
 #define GL_STACK_OVERFLOW_KHR 0x0503
 #define GL_STACK_UNDERFLOW_KHR 0x0504
 #define GL_DISPLAY_LIST 0x82E7
+#ifndef GL_ARB_clip_control
+#define GL_ARB_clip_control 1
+GLAPI int mjGLAD_GL_ARB_clip_control;
+typedef void (APIENTRYP PFNGLCLIPCONTROLPROC)(GLenum origin, GLenum depth);
+GLAPI PFNGLCLIPCONTROLPROC mjGlad_glClipControl;
+#define glClipControl mjGlad_glClipControl
+#endif
+#ifndef GL_ARB_depth_buffer_float
+#define GL_ARB_depth_buffer_float 1
+GLAPI int mjGLAD_GL_ARB_depth_buffer_float;
+#endif
 #ifndef GL_ARB_framebuffer_object
 #define GL_ARB_framebuffer_object 1
 GLAPI int mjGLAD_GL_ARB_framebuffer_object;

src/render/render_context.c

@@ -1062,7 +1062,7 @@ static void makeShadow(const mjModel* m, mjrContext* con) {
   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE, GL_COMPARE_R_TO_TEXTURE);
-  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LEQUAL);
+  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_GEQUAL);
   glTexParameteri(GL_TEXTURE_2D, GL_DEPTH_TEXTURE_MODE, GL_INTENSITY);
   glTexGeni(GL_S, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
   glTexGeni(GL_T, GL_TEXTURE_GEN_MODE, GL_EYE_LINEAR);
@@ -1126,11 +1126,13 @@ static void makeOff(mjrContext* con) {
     mju_error("Could not allocate offscreen depth and stencil buffer");
   }
   glBindRenderbuffer(GL_RENDERBUFFER, con->offDepthStencil);
+
+  GLenum depth_buffer_format = mjGLAD_GL_ARB_depth_buffer_float ? GL_DEPTH32F_STENCIL8 : GL_DEPTH24_STENCIL8;
   if (con->offSamples) {
-    glRenderbufferStorageMultisample(GL_RENDERBUFFER, con->offSamples, GL_DEPTH24_STENCIL8,
+    glRenderbufferStorageMultisample(GL_RENDERBUFFER, con->offSamples, depth_buffer_format,
                                      con->offWidth, con->offHeight);
   } else {
-    glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, con->offWidth, con->offHeight);
+    glRenderbufferStorage(GL_RENDERBUFFER, depth_buffer_format, con->offWidth, con->offHeight);
   }
   glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT,
                             GL_RENDERBUFFER, con->offDepthStencil);
@@ -1169,7 +1171,7 @@ static void makeOff(mjrContext* con) {
       mju_error("Could not allocate offscreen depth and stencil buffer_r");
     }
     glBindRenderbuffer(GL_RENDERBUFFER, con->offDepthStencil_r);
-    glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, con->offWidth, con->offHeight);
+    glRenderbufferStorage(GL_RENDERBUFFER, depth_buffer_format, con->offWidth, con->offHeight);
     glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT,
                               GL_RENDERBUFFER, con->offDepthStencil_r);
 
@@ -1535,7 +1537,6 @@ void mjr_makeContext_offSize(const mjModel* m, mjrContext* con, int fontscale,
   glPixelStorei(GL_PACK_ALIGNMENT, 1);
   glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
 
-  // free previous context
   mjr_freeContext(con);
 
   // no model: offscreen and font only
@@ -1607,6 +1608,9 @@ void mjr_makeContext_offSize(const mjModel* m, mjrContext* con, int fontscale,
 
   // set default color pixel format for mjr_readPixels
   con->readPixelFormat = GL_RGB;
+
+  // set default depth mapping for mjr_readPixels
+  con->readDepthMap = mjDEPTH_ZERONEAR;
 }
 
 
@@ -1830,17 +1834,17 @@ MJAPI void mjr_resizeOffscreen(int width, int height, mjrContext* con) {
 
   glBindRenderbuffer(GL_RENDERBUFFER, con->offDepthStencil);
   if (con->offSamples) {
-    glRenderbufferStorageMultisample(GL_RENDERBUFFER, con->offSamples, GL_DEPTH24_STENCIL8,
+    glRenderbufferStorageMultisample(GL_RENDERBUFFER, con->offSamples, GL_DEPTH32F_STENCIL8,
                                      con->offWidth, con->offHeight);
   } else {
-    glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, con->offWidth, con->offHeight);
+    glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH32F_STENCIL8, con->offWidth, con->offHeight);
   }
 
   if (con->offSamples) {
     glBindRenderbuffer(GL_RENDERBUFFER, con->offColor_r);
     glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA8, con->offWidth, con->offHeight);
 
     glBindRenderbuffer(GL_RENDERBUFFER, con->offDepthStencil_r);
-    glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, con->offWidth, con->offHeight);
+    glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH32F_STENCIL8, con->offWidth, con->offHeight);
   }
 }

src/render/render_gl2.c

@@ -115,6 +115,16 @@ void mjr_readPixels(unsigned char* rgb, float* depth,
     if (depth) {
       glReadPixels(viewport.left, viewport.bottom, viewport.width, viewport.height,
                    GL_DEPTH_COMPONENT, GL_FLOAT, depth);
+      if (con->readDepthMap == mjDEPTH_ZERONEAR) {
+        int N_pixels = viewport.width * viewport.height;
+        for (int i = 0; i < N_pixels; i++) depth[i] = 1.0 - depth[i]; // Reverse the reversed Z buffer
+      }
+      if (con->readDepthMap == mjDEPTH_ZEROFAR && !mjGLAD_GL_ARB_clip_control) {
+        mju_warning("mjDEPTH_ZEROFAR requested but ARB_clip_control unavailable, depth accuracy will be limited");
+      }
+      if (con->readDepthMap == mjDEPTH_ZEROFAR) {
+        mju_warning("mjDEPTH_ZEROFAR requested but window depth buffer precision may be limited");
+      }
     }
   }
 
@@ -159,6 +169,16 @@ void mjr_readPixels(unsigned char* rgb, float* depth,
     if (depth) {
       glReadPixels(viewport.left, viewport.bottom, viewport.width, viewport.height,
                    GL_DEPTH_COMPONENT, GL_FLOAT, depth);
+      if (con->readDepthMap == mjDEPTH_ZERONEAR) {
+        int N_pixels = viewport.width * viewport.height;
+        for (int i = 0; i < N_pixels; i++) depth[i] = 1.0 - depth[i]; // Reverse the reversed Z buffer
+      }
+      if (con->readDepthMap == mjDEPTH_ZEROFAR && !mjGLAD_GL_ARB_clip_control) {
+        mju_warning("mjDEPTH_ZEROFAR requested but ARB_clip_control unavailable, depth accuracy will be limited");
+      }
+      if (con->readDepthMap == mjDEPTH_ZEROFAR && !mjGLAD_GL_ARB_depth_buffer_float) {
+        mju_warning("mjDEPTH_ZEROFAR requested but ARB_depth_buffer_float unavailable, depth accuracy will be limited");
+      }
     }
 
     // restore currentBuffer