Unprojection function in the ViewControl (#2852)

This function can be used to obtain a ray to perform raycasting queries
on the visualizer.

Co-authored-by: Rene Sepulveda <errissa@gmail.com>

cpp/open3d/visualization/visualizer/ViewControl.cpp

@@ -291,6 +291,34 @@ void ViewControl::ChangeWindowSize(int width, int height) {
     SetProjectionParameters();
 }
 
+geometry::Ray3D ViewControl::UnprojectPoint(double x, double y) const {
+    if (!window_width_ || !window_height_) {
+        return geometry::Ray3D(Eigen::Vector3d::Zero(),
+                               Eigen::Vector3d::Zero());
+    }
+
+    gl_util::GLMatrix4f pvm = projection_matrix_ * view_matrix_ * model_matrix_;
+    gl_util::GLMatrix4f inv = pvm.inverse();
+    gl_util::GLVector4f clipping_pos(2.0f * (float)x / window_width_ - 1.0f,
+                                     1.0f - 2.0f * (float)y / window_height_,
+                                     1.0f, 1.0f);
+    gl_util::GLVector4f homogeneous_pos = inv * clipping_pos;
+    homogeneous_pos /= homogeneous_pos[3];
+    Eigen::Vector3d world_pos = Eigen::Vector3d(
+            homogeneous_pos[0], homogeneous_pos[1], homogeneous_pos[2]);
+
+    if (GetProjectionType() == ProjectionType::Orthogonal) {
+        Eigen::Vector3d camera_axis = lookat_ - eye_;
+        camera_axis.normalize();
+        Eigen::Vector3d delta = world_pos - eye_;
+        double projection = delta.dot(camera_axis);
+        Eigen::Vector3d origin = world_pos - projection * camera_axis;
+        return geometry::Ray3D(origin, camera_axis);
+    }
+
+    return geometry::Ray3D(eye_, world_pos - eye_);
+}
+
 void ViewControl::Scale(double scale) {
     zoom_ = std::max(std::min(zoom_ + scale * ZOOM_STEP, ZOOM_MAX), ZOOM_MIN);
     SetProjectionParameters();

cpp/open3d/visualization/visualizer/ViewControl.h

@@ -29,6 +29,7 @@
 #include "open3d/camera/PinholeCameraParameters.h"
 #include "open3d/geometry/BoundingVolume.h"
 #include "open3d/geometry/Geometry.h"
+#include "open3d/geometry/Line3D.h"
 #include "open3d/visualization/utility/GLHelper.h"
 #include "open3d/visualization/visualizer/ViewParameters.h"
 
@@ -101,6 +102,13 @@ class ViewControl {
     virtual void ChangeFieldOfView(double step);
     virtual void ChangeWindowSize(int width, int height);
 
+    /// Function to unproject a point on the window and obtain a ray from the
+    /// camera to that point in 3D.
+    ///
+    /// \param x The coordinate of the point in x-axis, in pixels
+    /// \param y The coordinate of the point in y-axis, in pixels
+    geometry::Ray3D UnprojectPoint(double x, double y) const;
+
     /// Function to process scaling
     ///
     /// \param scale is the scale ratio.

cpp/open3d/visualization/visualizer/Visualizer.h

@@ -192,6 +192,7 @@ class Visualizer {
 
     /// Function to retrieve the associated ViewControl
     ViewControl &GetViewControl() { return *view_control_ptr_; }
+    const ViewControl &GetViewControl() const { return *view_control_ptr_; }
     /// Function to retrieve the associated RenderOption.
     RenderOption &GetRenderOption() { return *render_option_ptr_; }
     /// \brief Function to capture screen and store RGB in a float buffer.

cpp/pybind/visualization/visualizer.cpp

@@ -112,9 +112,11 @@ void pybind_visualizer(py::module &m) {
                  "reset_bounding_box"_a = true)
             .def("clear_geometries", &Visualizer::ClearGeometries,
                  "Function to clear geometries from the visualizer")
-            .def("get_view_control", &Visualizer::GetViewControl,
-                 "Function to retrieve the associated ``ViewControl``",
-                 py::return_value_policy::reference_internal)
+            .def(
+                    "get_view_control",
+                    [](Visualizer &vis) { return vis.GetViewControl(); },
+                    "Function to retrieve the associated ``ViewControl``",
+                    py::return_value_policy::reference_internal)
             .def("get_render_option", &Visualizer::GetRenderOption,
                  "Function to retrieve the associated ``RenderOption``",
                  py::return_value_policy::reference_internal)