Feature/superpoint lightglue

.github/workflows/ci.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - main
+      - dev
 
 jobs:
   # Job 1: Linters for Pull Requests
@@ -28,7 +29,7 @@ jobs:
         run: flake8 parallax tests
         continue-on-error: true
 
-  # Job 2: Build Documentation for Pushes to Main
+  # Job 2: Build Documentation for PR to Main and Dev Branches
   build-docs:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -17,6 +17,7 @@ build/
 wheels/
 lib/
 dist/
+external/
 
 # Sphinx documentation
 docs/_build/

README.md

@@ -49,6 +49,31 @@ pip install parallax-app[camera]
 python -m parallax
 ```
 
+### Optional: Enable SuperPoint + SuperGlue Reticle Detection
+Parallax supports reticle detection using SuperPoint + LightGlue.
+To enable reticle detection using SuperPoint + SuperGlue, you must manually download 'SuperGluePretrainedNetwork' pretrained models.
+
+The SuperGluePretrainedNetwork is not included in this repository and is distributed under its own licensing terms.
+Please review their [license](https://github.com/magicleap/SuperGluePretrainedNetwork) before use.
+
+Manual Setup Instructions
+Clone the repository into the external/ folder in your Parallax project root:
+```bash
+pip install parallax-app[sfm]
+git clone https://github.com/magicleap/SuperGluePretrainedNetwork.git external/SuperGluePretrainedNetwork
+```
+Verify your folder structure looks like this:
+```bash
+parallax/
+├── external/
+│   └── SuperGluePretrainedNetwork/
+│       └── models/
+│           ├── superpoint.py
+│           └── weights/
+│               ├── superpoint_v1.pth
+│               └── superglue_indoor.pth
+```
+
 ### For developers:
 1. Clone the repository:
 ```bash

parallax/__init__.py

@@ -4,7 +4,7 @@
 
 import os
 
-__version__ = "1.7.0"
+__version__ = "1.8.0"
 
 # allow multiple OpenMP instances
 os.environ["KMP_DUPLICATE_LIB_OK"] = "True"

parallax/cameras/__init__.py

@@ -0,0 +1,3 @@
+"""
+Init
+"""

parallax/cameras/calibration_camera.py

@@ -11,6 +11,8 @@
 import logging
 import cv2
 import numpy as np
+import scipy.spatial.transform as Rscipy
+
 
 # Set logger name
 logger = logging.getLogger(__name__)
@@ -81,23 +83,25 @@ def __init__(self, camera_name):
 
     def _get_changed_data_format(self, x_axis, y_axis):
         """
-        Change data format for calibration.
+        Combine and format x and y axis coordinates into a single array.
 
         Args:
-            x_axis (list): X-axis coordinates.
-            y_axis (list): Y-axis coordinates.
+            x_axis (list or np.ndarray): X-axis coordinates (N, 2).
+            y_axis (list or np.ndarray): Y-axis coordinates (M, 2).
 
         Returns:
-            numpy.ndarray: Reshaped coordinates.
+            np.ndarray: Combined coordinates with shape (N + M, 2), dtype float32.
         """
-        x_axis = np.array(x_axis)
-        y_axis = np.array(y_axis)
+        x_axis = np.asarray(x_axis, dtype=np.float32)
+        y_axis = np.asarray(y_axis, dtype=np.float32)
+
+        if x_axis.ndim != 2 or x_axis.shape[1] != 2:
+            raise ValueError("x_axis must have shape (N, 2)")
+        if y_axis.ndim != 2 or y_axis.shape[1] != 2:
+            raise ValueError("y_axis must have shape (M, 2)")
+
         coords_lines = np.vstack([x_axis, y_axis])
-        nCoords_per_axis = self.n_interest_pixels * 2 + 1
-        coords_lines_reshaped = coords_lines.reshape(
-            (nCoords_per_axis * 2, 2)
-        ).astype(np.float32)
-        return coords_lines_reshaped
+        return coords_lines
 
     def _process_reticle_points(self, x_axis, y_axis):
         """
@@ -160,84 +164,6 @@ def calibrate_camera(self, x_axis, y_axis):
         )
         return ret, self.mtx, self.dist, self.rvecs, self.tvecs
 
-    def get_predefined_intrinsic(self, x_axis, y_axis):
-        """
-        Fetches predefined intrinsic camera parameters for specific models.
-        Parameters:
-        - x_axis (int or float): The x-axis value for reticle processing.
-        - y_axis (int or float): The y-axis value for reticle processing.
-
-        Returns:
-        - A tuple of (bool, numpy.ndarray or None, numpy.ndarray or None)
-        representing success status, intrinsic matrix,
-        and distortion coefficients respectively.
-        """
-        self._process_reticle_points(x_axis, y_axis)
-        if self.name == "22517664":
-            self.mtx = np.array([[1.55e+04, 0.0e+00, 2e+03],
-                                 [0.0e+00, 1.55e+04, 1.5e+03],
-                                 [0.0e+00, 0.0e+00, 1.0e+00]],
-                                dtype=np.float32)
-            self.dist = np.array([[-0.02, 8.26, -0.01, -0.00, -63.01]],
-                                 dtype=np.float32)
-            return True, self.mtx, self.dist
-
-        elif self.name == "22433200":
-            self.mtx = np.array([[1.55e+04, 0.0e+00, 2e+03],
-                                 [0.0e+00, 1.55e+04, 1.5e+03],
-                                 [0.0e+00, 0.0e+00, 1.0e+00]],
-                                dtype=np.float32)
-            self.dist = np.array([[-0.02, 1.90, -0.00, -0.01, 200.94]],
-                                 dtype=np.float32)
-            return True, self.mtx, self.dist
-
-        else:
-            return False, None, None
-
-    def get_origin_xyz(self):
-        """
-        Get origin (0,0) and axis points (x, y, z coords) in image coordinates.
-
-        Returns:
-            tuple: Origin, x-axis, y-axis, z-axis points.
-        """
-        axis = np.float32([[5, 0, 0], [0, 5, 0], [0, 0, 7]]).reshape(-1, 3)
-        # Find the rotation and translation vectors.
-        # Output rotation vector (see Rodrigues ) that, together with tvec,
-        # brings points from the model coordinate system
-        # to the camera coordinate system.
-        if self.objpoints is not None:
-            solvePnP_method = cv2.SOLVEPNP_ITERATIVE
-            _, rvecs, tvecs = cv2.solvePnP(
-                self.objpoints,
-                self.imgpoints,
-                self.mtx,
-                self.dist,
-                flags=solvePnP_method,
-            )
-            imgpts, _ = cv2.projectPoints(
-                axis, rvecs, tvecs, self.mtx, self.dist
-            )
-            origin = tuple(
-                self.imgpoints[0][CENTER_INDEX_X].ravel().astype(int)
-            )
-            x = tuple(imgpts[0].ravel().astype(int))
-            y = tuple(imgpts[1].ravel().astype(int))
-            z = tuple(imgpts[2].ravel().astype(int))
-
-            """
-            # Uncomment to print quaternion and translation vector
-            R, _ = cv2.Rodrigues(rvecs)
-            quat = Rscipy.from_matrix(R).as_quat()  # [QX, QY, QZ, QW]
-            QX, QY, QZ, QW = quat
-            TX, TY, TZ = tvecs.flatten()
-            print(f"{self.name}: {QW} {QX} {QY} {QZ} {TX} {TY} {TZ}")
-            """
-
-            return origin, x, y, z
-        else:
-            return None
-
 
 class CalibrationStereo(CalibrationCamera):
     """
@@ -593,33 +519,135 @@ def register_debug_points(self, camA, camB):
         objpoints = np.array([objpoint], dtype=np.float32)
 
         # Call the get_pixel_coordinates method using the object points
-        pixel_coordsA = self.get_pixel_coordinates(objpoints, self.rvecA, self.tvecA, self.mtxA, self.distA)
-        pixel_coordsB = self.get_pixel_coordinates(objpoints, self.rvecB, self.tvecB, self.mtxB, self.distB)
+        pixel_coordsA = get_projected_points(objpoints, self.rvecA, self.tvecA, self.mtxA, self.distA)
+        pixel_coordsB = get_projected_points(objpoints, self.rvecB, self.tvecB, self.mtxB, self.distB)
 
         # Register the pixel coordinates for the debug points
         self.model.add_coords_for_debug(camA, pixel_coordsA)
         self.model.add_coords_for_debug(camB, pixel_coordsB)
 
-    def get_pixel_coordinates(self, objpoints, rvec, tvec, mtx, dist):
-        """
-        Projects 3D object points onto the 2D image plane and returns pixel coordinates.
 
-        Parameters:
-            objpoints (list): List of 3D object points.
-            rvec (np.ndarray): Rotation vector.
-            tvec (np.ndarray): Translation vector.
-            mtx (np.ndarray): Camera matrix.
-            dist (np.ndarray): Distortion coefficients.
+# Utils
+def get_projected_points(objpoints, rvec, tvec, mtx, dist):
+    """
+    Projects 3D object points onto the 2D image plane and returns pixel coordinates.
+    Parameters:
+        objpoints (list): List of 3D object points.
+        rvec (np.ndarray): Rotation vector.
+        tvec (np.ndarray): Translation vector.
+        mtx (np.ndarray): Camera matrix.
+        dist (np.ndarray): Distortion coefficients.
+    Returns:
+        list: List of pixel coordinates corresponding to the object points.
+    """
 
-        Returns:
-            list: List of pixel coordinates corresponding to the object points.
-        """
-        pixel_coordinates = []
-        for points in objpoints:
-            # Project the 3D object points to 2D image points
-            imgpoints, _ = cv2.projectPoints(points, rvec, tvec, mtx, dist)
-            # Convert to integer tuples and append to the list
-            imgpoints_tuples = [tuple(map(lambda x: int(round(x)), point)) for point in imgpoints.reshape(-1, 2)]
-            pixel_coordinates.append(imgpoints_tuples)
-
-        return pixel_coordinates
+    """
+    pixel_coordinates = []
+    for points in objpoints:
+        # Project the 3D object points to 2D image points
+        imgpoints, _ = cv2.projectPoints(points, rvec, tvec, mtx, dist)
+        # Convert to integer tuples and append to the list
+        imgpoints_tuples = [tuple(map(lambda x: int(round(x)), point)) for point in imgpoints.reshape(-1, 2)]
+        pixel_coordinates.append(imgpoints_tuples)
+    return pixel_coordinates
+    """
+    imgpoints, _ = cv2.projectPoints(objpoints, rvec, tvec, mtx, dist)
+    return np.round(imgpoints.reshape(-1, 2)).astype(np.int32)
+
+
+def get_axis_object_points(axis='x', coord_range=10, world_scale=0.2):
+    """
+    Generate 1D object points along a given axis.
+
+    Args:
+        axis (str): 'x' or 'y' to indicate along which axis to generate points.
+        coord_range (int): Half-range for coordinates (from -range to +range).
+        world_scale (float): Scale factor to convert to real-world units (e.g., mm).
+
+    Returns:
+        numpy.ndarray: Object points (N x 3) along the specified axis.
+    """
+    coords = np.arange(-coord_range, coord_range + 1, dtype=np.float32)
+    points = np.zeros((len(coords), 3), dtype=np.float32)
+
+    if axis == 'x':
+        points[:, 0] = coords
+    elif axis == 'y':
+        points[:, 1] = coords
+    else:
+        raise ValueError("axis must be 'x' or 'y'")
+
+    return np.round(points * world_scale, 2)
+
+
+def get_origin_xyz(imgpoints, mtx, dist, rvecs, tvecs, center_index_x=0, axis_length=5):
+    """
+    Get origin (0,0) and axis points (x, y, z coords) in image coordinates using known pose.
+
+    Args:
+        imgpoints (np.ndarray): 2D image points corresponding to object points (N x 1 x 2 or N x 2).
+        mtx (np.ndarray): Camera intrinsic matrix.
+        dist (np.ndarray): Distortion coefficients.
+        rvecs (np.ndarray): Rotation vector (3x1).
+        tvecs (np.ndarray): Translation vector (3x1).
+        center_index_x (int): Index in imgpoints corresponding to the origin.
+
+    Returns:
+        tuple: Origin, x-axis, y-axis, z-axis image coordinates as integer tuples.
+    """
+    if imgpoints is None:
+        return None
+
+    # Define 3D axes in world coordinates (X, Y, Z directions from origin)
+    axis = np.float32([[axis_length, 0, 0], [0, axis_length, 0], [0, 0, axis_length]]).reshape(-1, 3)
+
+    # Project axis to 2D image points using known rvecs, tvecs
+    imgpts, _ = cv2.projectPoints(axis, rvecs, tvecs, mtx, dist)
+
+    origin = tuple(np.round(imgpoints[center_index_x].ravel()).astype(int))
+    x = tuple(np.round(imgpts[0].ravel()).astype(int))
+    y = tuple(np.round(imgpts[1].ravel()).astype(int))
+    z = tuple(np.round(imgpts[2].ravel()).astype(int))
+
+    return origin, x, y, z
+
+
+def get_quaternion_and_translation(rvecs, tvecs, name="Camera"):
+    """
+    Print the quaternion (QW, QX, QY, QZ) and translation vector (TX, TY, TZ)
+    derived from a rotation vector and translation vector.
+    Args:
+        rvecs (np.ndarray): Rotation vector (3x1 or 1x3).
+        tvecs (np.ndarray): Translation vector (3x1 or 1x3).
+        name (str): Optional name to include in the output.
+    """
+    R, _ = cv2.Rodrigues(rvecs)
+    quat = Rscipy.from_matrix(R).as_quat()  # [QX, QY, QZ, QW]
+    QX, QY, QZ, QW = quat
+    TX, TY, TZ = tvecs.flatten()
+    print(f"{name}: {QW:.6f} {QX:.6f} {QY:.6f} {QZ:.6f} {TX:.3f} {TY:.3f} {TZ:.3f}")
+
+    return QW, QX, QY, QZ, TX, TY, TZ
+
+
+def get_rvec_and_tvec(quat, tvecs):
+    """
+    Convert quaternion (QW, QX, QY, QZ) and translation vector (TX, TY, TZ)
+    to rotation vector (rvecs) and translation vector (tvecs).
+
+    Args:
+        quat (tuple): Quaternion as (QW, QX, QY, QZ).
+        tvecs (np.ndarray): Translation vector (3x1 or 1x3).
+
+    Returns:
+        rvecs (np.ndarray): Rotation vector (3x1).
+        tvecs (np.ndarray): Translation vector (3x1).
+    """
+    QX, QY, QZ, QW = quat  # scipy expects [QX, QY, QZ, QW] order
+    rotation = Rscipy.Rotation.from_quat([QX, QY, QZ, QW])
+    R_mat = rotation.as_matrix()
+    rvecs, _ = cv2.Rodrigues(R_mat)
+
+    tvecs = np.asarray(tvecs).reshape(3, 1)
+
+    return rvecs, tvecs