SE-1661: Support RAW Bayer and HDR

stereo_viewer/inc/io/data_thread.hpp

@@ -38,6 +38,7 @@ enum ImageDataType {
     IMTYPE_IO,  ///< single image
     IMTYPE_DO,  ///< disparity alone
     IMTYPE_LR,  ///< left+right images
+    IMTYPE_HDR, ///< HDR images
     IMTYPE_LD,  ///< left+disparity images
     IMTYPE_DR,  ///< disparity+right images
     IMTYPE_DC,  ///< disparity+confidence images

stereo_viewer/src/gev/pipeline.cc

@@ -204,6 +204,21 @@ void Pipeline::Stop() {
   // Discard all queued buffers
 }
 
+inline int cv_pixformat(PvPixelType pleora) {
+  //PvPixelBayerBG10
+  switch(pleora) {
+    case PvPixelYUV422_8:
+      return CV_8UC2;
+    case PvPixelBayerBG10:
+    case PvPixelBayerRG10:
+    case PvPixelBayerBG16:
+    case PvPixelBayerRG16:
+      return CV_16SC1; // Fixme use this as marker for bayer encoded images
+    default:
+      return CV_16UC1;
+  }
+}
+
 void Pipeline::run() {
   size_t consequitive_errors = 0;
   size_t timeout_count = MAX_CONS_ERRORS_IN_ACQUISITION;
@@ -253,8 +268,8 @@ void Pipeline::run() {
 
             // Protected image creation
             {
-              int cv_pixfmt0 = (img0->GetPixelType() == PvPixelYUV422_8)? CV_8UC2: CV_16UC1;
-              int cv_pixfmt1 = (img1->GetPixelType() == PvPixelYUV422_8)? CV_8UC2: CV_16UC1;
+              int cv_pixfmt0 = cv_pixformat(img0->GetPixelType());
+              int cv_pixfmt1 = cv_pixformat(img1->GetPixelType());
 
               QMutexLocker l(&m_image_lock);
               // See if there is chunk data attached
@@ -274,9 +289,9 @@ void Pipeline::run() {
             {
               QMutexLocker l(&m_image_lock);
               img0 = lBuffer->GetImage();
-              int cv_pixformat = (img0->GetPixelType() == PvPixelYUV422_8)? CV_8UC2: CV_16UC1;
+              int cv_pixfmt = cv_pixformat( img0->GetPixelType() );
 
-              m_images.enqueue({new Mat(img0->GetHeight(), img0->GetWidth(), cv_pixformat, img0->GetDataPointer()),
+              m_images.enqueue({new Mat(img0->GetHeight(), img0->GetWidth(), cv_pixfmt, img0->GetDataPointer()),
                                 new Mat(), timestamp, static_cast<int32_t>(minDisparity), pointcloud});
             }
 

stereo_viewer/src/io/data_thread.cc

@@ -112,6 +112,9 @@ bool DataThread::setFolder(QString new_folder){
   } else if (m_imtype == IMTYPE_DC){
     status = status && getFilename(m_disparity_fname, m_folder, m_disparity_subfolder, "disparity_");
     status = status && getFilename(m_conf_fname, m_folder, m_disparity_subfolder, "conf_");
+  } else if (m_imtype == IMTYPE_HDR) {
+    status = status && getFilename(m_left_fname, m_folder, m_left_subfolder, "hdr_high_");
+    status = status && getFilename(m_right_fname, m_folder, m_right_subfolder, "hdr_low_");
   }
 
   return status;
@@ -304,6 +307,10 @@ void DataThread::run() {
       imdata.right.save(m_conf_fname + suffix, ext.toStdString().c_str(), quality);
       //QString fname = m_disparity_fname + suffix.replace(ext.toLower(), "ply");
       //saveProjected3D(imdata, m_matQ, fname);
+    } else if (imdata.imtype == IMTYPE_HDR) {
+      suffix.replace(ext.toLower(), "png"); // PNG is 16-bit grayscale for bayer images
+      imdata.left.save(m_left_fname + suffix, "png", quality);
+      imdata.right.save(m_right_fname + suffix, "png", quality);
     }
     if((imdata.pc.size() > 0) && (imdata.imtype == IMTYPE_LR)){
       getFilename(m_pc_fname, m_folder, m_pc_subfolder, "spc_");

stereo_viewer/src/viewer.cc

@@ -90,6 +90,80 @@ static QImage s_yuv2_to_qimage(const cv::Mat*img) {
   return QImage((uchar*) res.data, res.cols, res.rows, res.step, QImage::Format_RGB888).copy();
 }
 
+#if(QT_VERSION >= QT_VERSION_CHECK(5, 15, 0))
+static QImage s_bayer_to_qimage(const cv::Mat*img) {
+  Mat res = img->clone();
+  // Iterate through the image and shift each pixel value
+  // Iterate through the image and shift each pixel value
+  for (int i = 0; i < img->rows; i++) {
+    for (int j = 0; j < img->cols; j++) {
+      // Access each pixel and shift it left by 6 bits
+      res.at<uint16_t>(i, j) = img->at<uint16_t>(i, j) << 6;
+    }
+  }
+  // data pointer looses scope, deep copy needed
+  return QImage((uchar*) res.data, res.cols, res.rows, res.step, QImage::Format_Grayscale16).copy();
+}
+#else // Older QT versions don't support 16-bit images
+static QImage s_bayer_to_qimage(const cv::Mat* img) {
+    cv::Mat res = cv::Mat::zeros(img->rows, img->cols, CV_8UC1); // Create an 8-bit result image
+
+    // Iterate through the image and shift each pixel value
+    for (int i = 0; i < img->rows; i++) {
+        for (int j = 0; j < img->cols; j++) {
+            // Access each pixel, shift it right by 2 bits to convert to 8-bit, and store in the result image
+            uint16_t pixel_value = img->at<uint16_t>(i, j);
+            res.at<uchar>(i, j) = static_cast<uchar>(pixel_value >> 2); // Shift down by 2 bits
+        }
+    }
+
+    // Create and return a deep copy of the QImage
+    return QImage((uchar*) res.data, res.cols, res.rows, res.step, QImage::Format_Mono).copy();
+}
+#endif
+
+/**
+ * @brief Convert an interlaced image to a pair of images
+ * @param img Interlaced image
+ * @param left First resulting image (row 0)
+ * @param right Second resulting image (row +1)
+ */
+static void s_interlaced_to_pair(const cv::Mat*img, QImage &left, QImage &right) {
+#if(QT_VERSION >= QT_VERSION_CHECK(5, 15, 0))
+  // Split the interlaced image into two separate images
+  Mat l = Mat(img->rows/2, img->cols, CV_16UC1);
+  Mat r = Mat(img->rows/2, img->cols, CV_16UC1);
+
+  for (int i = 0; i < l.rows; i++) {
+    for (int j = 0; j < l.cols; j++) {
+      // Copy the l image
+      l.at<uint16_t>(i, j) = img->at<uint16_t>(i * 2, j);
+      // Copy the right image
+      r.at<uint16_t>(i, j) = img->at<uint16_t>(i*2 + 1, j);
+    }
+  }
+//  QImage in((uchar*) img->data, img->cols, img->rows, img->step, QImage::Format_Grayscale16);
+//  in.save("interlaced.png");
+  left = QImage((uchar*) l.data, l.cols, l.rows, l.step, QImage::Format_Grayscale16).copy();
+  right = QImage((uchar*) r.data, r.cols, r.rows, r.step, QImage::Format_Grayscale16).copy();
+#else
+  // Split the interlaced image into two separate images
+  Mat l = Mat(img->rows/2, img->cols, CV_8UC1);
+  Mat r = Mat(img->rows/2, img->cols, CV_8UC1);
+
+  for (int i = 0; i < l.rows; i++) {
+    for (int j = 0; j < l.cols; j++) {
+      // Copy the l image
+      l.at<uint8_t>(i, j) = img->at<uint16_t>(i * 2, j) >> 8;
+      // Copy the right image
+      r.at<uint8_t>(i, j) = img->at<uint16_t>(i * 2 + 1, j) >> 8;
+    }
+  }
+  left = QImage((uchar*) l.data, l.cols, l.rows, l.step, QImage::Format_Mono).copy();
+  right = QImage((uchar*) r.data, r.cols, r.rows, r.step, QImage::Format_Mono).copy();
+#endif
+}
+
 static QImage s_mono_to_qimage(const cv::Mat*img, int colormap=COLORMAP_JET, int mindisp=0, int maxdisp=0) {
   Mat res;
   QImage::Format qformat = QImage::Format_Grayscale8;
@@ -803,7 +877,6 @@ void MainWindow::newData(uint64_t timestamp, QImage &left, QImage &right, QPair<
 
   cfg.widgetLeftSensor->setStyleSheet(QStringLiteral("background-color:black; border: 2px solid green;"));
   cfg.widgetRightSensor->setStyleSheet(QStringLiteral("background-color:black; border: 2px solid green;"));
-
 }
 
 void MainWindow::showStatusMessage(uint32_t rcv_images){
@@ -878,14 +951,19 @@ void MainWindow::handleStereoData() {
         label.second = "Right";
         m_data_thread->setImageDataType(labforge::io::IMTYPE_DR);
         is_disparity = true;
-      } else{
+      } else if((image.left->type() == CV_8UC2) && (image.right->type() == CV_8UC2)) {
         q1 = s_yuv2_to_qimage(image.left);
         q2 = s_yuv2_to_qimage(image.right);
         label.first = "Left";
         label.second = "Right";
         m_data_thread->setImageDataType(labforge::io::IMTYPE_LR);
+      } else if((image.left->type() == CV_16SC1) && (image.right->type() == CV_16SC1)) {
+        q1 = s_bayer_to_qimage(image.left);
+        q2 = s_bayer_to_qimage(image.right);
+        label.first = "Left";
+        label.second = "Right";
+        m_data_thread->setImageDataType(labforge::io::IMTYPE_LR);
       }
-
       newData(image.timestamp, q1, q2, label, is_disparity, raw_disparity, image.min_disparity, image.pc);
       delete image.left;
       delete image.right;
@@ -908,20 +986,24 @@ void MainWindow::handleMonoData(){
       QImage q2;
       QPair<QString, QString> label;
       bool is_disparity = false;
+      label.second = "";
 
       if(image.left->type() == CV_16UC1){
         q1 = s_mono_to_qimage(image.left, cfg.cbxColormap->currentIndex(), cfg.spinMinDisparity->value(), cfg.spinMaxDisparity->value());
         raw_disparity = (uint16_t*)image.left->data;
         label.first = "Disparity";
         m_data_thread->setImageDataType(labforge::io::IMTYPE_DO);
         is_disparity = true;
-      }
-      else{
+      } else if(image.left->type() == CV_16SC1) { // Interlaced HDR image
+        s_interlaced_to_pair(image.left, q1, q2);
+        label.first = "HDR high exposure";
+        label.second = "HDR low exposure";
+        m_data_thread->setImageDataType(labforge::io::IMTYPE_HDR);
+      } else {
         q1 = s_yuv2_to_qimage(image.left);
         label.first = "Display";
         m_data_thread->setImageDataType(labforge::io::IMTYPE_IO);
       }
-      label.second = "";
 
       newData(image.timestamp, q1, q2, label, is_disparity, raw_disparity, image.min_disparity, image.pc);
       delete image.left;