Create tensorflowDetection2D app

CMakeLists.txt

@@ -47,6 +47,7 @@ find_package(COLOR_DEBUG REQUIRED)
 # Soft dependencies.
 find_package(OpenCV QUIET)
 find_package(PCL 1.6 QUIET)
+find_package(TensorflowCC 1.12 QUIET)
 
 # Always build YARP devices as MODULE libraries.
 set(YARP_FORCE_DYNAMIC_PLUGINS TRUE CACHE INTERNAL "Force dynamic plugins")

doc/vision-install.md

@@ -9,6 +9,7 @@
 - [Install YARP 2.3.70+ and OpenNI2DeviceServer](https://github.com/roboticslab-uc3m/installation-guides/blob/master/install-yarp.md)
 - [Install OpenCV](https://github.com/roboticslab-uc3m/installation-guides/blob/master/install-opencv.md)
 - [Install color-debug](https://github.com/roboticslab-uc3m/color-debug)
+- [Install tensorflow_cc 1.12+](https://github.com/FloopCZ/tensorflow_cc)
 
 ### Install the Software on Ubuntu (working on all tested versions)
 

programs/CMakeLists.txt

@@ -8,3 +8,4 @@ add_subdirectory(haarDetection2D)
 add_subdirectory(kinectPxToReal)
 add_subdirectory(pointAtObjectServer)
 add_subdirectory(voxelOccupancyDetection)
+add_subdirectory(tensorflowDetection2D)

programs/tensorflowDetection2D/CMakeLists.txt

@@ -0,0 +1,46 @@
+
+if(NOT OpenCV_FOUND AND (NOT DEFINED ENABLE_tensorflowDetection2D OR ENABLE_tensorflowDetection2D))
+    message(WARNING "OpenCV package not found, disabling tensorflowDetection2D program")
+endif()
+
+if(NOT TensorflowCC_FOUND AND (NOT DEFINED ENABLE_tensorflowDetection2D OR ENABLE_tensorflowDetection2D))
+    message(WARNING "TensorflowCC package not found, disabling tensorflowDetection2D program")
+endif()
+
+cmake_dependent_option(ENABLE_tensorflowDetection2D "Enable/disable tensorflowDetection2D program" ON
+                       "OpenCV_FOUND;TensorflowCC_FOUND" OFF)
+
+
+if(ENABLE_tensorflowDetection2D)
+
+    set(CMAKE_CXX_STANDARD 11)
+    set(CMAKE_CXX_STANDARD_REQUIRED ON)
+    include_directories(${OpenCV_INCLUDE_DIRS})
+
+    add_executable(tensorflowDetection2D main.cpp
+                                   TensorflowDetection2D.hpp
+                                   TensorflowDetection2D.cpp
+                                   TensorflowSessionTest.h
+                                   TensorflowSessionTest.cpp
+                                   TensorflowDetector.hpp
+                                   TensorflowDetector.cpp
+                                   MainDetector.hpp
+                                   MainDetector.cpp
+                                   )
+
+    target_link_libraries(tensorflowDetection2D YARP::YARP_OS
+                                          YARP::YARP_init
+                                          YARP::YARP_dev
+                                          YARP::YARP_sig
+                                          ${OpenCV_LIBS}
+                                          TensorflowCC::Shared)
+
+
+    install(TARGETS tensorflowDetection2D
+            DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+else()
+
+    set(ENABLE_tensorflowDetection2D OFF CACHE BOOL "Enable/disable tensorflowDetection2D program" FORCE)
+
+endif()

programs/tensorflowDetection2D/MainDetector.cpp

@@ -0,0 +1,215 @@
+
+// Libraries
+
+#include <fstream>
+#include <utility>
+#include <vector>
+#include <iostream>
+#include <time.h>
+#include <cstdlib>
+#include <yarp/os/Bottle.h>
+#include <yarp/os/BufferedPort.h>
+#include <yarp/os/ConnectionReader.h>
+#include <yarp/os/Port.h>
+#include <yarp/os/PortReader.h>
+#include <yarp/sig/Image.h>
+#include <yarp/os/Time.h>
+#include <opencv2/core/mat.hpp>
+#include <opencv2/videoio.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+#include <cv.hpp>
+#include "tensorflow/cc/ops/const_op.h"
+#include "tensorflow/cc/ops/image_ops.h"
+#include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/graph/default_device.h"
+#include "tensorflow/core/graph/graph_def_builder.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/stringprintf.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/public/session.h"
+#include "tensorflow/core/util/command_line_flags.h"
+#include "TensorflowDetector.hpp"
+#include "MainDetector.hpp"
+#include <map>
+
+
+int maindetector::detect(std::string labels, std::string graph, std::string video_source, yarp::os::Port sender_port_pre, yarp::os::Port sender_port_post) {
+
+
+    std::system("clear");
+    std::cout<<std::endl;
+    std::cout<<std::endl;
+
+    // Path
+    std::string ROOTDIR = "../";
+    std::string LABELS = labels;
+    std::string GRAPH = graph;
+    std::string source_video=video_source;
+
+    // Set  nombres nodos entrada y salida
+    tensorflow::string inputLayer = "image_tensor:0";
+    std::vector<std::string> outputLayer = {"detection_boxes:0", "detection_scores:0", "detection_classes:0", "num_detections:0"};
+
+    // Load .pb frozen model
+    std::unique_ptr<tensorflow::Session> session;
+    tensorflow::string graphPath = tensorflow::io::JoinPath(ROOTDIR, GRAPH);
+    std::cout<<"The graph it´s going to be loaded:" << graphPath<<"."<<std::endl;
+    std::cout<<"Loading graph..."<<std::endl;
+    yarp::os::Time::delay(1);
+    //LOG(INFO) << "Loading graph:" << graphPath<<" ...";
+
+    tensorflow::Status loadGraphStatus = loadGraph(graphPath, &session);
+    if (!loadGraphStatus.ok()) {
+        std::system("clear");
+        std::cout<<std::endl;
+        std::cout<<std::endl;
+        std::cout<<"Fail loading graph "<<graphPath<<"."<<std::endl;
+        //LOG(ERROR) << "Loading graph: FAIL" << loadGraphStatus;
+        yarp::os::Time::delay(1);
+        return -1;
+    } else
+        std::system("clear");
+        std::cout<<std::endl;
+        std::cout<<std::endl;
+        std::cout<<"Graph "<<graphPath<<" loaded correctly."<<std::endl;
+        //LOG(INFO) << "Loading  graph: OK" << std::endl;
+        yarp::os::Time::delay(1);
+
+
+    // Cargar etiquetas
+    std::map<int, std::string> labelsMap = std::map<int,std::string>();
+    std::system("clear");
+    std::cout<<std::endl;
+    std::cout<<std::endl;
+    std::cout<<"Labels "<<LABELS<<" are going to be loaded."<<std::endl;
+    yarp::os::Time::delay(1);
+    tensorflow::Status readLabelsMapStatus = readLabelsMapFile(tensorflow::io::JoinPath(ROOTDIR, LABELS), labelsMap);
+    if (!readLabelsMapStatus.ok()) {
+        //LOG(ERROR) << "readLabelsMapFile(): ERROR" << loadGraphStatus;
+        std::cout<<"Fail loading labels "<<LABELS<<"."<<std::endl;
+        //LOG(INFO) << "Carga del graph: OK" << std::endl;
+        yarp::os::Time::delay(1);
+        return -1;
+    } else
+        //LOG(INFO) << "readLabelsMapFile(): labels map loaded with " << labelsMap.size() << " label(s)" << std::endl;
+        std::cout<<"Labels "<<LABELS<<" loaded correctly."<<std::endl;
+        std::cout<<labelsMap.size()<<" labels have been loaded."<<std::endl;
+        yarp::os::Time::delay(1);
+
+    std::system("clear");
+    std::cout<<std::endl;
+    std::cout<<std::endl;
+    std::cout<<"Video source frames are going to be taken."<<std::endl;
+    yarp::os::Time::delay(1);
+    cv::Mat frame;
+    tensorflow::Tensor tensor;
+    std::vector<tensorflow::Tensor> outputs;
+    double thresholdScore = 0.5;
+    double thresholdIOU = 0.8;
+
+    // Cuenta FPS
+    int nFrames = 25;
+    int iFrame = 0;
+    double fps = 0.;
+    time_t start, end;
+    time(&start);
+
+    // Adquiere imagen de la fuente de video
+    cv::VideoCapture cap(source_video);
+
+    tensorflow::TensorShape shape = tensorflow::TensorShape();
+    shape.AddDim(1);
+    shape.AddDim((tensorflow::int64)cap.get(cv::CAP_PROP_FRAME_HEIGHT));
+    shape.AddDim((tensorflow::int64)cap.get(cv::CAP_PROP_FRAME_WIDTH));
+    shape.AddDim(3);
+    std::system("clear");
+    std::cout<<std::endl;
+    std::cout<<std::endl;
+    std::cout<<"Taking frames..."<<std::endl;
+    yarp::os::Time::delay(1);
+    //tensorflowDetection2D yarp_sender;
+    while (cap.isOpened()) {
+        cap >> frame;
+
+        // Enviar imagen preprocesada por yarp
+        //yarp_sender.send_post(frame, puerto_pre);
+        // A mano
+        yarp::sig::ImageOf<yarp::sig::PixelBgr> B;
+        B.setExternal(frame.data,frame.size[1],frame.size[0]);
+        sender_port_pre.write(B);
+
+        cv::cvtColor(frame, frame, cv::COLOR_BGR2RGB);
+        std::cout << "Frame: " << iFrame << std::endl;
+
+        if (nFrames % (iFrame + 1) == 0) {
+            time(&end);
+            fps = 1. * nFrames / difftime(end, start);
+            time(&start);
+        }
+        iFrame++;
+
+        // Pasar Mat a Tensor
+        tensor = tensorflow::Tensor(tensorflow::DT_FLOAT, shape);
+        tensorflow::Status readTensorStatus = readTensorFromMat(frame, tensor);
+        if (!readTensorStatus.ok()) {
+            //LOG(ERROR) << "Mat->Tensor conversion failed: " << readTensorStatus;
+            std::system("clear");
+            std::cout<<std::endl;
+            std::cout<<std::endl;
+            std::cout<<"Mat OpenCV -> Tensor : FAIL"<<std::endl;
+            yarp::os::Time::delay(1);
+            return -1;
+        }
+
+        // Ejecutar graph
+        outputs.clear();
+        tensorflow::Status runStatus = session->Run({{inputLayer, tensor}}, outputLayer, {}, &outputs);
+        if (!runStatus.ok()) {
+            //LOG(ERROR) << "Running model failed: " << runStatus;
+            std::system("clear");
+            std::cout<<std::endl;
+            std::cout<<std::endl;
+            std::cout<<"Running model status: FAIL"<<std::endl;
+            yarp::os::Time::delay(1);
+            return -1;
+        }
+
+        // Extraer resultados del vector
+        tensorflow::TTypes<float>::Flat scores = outputs[1].flat<float>();
+        tensorflow::TTypes<float>::Flat classes = outputs[2].flat<float>();
+        tensorflow::TTypes<float>::Flat numDetections = outputs[3].flat<float>();
+        tensorflow::TTypes<float, 3>::Tensor boxes = outputs[0].flat_outer_dims<float,3>();
+
+        std::vector<size_t> goodIdxs = filterBoxes(scores, boxes, thresholdIOU, thresholdScore);
+        for (size_t i = 0; i < goodIdxs.size(); i++){
+        std::cout<<std::endl;
+        std::cout<<std::endl;
+        std::cout<<"Detection: "<<labelsMap[classes(goodIdxs.at(i))]<< " -> Score: "<<scores(goodIdxs.at(i))<<std::endl;
+
+        /* LOG(INFO) << "score:" << scores(goodIdxs.at(i)) << ",class:" << labelsMap[classes(goodIdxs.at(i))]
+                      << " (" << classes(goodIdxs.at(i)) << "), box:" << "," << boxes(0, goodIdxs.at(i), 0) << ","
+                      << boxes(0, goodIdxs.at(i), 1) << "," << boxes(0, goodIdxs.at(i), 2) << ","
+                      << boxes(0, goodIdxs.at(i), 3);*/
+
+
+        std::cout<<std::endl;
+
+        cv::cvtColor(frame, frame, cv::COLOR_BGR2RGB);
+        drawBoundingBoxesOnImage(frame, scores, classes, boxes, labelsMap, goodIdxs);
+
+        cv::putText(frame, std::to_string(fps).substr(0, 5), cv::Point(0, frame.rows), cv::FONT_HERSHEY_SIMPLEX, 0.7, cv::Scalar(255, 255, 255));
+        //yarp_sender.send_post(frame, puerto_post);
+        // A mano
+        yarp::sig::ImageOf<yarp::sig::PixelBgr> C;
+        C.setExternal(frame.data,frame.size[1],frame.size[0]);
+        sender_port_post.write(C);
+        cv::imshow("Video source: Processed", frame);
+        cv::waitKey(5);
+    }}
+    cv::destroyAllWindows();
+
+    return 0;
+}

programs/tensorflowDetection2D/MainDetector.hpp

@@ -0,0 +1,42 @@
+#ifndef MAINDETECTOR_HPP
+#define MAINDETECTOR_HPP
+
+#include <fstream>
+#include <utility>
+#include <vector>
+#include <iostream>
+#include <time.h>
+#include "tensorflow/cc/ops/const_op.h"
+#include "tensorflow/cc/ops/image_ops.h"
+#include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/graph/default_device.h"
+#include "tensorflow/core/graph/graph_def_builder.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/stringprintf.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/public/session.h"
+#include "tensorflow/core/util/command_line_flags.h"
+#include <yarp/os/Bottle.h>
+#include <yarp/os/BufferedPort.h>
+#include <yarp/os/ConnectionReader.h>
+#include <yarp/os/Port.h>
+#include <yarp/os/PortReader.h>
+#include <yarp/sig/Image.h>
+#include <yarp/os/Time.h>
+#include <opencv2/core/mat.hpp>
+#include <opencv2/videoio.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+#include <cv.hpp>
+#include "TensorflowDetector.hpp"
+
+
+class maindetector
+{
+public:
+int detect(std::string labels, std::string graph, std::string video_source, yarp::os::Port sender_port_preyarp, yarp::os::Port sender_port_post);
+
+};
+#endif //MAINDETECTOR_HPP