Merge branch 'backtrackToFindError' into convertingToPlugin

ILLIXR · Sep 4, 2024 · 28b54de · 28b54de
2 parents f0707d5 + b61369a
commit 28b54de
Show file tree

Hide file tree

Showing 19 changed files with 191 additions and 149 deletions.
diff --git a/.gitignore b/.gitignore
@@ -38,3 +38,9 @@
 build/*
 cmake-build*
 mediapipe/demo_run_graph_main.cc
+
+mediapipe/modules/hand_landmark/hand_landmark_cpu.pbtxt
+
+mediapipe/modules/hand_landmark/hand_landmark_model_loader.pbtxt
+
+mediapipe/modules/palm_detection/palm_detection_model_loader.pbtxt
diff --git a/cmake/make_pb_binary.cmake b/cmake/make_pb_binary.cmake
@@ -27,9 +27,9 @@ function(make_proto_binary)
     )
 
     target_link_libraries(${make_proto_binary_BINARY_NAME} PUBLIC
-                          ${Protobuf_LIBRARIES}
+                          protobuf::libprotobuf
                           ${glog_LIBRARIES}
-                          ${tensorflow-lite_LIBRARIES}
+                          tensorflow-lite::tensorflow-lite
                           ${zlib_LIBRARIES}
                           absl::status
                           absl::absl_log

diff --git a/mediapipe/CMakeLists.txt b/mediapipe/CMakeLists.txt
@@ -150,52 +150,54 @@ target_link_libraries(${PLUGIN_NAME} PUBLIC
                       ${ZLIB_LIBRARIES}
 )
 
-add_executable(hand_tracking_tflite
-               #example/simple_run_graph_main.cc
-               demo_run_graph_main.cc
-)
+if(BUILD_TEST_EXE)
+    add_executable(hand_tracking_tflite
+                   #example/simple_run_graph_main.cc
+                   demo_run_graph_main.cc
+    )
 
-target_link_libraries(hand_tracking_tflite PUBLIC
-                      protobuf::libprotobuf
-                      absl::algorithm
-                      absl::base
-                      absl::core_headers
-                      absl::dynamic_annotations
-                      absl::nullability
-                      absl::btree
-                      absl::flat_hash_map
-                      absl::flat_hash_set
-                      absl::node_hash_map
-                      absl::leak_check
-                      absl::flags
-                      absl::flags_parse
-                      absl::any_invocable
-                      absl::bind_front
-                      absl::function_ref
-                      absl::hash
-                      absl::absl_check
-                      absl::absl_log
-                      absl::check
-                      absl::memory
-                      absl::type_traits
-                      absl::status
-                      absl::statusor
-                      absl::strings
-                      absl::str_format
-                      absl::string_view
-                      absl::synchronization
-                      absl::time
-                      absl::optional
-                      absl::span
-                      absl::variant
-                      absl::utility
-                      ${cpuinfo_LIBRARIES}
-                      tfl-XNNPACK::tfl-XNNPACK
-                      pthreadpool::pthreadpool
-                      tensorflow-lite::tensorflow-lite
-                      ${gflags_LIBRARIES}
-                      ${glog_LIBRARIES}
-                      ${OpenCV_LIBRARIES}
-                      ${ZLIB_LIBRARIES}
-                      ${PLUGIN_NAME}
-)
+    target_link_libraries(hand_tracking_tflite PUBLIC
+                          protobuf::libprotobuf
+                          absl::algorithm
+                          absl::base
+                          absl::core_headers
+                          absl::dynamic_annotations
+                          absl::nullability
+                          absl::btree
+                          absl::flat_hash_map
+                          absl::flat_hash_set
+                          absl::node_hash_map
+                          absl::leak_check
+                          absl::flags
+                          absl::flags_parse
+                          absl::any_invocable
+                          absl::bind_front
+                          absl::function_ref
+                          absl::hash
+                          absl::absl_check
+                          absl::absl_log
+                          absl::check
+                          absl::memory
+                          absl::type_traits
+                          absl::status
+                          absl::statusor
+                          absl::strings
+                          absl::str_format
+                          absl::string_view
+                          absl::synchronization
+                          absl::time
+                          absl::optional
+                          absl::span
+                          absl::variant
+                          absl::utility
+                          ${cpuinfo_LIBRARIES}
+                          tfl-XNNPACK::tfl-XNNPACK
+                          pthreadpool::pthreadpool
+                          tensorflow-lite::tensorflow-lite
+                          ${gflags_LIBRARIES}
+                          ${glog_LIBRARIES}
+                          ${OpenCV_LIBRARIES}
+                          ${ZLIB_LIBRARIES}
+                          ${PLUGIN_NAME}
+    )
+endif()
diff --git a/mediapipe/calculators/util/annotation_overlay_calculator.cc b/mediapipe/calculators/util/annotation_overlay_calculator.cc
@@ -352,7 +352,6 @@ namespace mediapipe {
 
         // Reset the renderer with the image_mat. No copy here.
         renderer_->AdoptImage(image_mat.get());
-        renderer_->AdoptImage(image_mat.get());
         auto hand_points = absl::make_unique<std::vector<Points> >();
         // Render streams onto render target.
         for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId();
@@ -371,7 +370,7 @@ namespace mediapipe {
                 renderer_->RenderDataOnImage(render_data);
             } else {
                 RET_CHECK_EQ(kVectorTag, tag);
-                const std::vector<RenderPointData>& render_data_vec =
+                const auto& render_data_vec =
                         cc->Inputs().Get(id).Get<std::vector<RenderPointData>>();
                 for (const RenderPointData& rp_data : render_data_vec) {
                     hand_points->push_back(rp_data.points());
@@ -503,7 +502,7 @@ namespace mediapipe {
 
             image_mat = absl::make_unique<cv::Mat>(
                     input_frame.Height(), input_frame.Width(), CV_8UC4,
-                    cv::Scalar(0,0,0,0));
+                    cv::Scalar(0, 0, 0, 0));
 
         } else {
             image_mat = absl::make_unique<cv::Mat>(

diff --git a/mediapipe/calculators/util/build.cmake b/mediapipe/calculators/util/build.cmake
@@ -6,7 +6,7 @@ set(UTIL_SOURCES
     ${CMAKE_CURRENT_LIST_DIR}/detections_to_rects_calculator.cc
     ${CMAKE_CURRENT_LIST_DIR}/detections_to_render_data_calculator.cc
     ${CMAKE_CURRENT_LIST_DIR}/filter_collection_calculator.cc
-    #${CMAKE_CURRENT_LIST_DIR}/illixr_output_calculator.cc
+    ${CMAKE_CURRENT_LIST_DIR}/illixr_output_calculator.cc
     ${CMAKE_CURRENT_LIST_DIR}/labels_to_render_data_calculator.cc
     ${CMAKE_CURRENT_LIST_DIR}/landmark_letterbox_removal_calculator.cc
     ${CMAKE_CURRENT_LIST_DIR}/landmark_projection_calculator.cc
@@ -25,8 +25,8 @@ set(UTIL_HEADERS
     ${CMAKE_CURRENT_LIST_DIR}/detections_to_rects_calculator.h
     ${CMAKE_CURRENT_LIST_DIR}/filter_collection_calculator.h
     ${CMAKE_CURRENT_LIST_DIR}/landmarks_to_render_data_calculator.h
-    #${CMAKE_CURRENT_LIST_DIR}/illixr_output_calculator.h
-    #${CMAKE_CURRENT_LIST_DIR}/illixr_data.h
+    ${CMAKE_CURRENT_LIST_DIR}/illixr_output_calculator.h
+    ${CMAKE_CURRENT_LIST_DIR}/illixr_data.h
 )
 
 target_sources(${PLUGIN_NAME} PRIVATE

diff --git a/mediapipe/calculators/util/illixr_output_calculator.cc b/mediapipe/calculators/util/illixr_output_calculator.cc
@@ -226,17 +226,15 @@ absl::Status ILLIXROutputCalculator::Process(CalculatorContext* cc) {
     if (cc->Inputs().HasTag(kHandedness) &&
         !cc->Inputs().Tag(kHandedness).IsEmpty()) {
         const auto &hands = cc->Inputs().Tag(kHandedness).Get<std::vector<ClassificationList> >();
-        if (hands.size() == 1) {
-            for (int i = 0; i < hands[0].classification_size(); i++) {
-                if (hands[0].classification(i).label() == "Left") {
-                    left_idx = i;
-                    frame_data->left_confidence = hands[0].classification(i).score();
-                } else if (hands[0].classification(i).label() == "Right") {
-                    right_idx = i;
-                    frame_data->right_confidence = hands[0].classification(i).score();
-                } else {
-                    // something is wrong
-                }
+        for(int i = 0; i < hands.size(); i++) {
+            if (hands[i].classification(0).label() == "Left") {
+                left_idx = i;
+                frame_data->left_confidence = hands[i].classification(0).score();
+            } else if (hands[i].classification(0).label() == "Right") {
+                right_idx = i;
+                frame_data->right_confidence = hands[i].classification(0).score();
+            } else {
+                // something is wrong
             }
             component_count++;
         }
@@ -245,8 +243,8 @@ absl::Status ILLIXROutputCalculator::Process(CalculatorContext* cc) {
         !cc->Inputs().Tag(kHandPointsTag).IsEmpty()) {
         const auto &hp = cc->Inputs().Tag(kHandPointsTag).Get<std::vector<Points> >();
         for (auto i = 0; i < hp.size(); i++) {
-            auto hand_points = absl::make_unique<ILLIXR::hand_points>(21, ILLIXR::point());
-            for (auto j = 0; j < hp[i].points_size(); i++) {
+            auto hand_points = absl::make_unique<::ILLIXR::hand_points>(21, ::ILLIXR::point());
+            for (auto j = 0; j < hp[i].points_size(); j++) {
                 auto pnt = hp[i].points(j);
                 hand_points->at(j).set(pnt.x(), pnt.y(), pnt.z(), pnt.normalized());
             }
@@ -260,7 +258,7 @@ absl::Status ILLIXROutputCalculator::Process(CalculatorContext* cc) {
 
     if (cc->Inputs().HasTag(palm_map.at(palm_input)) &&
         !cc->Inputs().Tag(palm_map.at(palm_input)).IsEmpty()) {
-        ILLIXR::rect* p_rect;
+        ::ILLIXR::rect* p_rect;
         if (palm_input == palm_input_type::NORM_RECT || palm_input == palm_input_type::RECT) {
             if (palm_input == palm_input_type::NORM_RECT) {
                 const auto &rect = cc->Inputs().Tag(kNormPalmRectTag).Get<NormalizedRect>();
@@ -309,7 +307,7 @@ absl::Status ILLIXROutputCalculator::Process(CalculatorContext* cc) {
     }
     if (cc->Inputs().HasTag(hand_map.at(hand_input)) &&
         !cc->Inputs().Tag(hand_map.at(hand_input)).IsEmpty()) {
-        ILLIXR::rect* h_rect;
+        ::ILLIXR::rect* h_rect;
         if (hand_input == hand_input_type::NORM_RECT || hand_input == hand_input_type::RECT) {
             if (hand_input == hand_input_type::NORM_RECT) {
                 const auto &rect = cc->Inputs().Tag(kNormHandRectTag).Get<NormalizedRect>();

diff --git a/mediapipe/calculators/util/illixr_output_calculator.h b/mediapipe/calculators/util/illixr_output_calculator.h
@@ -58,8 +58,8 @@ class ILLIXROutputCalculator : public CalculatorBase {
 };
 
 template<class T>
-ILLIXR::rect* make_rect(const T in_rect, bool normalized) {
-    return new ILLIXR::rect(in_rect.x_center(), in_rect.y_center(), in_rect.width(),
+::ILLIXR::rect* make_rect(const T in_rect, bool normalized) {
+    return new ::ILLIXR::rect(in_rect.x_center(), in_rect.y_center(), in_rect.width(),
                             in_rect.height(), in_rect.rotation(), normalized);
 }
 } // mediapipe
diff --git a/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc b/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc
@@ -34,9 +34,8 @@ namespace {
 constexpr char kLandmarksTag[] = "LANDMARKS";
 constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS";
 constexpr char kRenderScaleTag[] = "RENDER_SCALE";
-constexpr char kRenderDataTag[] = "RENDER_DATA";
+constexpr char kRenderDataTag[] = "RENDER_POINT_DATA";
 constexpr char kLandmarkLabel[] = "KEYPOINT";
-constexpr char kHandPoints[] = "HAND_POINTS";
 
 inline Color DefaultMinDepthLineColor() {
   Color color;

diff --git a/mediapipe/graphs/hand_tracking/build.cmake b/mediapipe/graphs/hand_tracking/build.cmake
@@ -1 +1,5 @@
 include(${CMAKE_CURRENT_LIST_DIR}/subgraphs/build.cmake)
+
+install(FILES hand_detection_desktop_live.pbtxt
+        DESTINATION shared/mediapipe/graphs/hand_tracking
+)
diff --git a/mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt b/mediapipe/graphs/hand_tracking/hand_tracking_desktop_live.pbtxt
@@ -7,7 +7,7 @@
 input_stream: "input_video"
 
 # CPU image. (ImageFrame)
-output_stream: "output_video"
+output_stream: "illixr_data"
 
 # Generates side packet cotaining max number of hands to detect/track.
 node {
@@ -42,5 +42,5 @@ node {
   input_stream: "HANDEDNESS:handedness"
   input_stream: "NORM_RECTS:0:multi_palm_rects"
   input_stream: "NORM_RECTS:1:multi_hand_rects"
-  output_stream: "IMAGE:output_video"
+  output_stream: "ILLIXR_DATA:illixr_data"
 }
diff --git a/mediapipe/graphs/hand_tracking/subgraphs/hand_renderer_cpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/hand_renderer_cpu.pbtxt
@@ -74,13 +74,11 @@ node {
   output_stream: "BATCH_END:landmark_timestamp"
 }
 
-
 # Converts landmarks to drawing primitives for annotation overlay.
 node {
   calculator: "LandmarksToRenderDataCalculator"
   input_stream: "NORM_LANDMARKS:single_hand_landmarks"
-  output_stream: "RENDER_DATA:single_hand_landmark_render_data"
-  output_stream: "HAND_POINTS:single_hand_landmark_points"
+  output_stream: "RENDER_POINT_DATA:single_hand_landmark_render_data"
   node_options: {
     [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
       landmark_connections: 0
@@ -208,14 +206,15 @@ node {
   input_stream: "handedness_render_data"
   input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
   output_stream: "IMAGE:output_image"
+  output_stream: "HAND_POINTS:multi_hand_landmark_points"
 }
 
 # combine the results into an ILLIXR data frame
 node {
   calculator: "ILLIXROutputCalculator"
   input_stream: "IMAGE:output_image"
   input_stream: "HANDEDNESS:multi_handedness"
-  input_stream: "HAND_POINTS:single_hand_landmark_points"
+  input_stream: "HAND_POINTS:multi_hand_landmark_points"
   input_stream: "NORM_RECTS_PALM:multi_palm_rects"
   input_stream: "NORM_RECTS_HAND:multi_hand_rects"
   output_stream: "ILLIXR_DATA:illixr_data"

diff --git a/mediapipe/modules/hand_landmark/build.cmake b/mediapipe/modules/hand_landmark/build.cmake
@@ -1,5 +1,22 @@
 include(${CMAKE_SOURCE_DIR}/cmake/make_pb_binary.cmake)
 
+configure_file(${CMAKE_SOURCE_DIR}/mediapipe/modules/hand_landmark/hand_landmark_model_loader.pbtxt.in
+               ${CMAKE_SOURCE_DIR}/mediapipe/modules/hand_landmark/hand_landmark_model_loader.pbtxt
+               @ONLY
+)
+
+if(HT_ENABLE_GPU)
+    configure_file(${CMAKE_SOURCE_DIR}/mediapipe/modules/hand_landmark/hand_landmark_gpu.pbtxt.in
+                   ${CMAKE_SOURCE_DIR}/mediapipe/modules/hand_landmark/hand_landmark_gpu.pbtxt
+                   @ONLY
+    )
+else()
+    configure_file(${CMAKE_SOURCE_DIR}/mediapipe/modules/hand_landmark/hand_landmark_cpu.pbtxt.in
+                   ${CMAKE_SOURCE_DIR}/mediapipe/modules/hand_landmark/hand_landmark_cpu.pbtxt
+                   @ONLY
+    )
+endif()
+
 set(HLLR_LIBRARIES
     $<TARGET_OBJECTS:calculators.internal.callback_packet_calculator_proto>
     $<TARGET_OBJECTS:calculators.util.rect_transformation_calculator_proto>
@@ -215,3 +232,10 @@ make_proto_binary(BINARY_NAME palm_detection_detection_to_roi_graph_text_to_bina
                   PROTO_LIBRARIES ${PDDR_LIBRARIES})
 
 include(${CMAKE_CURRENT_LIST_DIR}/calculators/build.cmake)
+
+install(FILES
+        ${CMAKE_SOURCE_DIR}/mediapipe/modules/hand_landmark/hand_landmark_full.tflite
+        ${CMAKE_SOURCE_DIR}/mediapipe/modules/hand_landmark/hand_landmark_lite.tflite
+        ${CMAKE_SOURCE_DIR}/mediapipe/modules/hand_landmark/handedness.txt
+        DESTINATION ${CMAKE_INSTALL_PREFIX}/share/mediapipe/modules/hand_landmark
+)
diff --git a/...les/hand_landmark/hand_landmark_cpu.pbtxt → .../hand_landmark/hand_landmark_cpu.pbtxt.in b/...les/hand_landmark/hand_landmark_cpu.pbtxt → .../hand_landmark/hand_landmark_cpu.pbtxt.in
@@ -136,7 +136,7 @@ node {
   options: {
     [mediapipe.TensorsToClassificationCalculatorOptions.ext] {
       top_k: 1
-      label_map_path: "mediapipe/modules/hand_landmark/handedness.txt"
+      label_map_path: "@CMAKE_INSTALL_PREFIX@/share/mediapipe/modules/hand_landmark/handedness.txt"
       binary_classification: true
     }
   }

diff --git a/...les/hand_landmark/hand_landmark_gpu.pbtxt → .../hand_landmark/hand_landmark_gpu.pbtxt.in b/...les/hand_landmark/hand_landmark_gpu.pbtxt → .../hand_landmark/hand_landmark_gpu.pbtxt.in
@@ -130,7 +130,7 @@ node {
   options: {
     [mediapipe.TensorsToClassificationCalculatorOptions.ext] {
       top_k: 1
-      label_map_path: "mediapipe/modules/hand_landmark/handedness.txt"
+      label_map_path: "@CMAKE_INSTALL_PREFIX@/share/mediapipe/modules/hand_landmark/handedness.txt"
       binary_classification: true
     }
   }

diff --git a/...landmark/hand_landmark_model_loader.pbtxt → ...dmark/hand_landmark_model_loader.pbtxt.in b/...landmark/hand_landmark_model_loader.pbtxt → ...dmark/hand_landmark_model_loader.pbtxt.in
@@ -24,7 +24,7 @@ node {
         options: {
           [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
             packet {
-              string_value: "mediapipe/modules/hand_landmark/hand_landmark_lite.tflite"
+              string_value: "@CMAKE_INSTALL_PREFIX@/share/mediapipe/modules/hand_landmark/hand_landmark_lite.tflite"
             }
           }
         }
@@ -34,7 +34,7 @@ node {
         options: {
           [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
             packet {
-              string_value: "mediapipe/modules/hand_landmark/hand_landmark_full.tflite"
+              string_value: "@CMAKE_INSTALL_PREFIX@/share/mediapipe/modules/hand_landmark/hand_landmark_full.tflite"
             }
           }
         }

diff --git a/mediapipe/modules/palm_detection/build.cmake b/mediapipe/modules/palm_detection/build.cmake
@@ -1,5 +1,10 @@
 include(${CMAKE_SOURCE_DIR}/cmake/make_pb_binary.cmake)
 
+configure_file(${CMAKE_SOURCE_DIR}/mediapipe/modules/palm_detection/palm_detection_model_loader.pbtxt.in
+               ${CMAKE_SOURCE_DIR}/mediapipe/modules/palm_detection/palm_detection_model_loader.pbtxt
+               @ONLY
+)
+
 set(PDML_PROTO_LIBRARIES
     $<TARGET_OBJECTS:calculators.util.local_file_contents_calculator_proto>
     $<TARGET_OBJECTS:calculators.core.constant_side_packet_calculator_proto>
@@ -63,3 +68,9 @@ make_proto_binary(BINARY_NAME palm_detection_graph_text_to_binary_graph
                   FILE_BASE_NAME palm_detection_${NAME_POST}
                   CLASS_NAME PalmDetection${CLASS_POST}
                   PROTO_LIBRARIES ${PDGT_LIBRARIES})
+
+install(FILES
+        ${CMAKE_SOURCE_DIR}/mediapipe/modules/palm_detection/palm_detection_full.tflite
+        ${CMAKE_SOURCE_DIR}/mediapipe/modules/palm_detection/palm_detection_lite.tflite
+        DESTINATION ${CMAKE_INSTALL_PREFIX}/share/mediapipe/modules/palm_detection
+)