[Hackthon_4th 177] Support PP-YOLOE-R with BM1684

benchmark/cpp/CMakeLists.txt

@@ -22,6 +22,8 @@ add_executable(benchmark_ppmatting ${PROJECT_SOURCE_DIR}/benchmark_ppmatting.cc)
 add_executable(benchmark_ppocr_det ${PROJECT_SOURCE_DIR}/benchmark_ppocr_det.cc)
 add_executable(benchmark_ppocr_cls ${PROJECT_SOURCE_DIR}/benchmark_ppocr_cls.cc)
 add_executable(benchmark_ppocr_rec ${PROJECT_SOURCE_DIR}/benchmark_ppocr_rec.cc)
+add_executable(benchmark_ppyoloe_r ${PROJECT_SOURCE_DIR}/benchmark_ppyoloe_r.cc)
+add_executable(benchmark_ppyoloe_r_sophgo ${PROJECT_SOURCE_DIR}/benchmark_ppyoloe_r_sophgo.cc)
 add_executable(benchmark_ppyolo ${PROJECT_SOURCE_DIR}/benchmark_ppyolo.cc)
 add_executable(benchmark_yolov3 ${PROJECT_SOURCE_DIR}/benchmark_yolov3.cc)
 add_executable(benchmark_fasterrcnn ${PROJECT_SOURCE_DIR}/benchmark_fasterrcnn.cc)
@@ -44,6 +46,8 @@ if(UNIX AND (NOT APPLE) AND (NOT ANDROID))
   target_link_libraries(benchmark_ppyolov8 ${FASTDEPLOY_LIBS} gflags pthread)
   target_link_libraries(benchmark_ppyolox ${FASTDEPLOY_LIBS} gflags pthread)
   target_link_libraries(benchmark_ppyoloe ${FASTDEPLOY_LIBS} gflags pthread)
+  target_link_libraries(benchmark_ppyoloe_r ${FASTDEPLOY_LIBS} gflags pthread)
+  target_link_libraries(benchmark_ppyoloe_r_sophgo ${FASTDEPLOY_LIBS} gflags pthread)
   target_link_libraries(benchmark_picodet ${FASTDEPLOY_LIBS} gflags pthread)
   target_link_libraries(benchmark_ppcls ${FASTDEPLOY_LIBS} gflags pthread)
   target_link_libraries(benchmark_ppseg ${FASTDEPLOY_LIBS} gflags pthread)
@@ -72,6 +76,8 @@ else()
   target_link_libraries(benchmark_ppyolov8 ${FASTDEPLOY_LIBS} gflags)
   target_link_libraries(benchmark_ppyolox ${FASTDEPLOY_LIBS} gflags)
   target_link_libraries(benchmark_ppyoloe ${FASTDEPLOY_LIBS} gflags)
+  target_link_libraries(benchmark_ppyoloe_r ${FASTDEPLOY_LIBS} gflags)
+  target_link_libraries(benchmark_ppyoloe_r_sophgo ${FASTDEPLOY_LIBS} gflags)
   target_link_libraries(benchmark_picodet ${FASTDEPLOY_LIBS} gflags)
   target_link_libraries(benchmark_ppcls ${FASTDEPLOY_LIBS} gflags)
   target_link_libraries(benchmark_ppseg ${FASTDEPLOY_LIBS} gflags)

benchmark/cpp/benchmark_ppyoloe_r.cc

@@ -0,0 +1,167 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fstream>
+
+#include "flags.h"
+#include "macros.h"
+#include "option.h"
+
+namespace vision = fastdeploy::vision;
+namespace benchmark = fastdeploy::benchmark;
+
+DEFINE_bool(no_nms, false, "Whether the model contains nms.");
+
+static inline void StrSplitToFloat(std::vector<float>& res,
+                                   const std::string& s, const std::string& c) {
+  std::string::size_type pos1, pos2;
+  pos2 = s.find(c);
+  pos1 = 0;
+  while (std::string::npos != pos2) {
+    res.emplace_back(std::stof(s.substr(pos1, pos2 - pos1)));
+    pos1 = pos2 + strlen(c.c_str());
+    pos2 = s.find(c, pos1);
+  }
+  res.emplace_back(std::stof(s.substr(pos1)));
+}
+
+void showDiffStats(const std::vector<float>& data, const std::string& title) {
+  float max{0.0};
+  float min{0.0};
+  float average{0.0};
+
+  if (data.size() == 0) {
+    std::cout << title << ", max: " << max << ", min: " << min
+              << ", mean: " << average << std::endl;
+    return;
+  }
+  double sum = accumulate(begin(data), end(data), 0.0);
+  double mean = sum / data.size();
+  max = *max_element(data.begin(), data.end());
+  min = *min_element(data.begin(), data.end());
+  average = mean;
+  std::cout << title << " diff, max: " << max << ", min: " << min
+            << ", mean: " << average << std::endl;
+}
+
+void sortBoxes(vision::DetectionResult* result, std::vector<int>* indices) {
+  if (result->rotated_boxes.empty()) {
+    return;
+  }
+
+  indices->clear();
+  indices->resize(result->rotated_boxes.size());
+  for (int i = 0; i < result->rotated_boxes.size(); ++i) {
+    indices->at(i) = i;
+  }
+
+  auto& rotated_boxes = result->rotated_boxes;
+  std::sort(indices->begin(), indices->end(), [&rotated_boxes](int a, int b) {
+    if (rotated_boxes[a][0] == rotated_boxes[b][0]) {
+      return rotated_boxes[a][1] > rotated_boxes[b][1];
+    }
+    return rotated_boxes[a][0] > rotated_boxes[b][0];
+  });
+}
+
+int main(int argc, char* argv[]) {
+#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
+  // Initialization
+  auto option = fastdeploy::RuntimeOption();
+  if (!CreateRuntimeOption(&option, argc, argv, true)) {
+    return -1;
+  }
+  auto im = cv::imread(FLAGS_image);
+  std::unordered_map<std::string, std::string> config_info;
+  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
+                                                &config_info);
+  std::string model_name, params_name, config_name;
+  auto model_format = fastdeploy::ModelFormat::PADDLE;
+  if (!UpdateModelResourceName(&model_name, &params_name, &config_name,
+                               &model_format, config_info)) {
+    return -1;
+  }
+  auto model_file = FLAGS_model + sep + model_name;
+  auto params_file = FLAGS_model + sep + params_name;
+  auto config_file = FLAGS_model + sep + config_name;
+
+  auto model_ppyoloe_r = vision::detection::PPYOLOE_R(
+      model_file, params_file, config_file, option, model_format);
+
+  vision::DetectionResult res;
+  if (config_info["precision_compare"] == "true") {
+    // Run once at least
+    model_ppyoloe_r.Predict(im, &res);
+    // 1. Test result diff
+    std::cout << "=============== Test result diff =================\n";
+
+    // result from PaddleDetection, cls_id, conf, boxes(8)
+    std::string det_result_path = "ppyoloe_r_result.txt";
+    std::ifstream ifs(det_result_path.c_str());
+    std::string expect_res((std::istreambuf_iterator<char>(ifs)),
+                           std::istreambuf_iterator<char>());
+    ifs.close();
+    std::vector<float> expect_data;
+    StrSplitToFloat(expect_data, expect_res, ",");
+
+    std::vector<float> label_diff;
+    std::vector<float> score_diff;
+    std::vector<float> boxes_diff;
+    int num = int(res.label_ids.size());
+    if (expect_data.size() == num) {
+      std::cout << "boxes num is the same: " << expect_data.size() << std::endl;
+    }
+
+    vision::DetectionResult expect_result;
+    for (int i = 0; i < num; i++) {
+      expect_result.label_ids.push_back(
+          static_cast<int32_t>(expect_data[10 * i]));
+      expect_result.scores.push_back(expect_data[10 * i + 1]);
+      std::array<float, 8> box;
+      for (int j = 0; j < 8; j++) {
+        box[j] = expect_data[10 * i + 2 + j];
+      }
+      expect_result.rotated_boxes.push_back(box);
+    }
+    std::vector<int> exp_indices;
+    sortBoxes(&expect_result, &exp_indices);
+
+    std::vector<int> res_indices;
+    sortBoxes(&res, &res_indices);
+
+    for (int i = 0; i < num; i++) {
+      int res_idx = res_indices[i];
+      int exp_idx = exp_indices[i];
+      label_diff.push_back(
+          std::abs(expect_result.label_ids[exp_idx] - res.label_ids[res_idx]));
+      score_diff.push_back(
+          std::abs(expect_result.scores[exp_idx] - res.scores[res_idx]));
+      for (int j = 0; j < 8; j++) {
+        boxes_diff.push_back(std::abs(expect_result.rotated_boxes[exp_idx][j] -
+                                      res.rotated_boxes[res_idx][j]));
+      }
+    }
+    showDiffStats(label_diff, "labels");
+    showDiffStats(score_diff, "scores");
+    showDiffStats(boxes_diff, "boxes");
+  }
+  // Run profiling
+  BENCHMARK_MODEL(model_ppyoloe_r, model_ppyoloe_r.Predict(im, &res))
+
+  auto vis_im = vision::VisDetection(im, res);
+  cv::imwrite("vis_result.jpg", vis_im);
+  std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl;
+#endif
+  return 0;
+}

benchmark/cpp/benchmark_ppyoloe_r_sophgo.cc

@@ -0,0 +1,159 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <fstream>
+
+#include "flags.h"
+#include "macros.h"
+#include "option.h"
+
+namespace vision = fastdeploy::vision;
+namespace benchmark = fastdeploy::benchmark;
+
+DEFINE_bool(no_nms, false, "Whether the model contains nms.");
+
+static inline void StrSplitToFloat(std::vector<float>& res,
+                                   const std::string& s, const std::string& c) {
+  std::string::size_type pos1, pos2;
+  pos2 = s.find(c);
+  pos1 = 0;
+  while (std::string::npos != pos2) {
+    res.emplace_back(std::stof(s.substr(pos1, pos2 - pos1)));
+    pos1 = pos2 + strlen(c.c_str());
+    pos2 = s.find(c, pos1);
+  }
+  res.emplace_back(std::stof(s.substr(pos1)));
+}
+
+void showDiffStats(const std::vector<float>& data, const std::string& title) {
+  float max{0.0};
+  float min{0.0};
+  float average{0.0};
+
+  if (data.size() == 0) {
+    std::cout << title << ", max: " << max << ", min: " << min
+              << ", mean: " << average << std::endl;
+    return;
+  }
+  double sum = accumulate(begin(data), end(data), 0.0);
+  double mean = sum / data.size();
+  max = *max_element(data.begin(), data.end());
+  min = *min_element(data.begin(), data.end());
+  average = mean;
+  std::cout << title << " diff, max: " << max << ", min: " << min
+            << ", mean: " << average << std::endl;
+}
+
+void sortBoxes(vision::DetectionResult* result, std::vector<int>* indices) {
+  if (result->rotated_boxes.empty()) {
+    return;
+  }
+
+  indices->clear();
+  indices->resize(result->rotated_boxes.size());
+  for (int i = 0; i < result->rotated_boxes.size(); ++i) {
+    indices->at(i) = i;
+  }
+
+  auto& rotated_boxes = result->rotated_boxes;
+  std::sort(indices->begin(), indices->end(), [&rotated_boxes](int a, int b) {
+    if (rotated_boxes[a][0] == rotated_boxes[b][0]) {
+      return rotated_boxes[a][1] > rotated_boxes[b][1];
+    }
+    return rotated_boxes[a][0] > rotated_boxes[b][0];
+  });
+}
+
+int main(int argc, char* argv[]) {
+#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
+  // Initialization
+  auto option = fastdeploy::RuntimeOption();
+  if (!CreateRuntimeOption(&option, argc, argv, true)) {
+    return -1;
+  }
+  auto im = cv::imread(FLAGS_image);
+  std::unordered_map<std::string, std::string> config_info;
+  benchmark::ResultManager::LoadBenchmarkConfig(FLAGS_config_path,
+                                                &config_info);
+  std::string model_name, params_name, config_name;
+  auto model_format = fastdeploy::ModelFormat::SOPHGO;
+  auto model_file =
+      FLAGS_model + sep + "ppyoloe_r_crn_s_3x_dota_1684x_f32.bmodel";
+  auto params_file = "";
+  auto config_file = FLAGS_model + sep + "infer_cfg.yml";
+
+  auto model_ppyoloe_r = vision::detection::PPYOLOE_R(
+      model_file, params_file, config_file, option, model_format);
+
+  vision::DetectionResult res;
+  if (config_info["precision_compare"] == "true") {
+    // Run once at least
+    model_ppyoloe_r.Predict(im, &res);
+    // 1. Test result diff
+    std::cout << "=============== Test result diff =================\n";
+
+    // result from PaddleDetection, cls_id, conf, boxes(8)
+    std::string det_result_path = "ppyoloe_r_result.txt";
+    std::ifstream ifs(det_result_path.c_str());
+    std::string expect_res((std::istreambuf_iterator<char>(ifs)),
+                           std::istreambuf_iterator<char>());
+    ifs.close();
+    std::vector<float> expect_data;
+    StrSplitToFloat(expect_data, expect_res, ",");
+
+    std::vector<float> label_diff;
+    std::vector<float> score_diff;
+    std::vector<float> boxes_diff;
+    int num = int(res.label_ids.size());
+    if (expect_data.size() == num) {
+      std::cout << "boxes num is the same: " << expect_data.size() << std::endl;
+    }
+
+    vision::DetectionResult expect_result;
+    for (int i = 0; i < num; i++) {
+      expect_result.label_ids.push_back(
+          static_cast<int32_t>(expect_data[10 * i]));
+      expect_result.scores.push_back(expect_data[10 * i + 1]);
+      std::array<float, 8> box;
+      for (int j = 0; j < 8; j++) {
+        box[j] = expect_data[10 * i + 2 + j];
+      }
+      expect_result.rotated_boxes.push_back(box);
+    }
+    std::vector<int> exp_indices;
+    sortBoxes(&expect_result, &exp_indices);
+
+    std::vector<int> res_indices;
+    sortBoxes(&res, &res_indices);
+
+    for (int i = 0; i < num; i++) {
+      int res_idx = res_indices[i];
+      int exp_idx = exp_indices[i];
+      label_diff.push_back(
+          std::abs(expect_result.label_ids[exp_idx] - res.label_ids[res_idx]));
+      score_diff.push_back(
+          std::abs(expect_result.scores[exp_idx] - res.scores[res_idx]));
+      for (int j = 0; j < 8; j++) {
+        boxes_diff.push_back(std::abs(expect_result.rotated_boxes[exp_idx][j] -
+                                      res.rotated_boxes[res_idx][j]));
+      }
+    }
+    showDiffStats(label_diff, "labels");
+    showDiffStats(score_diff, "scores");
+    showDiffStats(boxes_diff, "boxes");
+  }
+
+#endif
+  return 0;
+}

benchmark/cpp/option.h

@@ -103,6 +103,9 @@ static bool CreateRuntimeOption(fastdeploy::RuntimeOption* option,
       if (config_info["use_fp16"] == "true") {
         option->paddle_lite_option.enable_fp16 = true;
       }
+    } else if (config_info["backend"] == "sophgo") {
+      option->UseSophgo();
+      option->UseSophgoBackend();
     } else if (config_info["backend"] == "default") {
       PrintBenchmarkInfo(config_info);
       return true;

c_api/fastdeploy_capi/vision/result.h

@@ -50,6 +50,7 @@ typedef struct FD_C_OneDimMask {
 
 typedef struct FD_C_DetectionResult {
   FD_C_TwoDimArrayFloat boxes;
+  FD_C_TwoDimArrayFloat rotated_boxes;
   FD_C_OneDimArrayFloat scores;
   FD_C_OneDimArrayInt32 label_ids;
   FD_C_OneDimMask masks;

csharp/fastdeploy/types_internal_c.cs

@@ -135,6 +135,7 @@ public struct FD_OneDimMask {
 [StructLayout(LayoutKind.Sequential)]
 public struct FD_DetectionResult {
   public FD_TwoDimArrayFloat boxes;
+  public FD_TwoDimArrayFloat rotated_boxes;
   public FD_OneDimArrayFloat scores;
   public FD_OneDimArrayInt32 label_ids;
   public FD_OneDimMask masks;

examples/vision/detection/paddledetection/README.md

@@ -10,6 +10,7 @@ English | [简体中文](README_CN.md)
 Now FastDeploy supports the deployment of the following models
 
 - [PP-YOLOE(including PP-YOLOE+) models](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyoloe)
+- [PP-YOLOE-R models](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.6/configs/rotate/ppyoloe_r)
 - [PicoDet models](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/picodet)
 - [PP-YOLO models(including v2)](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/ppyolo)
 - [YOLOv3 models](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.4/configs/yolov3)
@@ -43,7 +44,7 @@ Before deployment, PaddleDetection needs to be exported into the deployment mode
 
 ## Download Pre-trained Model
 
-For developers' testing, models exported by PaddleDetection are provided below. Developers can download them directly. 
+For developers' testing, models exported by PaddleDetection are provided below. Developers can download them directly.
 
 The accuracy metric is from model descriptions in PaddleDetection. Refer to them for details.