【Hackthon_4th 180】Support HORIZON BPU Backend for FastDeploy

CMakeLists.txt

@@ -63,6 +63,7 @@ option(ENABLE_OPENVINO_BACKEND "Whether to enable openvino backend." OFF)
 option(ENABLE_RKNPU2_BACKEND "Whether to enable RKNPU2 backend." OFF)
 option(ENABLE_SOPHGO_BACKEND "Whether to enable SOPHON backend." OFF)
 option(ENABLE_LITE_BACKEND "Whether to enable paddle lite backend." OFF)
+option(ENABLE_HORIZON_BACKEND "Whether to enable HORIZON backend." OFF)
 option(ENABLE_VISION "Whether to enable vision models usage." OFF)
 option(ENABLE_TEXT "Whether to enable text models usage." OFF)
 option(ENABLE_FLYCV "Whether to enable flycv to boost image preprocess." OFF)
@@ -165,6 +166,7 @@ file(GLOB_RECURSE DEPLOY_POROS_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fast
 file(GLOB_RECURSE DEPLOY_TRT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/tensorrt/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/tensorrt/*.cpp)
 file(GLOB_RECURSE DEPLOY_OPENVINO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/openvino/*.cc)
 file(GLOB_RECURSE DEPLOY_RKNPU2_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/rknpu2/*.cc)
+file(GLOB_RECURSE DEPLOY_HORIZON_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/horizon/*.cc)
 file(GLOB_RECURSE DEPLOY_SOPHGO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/sophgo/*.cc)
 file(GLOB_RECURSE DEPLOY_LITE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/runtime/backends/lite/*.cc)
 file(GLOB_RECURSE DEPLOY_ENCRYPTION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/fastdeploy/encryption/*.cc)
@@ -185,7 +187,7 @@ list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS}
                                  ${DEPLOY_OPENVINO_SRCS} ${DEPLOY_LITE_SRCS}
                                  ${DEPLOY_VISION_SRCS} ${DEPLOY_TEXT_SRCS}
                                  ${DEPLOY_PIPELINE_SRCS} ${DEPLOY_RKNPU2_SRCS}
-                                 ${DEPLOY_SOPHGO_SRCS} ${DEPLOY_ENCRYPTION_SRCS})
+                                 ${DEPLOY_SOPHGO_SRCS} ${DEPLOY_ENCRYPTION_SRCS} ${DEPLOY_HORIZON_SRCS})
 
 set(DEPEND_LIBS "")
 
@@ -253,6 +255,13 @@ if(ENABLE_RKNPU2_BACKEND)
   list(APPEND DEPEND_LIBS ${RKNN_RT_LIB})
 endif()
 
+if(ENABLE_HORIZON_BACKEND)
+  add_definitions(-DENABLE_HORIZON_BACKEND)
+  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_HORIZON_SRCS})
+  include(${PROJECT_SOURCE_DIR}/cmake/horizon.cmake)
+  list(APPEND DEPEND_LIBS ${BPU_libs})
+endif()
+
 if(ENABLE_SOPHGO_BACKEND)
   add_definitions(-DENABLE_SOPHGO_BACKEND)
   list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_SOPHGO_SRCS})

FastDeploy.cmake.in

@@ -23,6 +23,7 @@ set(RKNN2_TARGET_SOC "@RKNN2_TARGET_SOC@")
 # Inference backend and FastDeploy Moudle
 set(ENABLE_ORT_BACKEND @ENABLE_ORT_BACKEND@)
 set(ENABLE_RKNPU2_BACKEND @ENABLE_RKNPU2_BACKEND@)
+set(ENABLE_HORIZON_BACKEND @ENABLE_HORIZON_BACKEND@)
 set(ENABLE_SOPHGO_BACKEND @ENABLE_SOPHGO_BACKEND@)
 set(ENABLE_LITE_BACKEND @ENABLE_LITE_BACKEND@)
 set(ENABLE_PADDLE_BACKEND @ENABLE_PADDLE_BACKEND@)
@@ -167,6 +168,21 @@ if(ENABLE_RKNPU2_BACKEND)
   list(APPEND FASTDEPLOY_LIBS ${RKNPU2_LIB})
 endif()
 
+if(ENABLE_HORIZON_BACKEND)
+  set(DNN_PATH ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/dnn)
+  set(APPSDK_PATH ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/appsdk/appuser/)
+
+  set(DNN_LIB_PATH ${DNN_PATH}/lib)
+  set(APPSDK_LIB_PATH ${APPSDK_PATH}/lib/hbbpu)
+  set(BPU_libs dnn cnn_intf hbrt_bernoulli_aarch64)
+
+  link_directories(${DNN_LIB_PATH}
+                  ${APPSDK_PATH}/lib/hbbpu
+                  ${APPSDK_PATH}/lib)
+
+  list(APPEND FASTDEPLOY_LIBS ${BPU_libs})
+
+endif()
 if(ENABLE_LITE_BACKEND)
   set(LITE_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/${PADDLELITE_FILENAME})
   if(ANDROID)

cmake/horizon.cmake

@@ -0,0 +1,24 @@
+# get Horizon_URL
+set(HORIZON_URL_BASE "https://bj.bcebos.com/fastdeploy/third_libs/")
+
+set(HORIZON_VERSION "2.5.2")
+set(HORIZON_FILE "horizon_runtime-xj3-aarch64-${HORIZON_VERSION}.tgz")
+set(HORIZON_URL "${HORIZON_URL_BASE}${HORIZON_FILE}")
+
+# download_and_decompress
+download_and_decompress(${HORIZON_URL} ${CMAKE_CURRENT_BINARY_DIR}/${HORIZON_FILE} ${THIRD_PARTY_PATH}/install)
+# set path
+set(HORIZON_RUNTIME_PATH ${THIRD_PARTY_PATH}/install/)
+
+set(DNN_PATH ${HORIZON_RUNTIME_PATH}/dnn/)
+set(APPSDK_PATH ${HORIZON_RUNTIME_PATH}/appsdk/appuser/)
+
+set(DNN_LIB_PATH ${DNN_PATH}/lib)
+set(APPSDK_LIB_PATH ${APPSDK_PATH}/lib/hbbpu)
+set(BPU_libs dnn cnn_intf hbrt_bernoulli_aarch64)
+
+include_directories(${DNN_PATH}/include
+                    ${APPSDK_PATH}/include)
+link_directories(${DNN_LIB_PATH}
+                ${APPSDK_PATH}/lib/hbbpu
+                ${APPSDK_PATH}/lib)

cmake/summary.cmake

@@ -33,6 +33,7 @@ function(fastdeploy_summary)
   message(STATUS "  FastDeploy version        : ${FASTDEPLOY_VERSION}")
   message(STATUS "  ENABLE_ORT_BACKEND        : ${ENABLE_ORT_BACKEND}")
   message(STATUS "  ENABLE_RKNPU2_BACKEND     : ${ENABLE_RKNPU2_BACKEND}")
+  message(STATUS "  ENABLE_HORIZON_BACKEND    : ${ENABLE_HORIZON_BACKEND}")
   message(STATUS "  ENABLE_SOPHGO_BACKEND     : ${ENABLE_SOPHGO_BACKEND}")
   message(STATUS "  ENABLE_PADDLE_BACKEND     : ${ENABLE_PADDLE_BACKEND}")
   message(STATUS "  ENABLE_LITE_BACKEND       : ${ENABLE_LITE_BACKEND}")

docs/cn/build_and_install/horizon.md

@@ -0,0 +1,76 @@
+[English](../../en/build_and_install/horizon.md) | 简体中文
+
+# FastDeploy Horizon 导航文档
+
+Horizon指的是地平线推出的旭日X3系列芯片的BPU。
+目前FastDeploy已经初步支持使用Horizon来部署模型。
+如果您在使用的过程中出现问题，请附带上您的运行环境，在Issues中反馈。
+
+## FastDeploy Horizon 环境安装简介
+
+如果您想在FastDeploy中使用Horizon推理引擎，你需要配置以下几个环境。
+
+| 工具名          | 是否必须 | 安装设备  | 用途                              |  
+|--------------|------|-------|---------------------------------|
+| Paddle2ONNX  | 必装   | PC    | 用于转换PaddleInference模型到ONNX模型    |  
+| 地平线XJ3芯片工具链镜像 | 必装   | PC    | 用于转换ONNX模型到地平线模型               |  
+| 地平线 XJ3 OpenExplorer       | 必装   | PC | 地平线模型转换的关键头文件和动态库 |
+
+## 安装模型转换环境
+
+地平线提供了一套完整的模型转换环境（XJ3芯片工具链镜像），FastDeploy采用的镜像版本为[2.5.2](ftp://vrftp.horizon.ai/Open_Explorer_gcc_9.3.0/2.5.2/docker_openexplorer_ubuntu_20_xj3_gpu_v2.5.2_py38.tar.gz)，你可以通过地平线开发者平台获取。
+
+
+## 安装必备的依赖软件包
+
+地平线同样提供了一整套工具包(地平线 XJ3 OpenExplorer)，FastDeploy采用的开发包版本为[2.5.2](ftp://vrftp.horizon.ai/Open_Explorer_gcc_9.3.0/2.5.2/horizon_xj3_openexplorer_v2.5.2_py38_20230331.tar.gz),你可以通过地平线开发者平台获取。
+
+由于板端CPU性能较弱，所以推荐在PC机上进行交叉编译。以下教程在地平线提供的docker环境下完成。
+
+### 启动docker环境
+将地平线XJ3芯片工具链镜像下载到本地之后，执行如下命令，将镜像包导入docker环境：
+
+```bash
+docker load < docker_openexplorer_ubuntu_20_xj3_gpu_v2.5.2_py38.tar.gz
+```
+将依赖的软件包下载至本地之后，解压：
+```bash
+tar -xvf horizon_xj3_openexplorer_v2.5.2_py38_20230331.tar.gz
+```
+解压完成之后，cd至改目录：
+```bash
+cd horizon_xj3_open_explorer_v2.5.2-py38_20230331/
+```
+
+根目录下有运行docker的脚本，运行以下命令：
+```bash
+sh run_docker.sh /home gpu
+```
+
+第一个目录为要挂载到容器上的目录，后一个参数为该docker启用gpu进行加速。
+
+至此，所需环境准备完毕。
+
+## 安装FastDeploy C++ SDK
+下载交叉编译工具，[gcc_linaro_6.5.0_2018.12_x86_64_aarch64_linux_gnu](https://bj.bcebos.com/fastdeploy/third_libs/gcc_linaro_6.5.0_2018.12_x86_64_aarch64_linux_gnu.tar.xz)，建议解压后放到`/opt`目录下。
+```bash
+git clone https://github.com/PaddlePaddle/FastDeploy.git
+cd FastDeploy
+
+# 如果您使用的是develop分支输入以下命令
+git checkout develop
+
+mkdir build && cd build
+cmake ..  -DCMAKE_C_COMPILER=/opt/gcc_linaro_6.5.0_2018.12_x86_64_aarch64_linux_gnu/gcc-linaro-6.5.0-2018.12-x86_64_aarch64-linux-gnu/bin/aarch64-linux-gnu-gcc \
+          -DCMAKE_CXX_COMPILER=/opt/gcc_linaro_6.5.0_2018.12_x86_64_aarch64_linux_gnu/gcc-linaro-6.5.0-2018.12-x86_64_aarch64-linux-gnu/bin/aarch64-linux-gnu-g++ \
+          -DCMAKE_TOOLCHAIN_FILE=./../cmake/toolchain.cmake \
+          -DTARGET_ABI=arm64 \
+          -WITH_TIMVX=ON \
+          -DENABLE_HORIZON_BACKEND=ON \
+          -DENABLE_VISION=ON \
+          -DCMAKE_INSTALL_PREFIX=${PWD}/fastdeploy-0.0.0 \
+          -Wno-dev ..
+make -j16
+make install
+```
+

docs/cn/faq/horizon/export.md

@@ -0,0 +1,179 @@
+[English](../../../en/faq/horizon/export.md) | 中文
+
+# 导出模型指南
+
+## 简介
+
+地平线的模型转换以及量化工具均封装在提供的docker镜像中，在进行模型转换前请根据[Horizon导航文档](../../build_and_install/horizon.md)将环境安装成功。
+
+## 转换流程
+由于地平线不支持直接从paddle模型到horizon模型的转换，因此，首先要将paddle的模型转换为ONNX模型，地平线目前主要支持的opset版本是opset10和opset11，ir_version <= 7，转换过程需特别注意，具体可参考[地平线提供的官方文档](https://developer.horizon.ai/api/v1/fileData/doc/ddk_doc/navigation/ai_toolchain/docs_cn/horizon_ai_toolchain_user_guide/model_conversion.html#fp-model-preparation)。
+
+paddle模型转ONNX模型可以运行以下命令
+```bash
+## 转为ONNX
+paddle2onnx --model_dir model/ \
+            --model_filename inference.pdmodel \
+            --params_filename inference.pdiparams \
+            --save_file model.onnx \
+            --enable_dev_version True \
+            --opset_version 11
+```
+更改ir_version，可参考以下代码
+```python
+import onnx
+model = onnx.load("model.onnx")
+model.ir_version = 7
+onnx.save(model, "model.onnx")
+```
+转换为ONNX模型之后，开始进行地平线模型的转换，可参考官方文档进行转换，此处给出MobileNetv2的模型转换示例。
+
+进入docker实例，cd至如下目录:
+
+```bash
+cd ddk/samples/ai_toolchain/horizon_model_convert_sample/03_classification/04_mobilenet_onnx/mapper/
+```
+该目录下有`01_check.sh`，`02_preprocess.sh`，`03_build.sh`，在模型转换阶段主要用这三个脚本就可完成，下面详细介绍使用这三个脚本的注意事项。
+
+`01_check.sh`，对模型以及运行环境进行检查，只需要修改`caffe_model`为自己的ONNX模型路径就可以完成。
+
+```bash
+set -ex
+cd $(dirname $0) || exit
+
+model_type="onnx"
+caffe_model="../../../01_common/model_zoo/mapper/classification/mobilenet_onnx/mobilenetv2.onnx"
+march="bernoulli2"
+
+hb_mapper checker --model-type ${model_type} \
+                  --model ${caffe_model} \
+                  --march ${march}
+```
+
+`02_preprocess.sh`，准备量化所需的数据格式，FastDeploy选择以下的配置。
+
+```bash
+python3 ../../../data_preprocess.py \
+  --src_dir ../../../01_common/calibration_data/imagenet \
+  --dst_dir ./calibration_data_rgb \
+  --pic_ext .rgb \
+  --read_mode opencv \
+  --saved_data_type uint8
+```
+
+`03_build.sh`，将ONNX模型转换为Horizon可运行的模型，其中转换需要进行参数的配置，包括输入数据格式等。
+FastDeploy对模型路径的配置如下：
+
+```yaml
+model_parameters:
+  # the model file of floating-point ONNX neural network data
+  onnx_model: '../../../01_common/model_zoo/mapper/classification/mobilenet_onnx/mobilenetv2.onnx'
+
+  # the applicable BPU architecture
+  march: "bernoulli2"
+
+  # specifies whether or not to dump the intermediate results of all layers in conversion
+  # if set to True, then the intermediate results of all layers shall be dumped
+  layer_out_dump: False
+
+  # the directory in which model conversion results are stored
+  working_dir: 'model_output_rgb'
+
+  # model conversion generated name prefix of those model files used for dev board execution
+  output_model_file_prefix: 'mobilenetv2_224x224_rgb'
+
+```
+对模型输入格式的配置如下：
+
+```yaml
+
+input_parameters:
+
+  # (Optional) node name of model input,
+  # it shall be the same as the name of model file, otherwise an error will be reported,
+  # the node name of model file will be used when left blank
+  input_name: ""
+
+  # the data formats to be passed into neural network when actually performing neural network
+  # available options: nv12/rgb/bgr/yuv444/gray/featuremap,
+  input_type_rt: 'rgb'
+
+  # the data layout formats to be passed into neural network when actually performing neural network, available options: NHWC/NCHW
+  # If input_type_rt is configured as nv12, then this parameter does not need to be configured
+  input_layout_rt: 'NHWC'
+
+  # the data formats in network training
+  # available options: rgb/bgr/gray/featuremap/yuv444
+  input_type_train: 'rgb'
+
+  # the data layout in network training, available options: NHWC/NCHW
+  input_layout_train: 'NCHW'
+
+  # (Optional)the input size of model network, seperated by 'x'
+  # note that the network input size of model file will be used if left blank
+  # otherwise it will overwrite the input size of model file
+  input_shape: ''
+
+  # the data batch_size to be passed into neural network when actually performing neural network, default value: 1
+  #input_batch: 1
+
+  # preprocessing methods of network input, available options:
+  # 'no_preprocess' indicates that no preprocess will be made 
+  # 'data_mean' indicates that to minus the channel mean, i.e. mean_value
+  # 'data_scale' indicates that image pixels to multiply data_scale ratio
+  # 'data_mean_and_scale' indicates that to multiply scale ratio after channel mean is minused
+  norm_type: 'data_mean_and_scale'
+
+  # the mean value minused by image
+  # note that values must be seperated by space if channel mean value is used
+  mean_value: 123.675 116.28 103.53
+
+  # scale value of image preprocess
+  # note that values must be seperated by space if channel scale value is used
+  scale_value: 0.01712 0.0175 0.01743
+
+```
+对模型量化参数的配置如下：
+
+```yaml
+calibration_parameters:
+
+  # the directory where reference images of model quantization are stored
+  # image formats include JPEG, BMP etc.
+  # should be classic application scenarios, usually 20~100 images are picked out from test datasets
+  # in addition, note that input images should cover typical scenarios
+  # and try to avoid those overexposed, oversaturated, vague, 
+  # pure blank or pure white images
+  # use ';' to seperate when there are multiple input nodes
+  cal_data_dir: './calibration_data_rgb'
+
+  # calibration data binary file save type, available options: float32, uint8
+#   cal_data_type: 'float32'
+
+  # In case the size of input image file is different from that of in model training
+  # and that preprocess_on is set to True,
+  # shall the default preprocess method(skimage resize) be used
+  # i.e., to resize or crop input image into specified size
+  # otherwise user must keep image size as that of in training in advance
+  # preprocess_on: False
+
+  # The algorithm type of model quantization, support default, mix, kl, max, load, usually use default can meet the requirements.
+  # If it does not meet the expectation, you can try to change it to mix first. If there is still no expectation, try kl or max again.
+  # When using QAT to export the model, this parameter should be set to load.
+  # For more details of the parameters, please refer to the parameter details in PTQ Principle And Steps section of the user manual.
+  calibration_type: 'max'
+
+  # this is the parameter of the 'max' calibration method and it is used for adjusting the intercept point of the 'max' calibration.
+  # this parameter will only become valid when the calibration_type is specified as 'max'.
+  # RANGE: 0.0 - 1.0. Typical options includes: 0.99999/0.99995/0.99990/0.99950/0.99900.
+  max_percentile: 0.9999
+```
+其余参数选择默认值，运行`03_build.sh`,
+```bash
+config_file="./mobilenetv2_config.yaml"
+model_type="onnx"
+# build model
+hb_mapper makertbin --config ${config_file}  \
+                    --model-type  ${model_type}
+```
+至此，在同路径下`model_output_rgb`会生成转换完成的模型文件（后缀为.bin）