Support risc-v platform (#910)

* add ppl.nn riscv engine

* update ppl.nn riscv engine

* udpate riscv service (ncnn backend)

* update _build_wrapper for ncnn

* fix build

* fix lint

* update default uri

* update file structure & add cn doc

* remove copy input data

* update docs

* remove ncnn server

* fix docs

* update zh doc

* update toolchain

* remove unused

* update doc

* update doc

* update doc

* rename cross build dirname

* add riscv.md to build_from_source.md

* update cls model

* test ci

* test ci

* test ci

* test ci

* test ci

* update ci

* update ci

.github/workflows/linux-riscv64-gcc.yml

@@ -0,0 +1,56 @@
+name: build_riscv64_gcc
+
+on:
+ push:
+ paths:
+ - "csrc/**"
+ - "demo/csrc/**"
+ - "CMakeLists.txt"
+
+ pull_request:
+ paths-ignore:
+ - "csrc/**"
+ - "demo/csrc/**"
+ - "CMakeLists.txt"
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build_riscv64_gcc:
+ runs-on: ubuntu-20.04
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+ with:
+ submodules: 'recursive'
+ - name: riscv64-gnu-toolchain
+ run: |
+ sudo apt-get update
+ sudo apt-get install g++-riscv64-linux-gnu
+ - name: install opencv
+ run: |
+ mkdir $GITHUB_WORKSPACE/opencv-install
+ wget https://github.com/irexyc/mmdeploy-ci-resource/raw/opencv/opencv_4.6.0_linux_riscv64.tar.gz
+ tar xf opencv_4.6.0_linux_riscv64.tar.gz -C $GITHUB_WORKSPACE/opencv-install
+ - name: install ncnn
+ run: |
+ mkdir $GITHUB_WORKSPACE/ncnn-install
+ wget https://github.com/irexyc/mmdeploy-ci-resource/raw/ncnn/ncnn_20220729_linux_riscv64.tar.gz
+ tar xf ncnn_20220729_linux_riscv64.tar.gz -C $GITHUB_WORKSPACE/ncnn-install
+ - name: build
+ run: |
+ mkdir build && cd build
+ cmake .. \
+ -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/riscv64-linux-gnu.cmake \
+ -DMMDEPLOY_BUILD_SDK=ON \
+ -DMMDEPLOY_SHARED_LIBS=ON \
+ -DMMDEPLOY_BUILD_EXAMPLES=ON \
+ -DMMDEPLOY_TARGET_DEVICES="cpu" \
+ -DMMDEPLOY_TARGET_BACKENDS="ncnn" \
+ -Dncnn_DIR=$GITHUB_WORKSPACE/ncnn-install/lib/cmake/ncnn/ \
+ -DMMDEPLOY_CODEBASES=all \
+ -DOpenCV_DIR=$GITHUB_WORKSPACE/opencv-install/lib/cmake/opencv4
+ make -j$(nproc)
+ make install

cmake/toolchains/riscv64-linux-gnu.cmake

@@ -0,0 +1,17 @@
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_PROCESSOR riscv)
+
+set(CMAKE_C_COMPILER "riscv64-linux-gnu-gcc")
+set(CMAKE_CXX_COMPILER "riscv64-linux-gnu-g++")
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
+
+set(CMAKE_C_FLAGS "-march=rv64gc")
+set(CMAKE_CXX_FLAGS "-march=rv64gc")
+
+# cache flags
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "c flags")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "c++ flags")

cmake/toolchains/riscv64-unknown-linux-gnu.cmake

@@ -0,0 +1,26 @@
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_PROCESSOR riscv)
+
+if(DEFINED ENV{RISCV_ROOT_PATH})
+ file(TO_CMAKE_PATH $ENV{RISCV_ROOT_PATH} RISCV_ROOT_PATH)
+else()
+ message(FATAL_ERROR "RISCV_ROOT_PATH env must be defined")
+endif()
+
+set(CMAKE_C_COMPILER ${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-gcc)
+set(CMAKE_CXX_COMPILER ${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-g++)
+
+set(CMAKE_SYSROOT "${RISCV_ROOT_PATH}/sysroot" CACHE PATH "riscv sysroot")
+set(CMAKE_FIND_ROOT_PATH ${RISCV_ROOT_PATH}/riscv64-unknown-linux-gnu)
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
+
+set(CMAKE_C_FLAGS "-march=rv64gc")
+set(CMAKE_CXX_FLAGS "-march=rv64gc")
+
+# cache flags
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "c flags")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "c++ flags")

csrc/mmdeploy/backend_ops/ncnn/CMakeLists.txt

@@ -10,7 +10,7 @@ else ()
 endif ()
 
 
-if (NOT ANDROID AND NOT IOS)
+if (NOT ANDROID AND NOT IOS AND NOT CMAKE_CROSSCOMPILING)
  add_subdirectory(ops)
  add_subdirectory(onnx2ncnn)
  add_subdirectory(pyncnn_ext)

csrc/mmdeploy/net/ppl/CMakeLists.txt

@@ -18,4 +18,7 @@ if (PPLNN_USE_CUDA AND ("cuda" IN_LIST MMDEPLOY_TARGET_DEVICES))
 endif ()
 target_link_libraries(${PROJECT_NAME}
  PRIVATE ${PPLNN_LIBRARIES})
+if (PPLNN_USE_RISCV AND ("cpu" IN_LIST MMDEPLOY_TARGET_DEVICES))
+ target_compile_definitions(${PROJECT_NAME} PRIVATE -DPPL_NN_HAS_RISCV=1)
+endif ()
 add_library(mmdeploy::pplnn_net ALIAS ${PROJECT_NAME})

csrc/mmdeploy/net/ppl/ppl_net.cpp

@@ -18,6 +18,11 @@
 #include "ppl/nn/engines/cuda/ops.h"
 #define PPL_CUDA_IMPORT_FROM_BUFFER 1
 #endif
+#if PPL_NN_HAS_RISCV
+#include "ppl/nn/engines/riscv/engine_factory.h"
+#include "ppl/nn/engines/riscv/engine_options.h"
+#include "ppl/nn/engines/riscv/ops.h"
+#endif
 
 namespace mmdeploy {
 
@@ -92,6 +97,18 @@ Result<void> PPLNet::Init(const Value& args) {
  engines_.emplace_back(ppl::nn::x86::EngineFactory::Create({}));
  }
 #endif
+#if PPL_NN_HAS_RISCV
+ if (device_.is_host()) {
+ ppl::nn::riscv::RegisterBuiltinOpImpls();
+ ppl::nn::riscv::EngineOptions options{};
+ // TODO:
+ // FP16 -> postprocess
+ options.forward_precision = ppl::common::DATATYPE_FLOAT32;
+ options.dynamic_tuning_level = 0;
+ options.winograd_level = 1;
+ engines_.emplace_back(ppl::nn::riscv::EngineFactory::Create(options));
+ }
+#endif
 
  std::vector<ppl::nn::Engine*> engines;
  for (const auto& engine : engines_) {

docs/en/01-how-to-build/build_from_source.md

@@ -38,3 +38,4 @@ Please visit the following links to find out how to build MMDeploy according to
 - [Android-aarch64](android.md)
 - [NVIDIA Jetson](jetsons.md)
 - [SNPE](snpe.md)
+- [RISC-V](riscv.md)

docs/en/01-how-to-build/riscv.md

@@ -0,0 +1,104 @@
+# Build for RISC-V
+
+MMDeploy chooses ncnn as the inference backend on RISC-V platform. The deployment process consists of two steps:
+
+Model conversion: Convert the PyTorch model to the ncnn model on the host side, and then upload the converted model to the device.
+
+Model deployment: Compile ncnn and MMDeploy in cross-compilation mode on the host side, and then upload the executable for inference.
+
+## 1. Model conversion
+
+a) Install MMDeploy
+
+You can refer to [Build document](./linux-x86_64.md) to install ncnn inference engine and MMDeploy
+
+b) Convert model
+
+```bash
+export MODEL_CONFIG=/path/to/mmclassification/configs/resnet/resnet18_8xb32_in1k.py
+export MODEL_PATH=https://download.openmmlab.com/mmclassification/v0/resnet/resnet18_8xb32_in1k_20210831-fbbb1da6.pth
+
+# Convert the model
+cd /path/to/mmdeploy
+python tools/deploy.py \
+ configs/mmcls/classification_ncnn_static.py \
+ $MODEL_CONFIG \
+ $MODEL_PATH \
+ tests/data/tiger.jpeg \
+ --work-dir resnet18 \
+ --device cpu \
+ --dump-info
+```
+
+## 2. Model deployment
+
+a) Download the compiler toolchain and set environment
+
+```bash
+# download Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6-20220516.tar.gz
+# https://occ.t-head.cn/community/download?id=4046947553902661632
+tar xf Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6-20220516.tar.gz
+export RISCV_ROOT_PATH=`realpath Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6`
+```
+
+b) Compile ncnn & opencv
+
+```bash
+# ncnn
+# refer to https://github.com/Tencent/ncnn/wiki/how-to-build#build-for-allwinner-d1
+
+# opencv
+git clone https://github.com/opencv/opencv.git
+mkdir build_riscv && cd build_riscv
+cmake .. \
+ -DCMAKE_TOOLCHAIN_FILE=/path/to/mmdeploy/cmake/toolchains/riscv64-unknown-linux-gnu.cmake \
+ -DCMAKE_INSTALL_PREFIX=install \
+ -DBUILD_PERF_TESTS=OFF \
+ -DBUILD_SHARED_LIBS=OFF \
+ -DBUILD_TESTS=OFF \
+ -DCMAKE_BUILD_TYPE=Release
+make -j$(nproc) && make install
+```
+
+c) Compile mmdeploy SDK & demo
+
+```bash
+cd /path/to/mmdeploy
+mkdir build_riscv && cd build_riscv
+cmake .. \
+ -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/riscv64-unknown-linux-gnu.cmake \
+ -DMMDEPLOY_BUILD_SDK=ON \
+ -DMMDEPLOY_SHARED_LIBS=OFF \
+ -DMMDEPLOY_BUILD_EXAMPLES=ON \
+ -DMMDEPLOY_TARGET_DEVICES="cpu" \
+ -DMMDEPLOY_TARGET_BACKENDS="ncnn" \
+ -Dncnn_DIR=${ncnn_DIR}/build-c906/install/lib/cmake/ncnn/ \
+ -DMMDEPLOY_CODEBASES=all \
+ -DOpenCV_DIR=${OpenCV_DIR}/build_riscv/install/lib/cmake/opencv4
+
+make -j$(nproc) && make install
+```
+
+After `make install`, the examples will locate in `install\bin`
+
+```
+tree -L 1 install/bin/
+.
+├── image_classification
+├── image_restorer
+├── image_segmentation
+├── object_detection
+├── ocr
+├── pose_detection
+└── rotated_object_detection
+```
+
+### 4) Run the demo
+
+First make sure that `--dump-info` is used during convert model, so that the `resnet18` directory has the files required by the SDK such as `pipeline.json`.
+
+Copy the model folder(resnet18), executable(image_classification) file and test image(tests/data/tiger.jpeg) to the device.
+
+```bash
+./image_classification cpu ./resnet18 tiger.jpeg
+```

docs/zh_cn/01-how-to-build/build_from_source.md

@@ -41,3 +41,4 @@ git clone -b master git@github.com:open-mmlab/mmdeploy.git --recursive
 - [Android-aarch64](android.md)
 - [NVIDIA Jetson](jetsons.md)
 - [Qcom SNPE](snpe.md)
+- [RISC-V](riscv.md)

docs/zh_cn/01-how-to-build/riscv.md

@@ -0,0 +1,107 @@
+# 支持 RISC-V
+
+MMDeploy 选择 ncnn 作为 RISC-V 平台下的推理后端。完整的部署过程包含两个步骤：
+
+模型转换：在 Host 端将 PyTorch 模型转为 ncnn 模型。并将转换后的模型传到 device。
+
+模型部署：在 Host 端以交叉编译方式编译 ncnn 和 MMDeploy。传到 Device 端进行推理。
+
+## 1. 模型转换
+
+a) 安装 MMDeploy
+
+可参考 [BUILD 文档](./linux-x86_64.md)，安装 ncnn 推理引擎以及 MMDeploy。
+
+b) 模型转换
+
+以 Resnet-18 为例。先参照[文档](https://github.com/open-mmlab/mmclassification)安装 mmcls，然后使用 `tools/deploy.py` 转换模型。
+
+```bash
+export MODEL_CONFIG=/path/to/mmclassification/configs/resnet/resnet18_8xb32_in1k.py
+export MODEL_PATH=https://download.openmmlab.com/mmclassification/v0/resnet/resnet18_8xb32_in1k_20210831-fbbb1da6.pth
+
+# 模型转换
+cd /path/to/mmdeploy
+python tools/deploy.py \
+ configs/mmcls/classification_ncnn_static.py \
+ $MODEL_CONFIG \
+ $MODEL_PATH \
+ tests/data/tiger.jpeg \
+ --work-dir resnet18 \
+ --device cpu \
+ --dump-info
+```
+
+## 2. 模型部署
+
+a) 下载交叉编译工具链，设置环境变量
+
+```bash
+# 下载 Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6-20220516.tar.gz
+# https://occ.t-head.cn/community/download?id=4046947553902661632
+tar xf Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6-20220516.tar.gz
+export RISCV_ROOT_PATH=`realpath Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6`
+```
+
+b) 编译 ncnn & opencv
+
+```bash
+# ncnn
+# 可参考 https://github.com/Tencent/ncnn/wiki/how-to-build#build-for-allwinner-d1
+
+# opencv
+git clone https://github.com/opencv/opencv.git
+mkdir build_riscv && cd build_riscv
+cmake .. \
+ -DCMAKE_TOOLCHAIN_FILE=/path/to/mmdeploy/cmake/toolchains/riscv64-unknown-linux-gnu.cmake \
+ -DCMAKE_INSTALL_PREFIX=install \
+ -DBUILD_PERF_TESTS=OFF \
+ -DBUILD_SHARED_LIBS=OFF \
+ -DBUILD_TESTS=OFF \
+ -DCMAKE_BUILD_TYPE=Release
+
+make -j$(nproc) && make install
+```
+
+c) 编译 mmdeploy SDK & demo
+
+```bash
+cd /path/to/mmdeploy
+mkdir build_riscv && cd build_riscv
+cmake .. \
+ -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/riscv64-unknown-linux-gnu.cmake \
+ -DMMDEPLOY_BUILD_SDK=ON \
+ -DMMDEPLOY_SHARED_LIBS=OFF \
+ -DMMDEPLOY_BUILD_EXAMPLES=ON \
+ -DMMDEPLOY_TARGET_DEVICES="cpu" \
+ -DMMDEPLOY_TARGET_BACKENDS="ncnn" \
+ -Dncnn_DIR=${ncnn_DIR}/build-c906/install/lib/cmake/ncnn/ \
+ -DMMDEPLOY_CODEBASES=all \
+ -DOpenCV_DIR=${OpenCV_DIR}/build_riscv/install/lib/cmake/opencv4
+
+make -j$(nproc) && make install
+```
+
+执行 `make install` 之后, examples的可执行文件会保存在 `install/bin`
+
+```
+tree -L 1 install/bin/
+.
+├── image_classification
+├── image_restorer
+├── image_segmentation
+├── object_detection
+├── ocr
+├── pose_detection
+└── rotated_object_detection
+```
+
+d) 运行 demo
+
+先确认测试模型用了 `--dump-info`，这样 `resnet18` 目录才有 `pipeline.json` 等 SDK 所需文件。
+
+把 dump 好的模型目录(resnet18)、可执行文件(image_classification)、测试图片(tests/data/tiger.jpeg)拷贝到设备中
+
+```bash
+./image_classification cpu ./resnet18 tiger.jpeg
+```

tools/test.py

@@ -81,8 +81,6 @@ def parse_args():
  'in data config.')
  parser.add_argument(
  '--uri',
- action='store_true',
- default='192.168.1.1:60000',
  help='Remote ipv4:port or ipv6:port for inference on edge device.')
 
  args = parser.parse_args()