diff --git a/.github/workflows/linux-riscv64-gcc.yml b/.github/workflows/linux-riscv64-gcc.yml new file mode 100644 index 0000000000..462a85296a --- /dev/null +++ b/.github/workflows/linux-riscv64-gcc.yml @@ -0,0 +1,56 @@ +name: build_riscv64_gcc + +on: + push: + paths: + - "csrc/**" + - "demo/csrc/**" + - "CMakeLists.txt" + + pull_request: + paths-ignore: + - "csrc/**" + - "demo/csrc/**" + - "CMakeLists.txt" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_riscv64_gcc: + runs-on: ubuntu-20.04 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: riscv64-gnu-toolchain + run: | + sudo apt-get update + sudo apt-get install g++-riscv64-linux-gnu + - name: install opencv + run: | + mkdir $GITHUB_WORKSPACE/opencv-install + wget https://github.com/irexyc/mmdeploy-ci-resource/raw/opencv/opencv_4.6.0_linux_riscv64.tar.gz + tar xf opencv_4.6.0_linux_riscv64.tar.gz -C $GITHUB_WORKSPACE/opencv-install + - name: install ncnn + run: | + mkdir $GITHUB_WORKSPACE/ncnn-install + wget https://github.com/irexyc/mmdeploy-ci-resource/raw/ncnn/ncnn_20220729_linux_riscv64.tar.gz + tar xf ncnn_20220729_linux_riscv64.tar.gz -C $GITHUB_WORKSPACE/ncnn-install + - name: build + run: | + mkdir build && cd build + cmake .. \ + -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/riscv64-linux-gnu.cmake \ + -DMMDEPLOY_BUILD_SDK=ON \ + -DMMDEPLOY_SHARED_LIBS=ON \ + -DMMDEPLOY_BUILD_EXAMPLES=ON \ + -DMMDEPLOY_TARGET_DEVICES="cpu" \ + -DMMDEPLOY_TARGET_BACKENDS="ncnn" \ + -Dncnn_DIR=$GITHUB_WORKSPACE/ncnn-install/lib/cmake/ncnn/ \ + -DMMDEPLOY_CODEBASES=all \ + -DOpenCV_DIR=$GITHUB_WORKSPACE/opencv-install/lib/cmake/opencv4 + make -j$(nproc) + make install diff --git a/cmake/toolchains/riscv64-linux-gnu.cmake b/cmake/toolchains/riscv64-linux-gnu.cmake new file mode 100644 index 0000000000..e3b3b2adbc --- /dev/null +++ b/cmake/toolchains/riscv64-linux-gnu.cmake @@ -0,0 +1,17 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR riscv) + +set(CMAKE_C_COMPILER "riscv64-linux-gnu-gcc") +set(CMAKE_CXX_COMPILER "riscv64-linux-gnu-g++") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +set(CMAKE_C_FLAGS "-march=rv64gc") +set(CMAKE_CXX_FLAGS "-march=rv64gc") + +# cache flags +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "c flags") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "c++ flags") diff --git a/cmake/toolchains/riscv64-unknown-linux-gnu.cmake b/cmake/toolchains/riscv64-unknown-linux-gnu.cmake new file mode 100644 index 0000000000..c24661f6e6 --- /dev/null +++ b/cmake/toolchains/riscv64-unknown-linux-gnu.cmake @@ -0,0 +1,26 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR riscv) + +if(DEFINED ENV{RISCV_ROOT_PATH}) + file(TO_CMAKE_PATH $ENV{RISCV_ROOT_PATH} RISCV_ROOT_PATH) +else() + message(FATAL_ERROR "RISCV_ROOT_PATH env must be defined") +endif() + +set(CMAKE_C_COMPILER ${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-gcc) +set(CMAKE_CXX_COMPILER ${RISCV_ROOT_PATH}/bin/riscv64-unknown-linux-gnu-g++) + +set(CMAKE_SYSROOT "${RISCV_ROOT_PATH}/sysroot" CACHE PATH "riscv sysroot") +set(CMAKE_FIND_ROOT_PATH ${RISCV_ROOT_PATH}/riscv64-unknown-linux-gnu) + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +set(CMAKE_C_FLAGS "-march=rv64gc") +set(CMAKE_CXX_FLAGS "-march=rv64gc") + +# cache flags +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "c flags") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "c++ flags") diff --git a/csrc/mmdeploy/backend_ops/ncnn/CMakeLists.txt b/csrc/mmdeploy/backend_ops/ncnn/CMakeLists.txt index 42901f77f9..4df9ad1233 100755 --- a/csrc/mmdeploy/backend_ops/ncnn/CMakeLists.txt +++ b/csrc/mmdeploy/backend_ops/ncnn/CMakeLists.txt @@ -10,7 +10,7 @@ else () endif () -if (NOT ANDROID AND NOT IOS) +if (NOT ANDROID AND NOT IOS AND NOT CMAKE_CROSSCOMPILING) add_subdirectory(ops) add_subdirectory(onnx2ncnn) add_subdirectory(pyncnn_ext) diff --git a/csrc/mmdeploy/net/ppl/CMakeLists.txt b/csrc/mmdeploy/net/ppl/CMakeLists.txt index a65c64d4de..c92b80ba72 100644 --- a/csrc/mmdeploy/net/ppl/CMakeLists.txt +++ b/csrc/mmdeploy/net/ppl/CMakeLists.txt @@ -18,4 +18,7 @@ if (PPLNN_USE_CUDA AND ("cuda" IN_LIST MMDEPLOY_TARGET_DEVICES)) endif () target_link_libraries(${PROJECT_NAME} PRIVATE ${PPLNN_LIBRARIES}) +if (PPLNN_USE_RISCV AND ("cpu" IN_LIST MMDEPLOY_TARGET_DEVICES)) + target_compile_definitions(${PROJECT_NAME} PRIVATE -DPPL_NN_HAS_RISCV=1) +endif () add_library(mmdeploy::pplnn_net ALIAS ${PROJECT_NAME}) diff --git a/csrc/mmdeploy/net/ppl/ppl_net.cpp b/csrc/mmdeploy/net/ppl/ppl_net.cpp index 3f404983d8..43d0f22c2d 100644 --- a/csrc/mmdeploy/net/ppl/ppl_net.cpp +++ b/csrc/mmdeploy/net/ppl/ppl_net.cpp @@ -18,6 +18,11 @@ #include "ppl/nn/engines/cuda/ops.h" #define PPL_CUDA_IMPORT_FROM_BUFFER 1 #endif +#if PPL_NN_HAS_RISCV +#include "ppl/nn/engines/riscv/engine_factory.h" +#include "ppl/nn/engines/riscv/engine_options.h" +#include "ppl/nn/engines/riscv/ops.h" +#endif namespace mmdeploy { @@ -92,6 +97,18 @@ Result PPLNet::Init(const Value& args) { engines_.emplace_back(ppl::nn::x86::EngineFactory::Create({})); } #endif +#if PPL_NN_HAS_RISCV + if (device_.is_host()) { + ppl::nn::riscv::RegisterBuiltinOpImpls(); + ppl::nn::riscv::EngineOptions options{}; + // TODO: + // FP16 -> postprocess + options.forward_precision = ppl::common::DATATYPE_FLOAT32; + options.dynamic_tuning_level = 0; + options.winograd_level = 1; + engines_.emplace_back(ppl::nn::riscv::EngineFactory::Create(options)); + } +#endif std::vector engines; for (const auto& engine : engines_) { diff --git a/docs/en/01-how-to-build/build_from_source.md b/docs/en/01-how-to-build/build_from_source.md index 2aa6ecc851..d05d9e27fa 100644 --- a/docs/en/01-how-to-build/build_from_source.md +++ b/docs/en/01-how-to-build/build_from_source.md @@ -38,3 +38,4 @@ Please visit the following links to find out how to build MMDeploy according to - [Android-aarch64](android.md) - [NVIDIA Jetson](jetsons.md) - [SNPE](snpe.md) +- [RISC-V](riscv.md) diff --git a/docs/en/01-how-to-build/riscv.md b/docs/en/01-how-to-build/riscv.md new file mode 100644 index 0000000000..28e78e57a5 --- /dev/null +++ b/docs/en/01-how-to-build/riscv.md @@ -0,0 +1,104 @@ +# Build for RISC-V + +MMDeploy chooses ncnn as the inference backend on RISC-V platform. The deployment process consists of two steps: + +Model conversion: Convert the PyTorch model to the ncnn model on the host side, and then upload the converted model to the device. + +Model deployment: Compile ncnn and MMDeploy in cross-compilation mode on the host side, and then upload the executable for inference. + +## 1. Model conversion + +a) Install MMDeploy + +You can refer to [Build document](./linux-x86_64.md) to install ncnn inference engine and MMDeploy + +b) Convert model + +```bash +export MODEL_CONFIG=/path/to/mmclassification/configs/resnet/resnet18_8xb32_in1k.py +export MODEL_PATH=https://download.openmmlab.com/mmclassification/v0/resnet/resnet18_8xb32_in1k_20210831-fbbb1da6.pth + +# Convert the model +cd /path/to/mmdeploy +python tools/deploy.py \ + configs/mmcls/classification_ncnn_static.py \ + $MODEL_CONFIG \ + $MODEL_PATH \ + tests/data/tiger.jpeg \ + --work-dir resnet18 \ + --device cpu \ + --dump-info +``` + +## 2. Model deployment + +a) Download the compiler toolchain and set environment + +```bash +# download Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6-20220516.tar.gz +# https://occ.t-head.cn/community/download?id=4046947553902661632 +tar xf Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6-20220516.tar.gz +export RISCV_ROOT_PATH=`realpath Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6` +``` + +b) Compile ncnn & opencv + +```bash +# ncnn +# refer to https://github.com/Tencent/ncnn/wiki/how-to-build#build-for-allwinner-d1 + +# opencv +git clone https://github.com/opencv/opencv.git +mkdir build_riscv && cd build_riscv +cmake .. \ + -DCMAKE_TOOLCHAIN_FILE=/path/to/mmdeploy/cmake/toolchains/riscv64-unknown-linux-gnu.cmake \ + -DCMAKE_INSTALL_PREFIX=install \ + -DBUILD_PERF_TESTS=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release +make -j$(nproc) && make install +``` + +c) Compile mmdeploy SDK & demo + +```bash +cd /path/to/mmdeploy +mkdir build_riscv && cd build_riscv +cmake .. \ + -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/riscv64-unknown-linux-gnu.cmake \ + -DMMDEPLOY_BUILD_SDK=ON \ + -DMMDEPLOY_SHARED_LIBS=OFF \ + -DMMDEPLOY_BUILD_EXAMPLES=ON \ + -DMMDEPLOY_TARGET_DEVICES="cpu" \ + -DMMDEPLOY_TARGET_BACKENDS="ncnn" \ + -Dncnn_DIR=${ncnn_DIR}/build-c906/install/lib/cmake/ncnn/ \ + -DMMDEPLOY_CODEBASES=all \ + -DOpenCV_DIR=${OpenCV_DIR}/build_riscv/install/lib/cmake/opencv4 + +make -j$(nproc) && make install +``` + +After `make install`, the examples will locate in `install\bin` + +``` +tree -L 1 install/bin/ +. +├── image_classification +├── image_restorer +├── image_segmentation +├── object_detection +├── ocr +├── pose_detection +└── rotated_object_detection +``` + +### 4) Run the demo + +First make sure that `--dump-info` is used during convert model, so that the `resnet18` directory has the files required by the SDK such as `pipeline.json`. + +Copy the model folder(resnet18), executable(image_classification) file and test image(tests/data/tiger.jpeg) to the device. + +```bash +./image_classification cpu ./resnet18 tiger.jpeg +``` diff --git a/docs/zh_cn/01-how-to-build/build_from_source.md b/docs/zh_cn/01-how-to-build/build_from_source.md index d43d80a316..66f87c0158 100644 --- a/docs/zh_cn/01-how-to-build/build_from_source.md +++ b/docs/zh_cn/01-how-to-build/build_from_source.md @@ -41,3 +41,4 @@ git clone -b master git@github.com:open-mmlab/mmdeploy.git --recursive - [Android-aarch64](android.md) - [NVIDIA Jetson](jetsons.md) - [Qcom SNPE](snpe.md) +- [RISC-V](riscv.md) diff --git a/docs/zh_cn/01-how-to-build/riscv.md b/docs/zh_cn/01-how-to-build/riscv.md new file mode 100644 index 0000000000..eb1baedf30 --- /dev/null +++ b/docs/zh_cn/01-how-to-build/riscv.md @@ -0,0 +1,107 @@ +# 支持 RISC-V + +MMDeploy 选择 ncnn 作为 RISC-V 平台下的推理后端。完整的部署过程包含两个步骤: + +模型转换:在 Host 端将 PyTorch 模型转为 ncnn 模型。并将转换后的模型传到 device。 + +模型部署:在 Host 端以交叉编译方式编译 ncnn 和 MMDeploy。传到 Device 端进行推理。 + +## 1. 模型转换 + +a) 安装 MMDeploy + +可参考 [BUILD 文档](./linux-x86_64.md),安装 ncnn 推理引擎以及 MMDeploy。 + +b) 模型转换 + +以 Resnet-18 为例。先参照[文档](https://github.com/open-mmlab/mmclassification)安装 mmcls,然后使用 `tools/deploy.py` 转换模型。 + +```bash +export MODEL_CONFIG=/path/to/mmclassification/configs/resnet/resnet18_8xb32_in1k.py +export MODEL_PATH=https://download.openmmlab.com/mmclassification/v0/resnet/resnet18_8xb32_in1k_20210831-fbbb1da6.pth + +# 模型转换 +cd /path/to/mmdeploy +python tools/deploy.py \ + configs/mmcls/classification_ncnn_static.py \ + $MODEL_CONFIG \ + $MODEL_PATH \ + tests/data/tiger.jpeg \ + --work-dir resnet18 \ + --device cpu \ + --dump-info +``` + +## 2. 模型部署 + +a) 下载交叉编译工具链,设置环境变量 + +```bash +# 下载 Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6-20220516.tar.gz +# https://occ.t-head.cn/community/download?id=4046947553902661632 +tar xf Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6-20220516.tar.gz +export RISCV_ROOT_PATH=`realpath Xuantie-900-gcc-linux-5.10.4-glibc-x86_64-V2.2.6` +``` + +b) 编译 ncnn & opencv + +```bash +# ncnn +# 可参考 https://github.com/Tencent/ncnn/wiki/how-to-build#build-for-allwinner-d1 + +# opencv +git clone https://github.com/opencv/opencv.git +mkdir build_riscv && cd build_riscv +cmake .. \ + -DCMAKE_TOOLCHAIN_FILE=/path/to/mmdeploy/cmake/toolchains/riscv64-unknown-linux-gnu.cmake \ + -DCMAKE_INSTALL_PREFIX=install \ + -DBUILD_PERF_TESTS=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=Release + +make -j$(nproc) && make install +``` + +c) 编译 mmdeploy SDK & demo + +```bash +cd /path/to/mmdeploy +mkdir build_riscv && cd build_riscv +cmake .. \ + -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/riscv64-unknown-linux-gnu.cmake \ + -DMMDEPLOY_BUILD_SDK=ON \ + -DMMDEPLOY_SHARED_LIBS=OFF \ + -DMMDEPLOY_BUILD_EXAMPLES=ON \ + -DMMDEPLOY_TARGET_DEVICES="cpu" \ + -DMMDEPLOY_TARGET_BACKENDS="ncnn" \ + -Dncnn_DIR=${ncnn_DIR}/build-c906/install/lib/cmake/ncnn/ \ + -DMMDEPLOY_CODEBASES=all \ + -DOpenCV_DIR=${OpenCV_DIR}/build_riscv/install/lib/cmake/opencv4 + +make -j$(nproc) && make install +``` + +执行 `make install` 之后, examples的可执行文件会保存在 `install/bin` + +``` +tree -L 1 install/bin/ +. +├── image_classification +├── image_restorer +├── image_segmentation +├── object_detection +├── ocr +├── pose_detection +└── rotated_object_detection +``` + +d) 运行 demo + +先确认测试模型用了 `--dump-info`,这样 `resnet18` 目录才有 `pipeline.json` 等 SDK 所需文件。 + +把 dump 好的模型目录(resnet18)、可执行文件(image_classification)、测试图片(tests/data/tiger.jpeg)拷贝到设备中 + +```bash +./image_classification cpu ./resnet18 tiger.jpeg +``` diff --git a/tools/test.py b/tools/test.py index d3957a19d4..07783eaaeb 100644 --- a/tools/test.py +++ b/tools/test.py @@ -81,8 +81,6 @@ def parse_args(): 'in data config.') parser.add_argument( '--uri', - action='store_true', - default='192.168.1.1:60000', help='Remote ipv4:port or ipv6:port for inference on edge device.') args = parser.parse_args()