diff --git a/apps/bundle_deploy/Makefile b/apps/bundle_deploy/Makefile index bd4053f2911c..c80765f28e09 100644 --- a/apps/bundle_deploy/Makefile +++ b/apps/bundle_deploy/Makefile @@ -33,22 +33,36 @@ PKG_LDFLAGS = -pthread build_dir := build -demo: $(build_dir)/demo $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/cat.bin - TVM_NUM_THREADS=1 $(build_dir)/demo $(build_dir)/bundle.so $(build_dir)/cat.bin - TVM_NUM_THREADS=1 $(build_dir)/demo $(build_dir)/bundle_c.so $(build_dir)/cat.bin +demo_dynamic: $(build_dir)/demo_dynamic $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/cat.bin + TVM_NUM_THREADS=1 $(build_dir)/demo_dynamic $(build_dir)/bundle.so $(build_dir)/cat.bin + TVM_NUM_THREADS=1 $(build_dir)/demo_dynamic $(build_dir)/bundle_c.so $(build_dir)/cat.bin -test: $(build_dir)/test $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin - TVM_NUM_THREADS=1 $(build_dir)/test $(build_dir)/test_bundle.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin - TVM_NUM_THREADS=1 $(build_dir)/test $(build_dir)/test_bundle_c.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin +test_dynamic: $(build_dir)/test_dynamic $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin + TVM_NUM_THREADS=1 $(build_dir)/test_dynamic $(build_dir)/test_bundle.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin + TVM_NUM_THREADS=1 $(build_dir)/test_dynamic $(build_dir)/test_bundle_c.so $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin -$(build_dir)/demo: demo.cc ${build_dir}/graph.json.c ${build_dir}/params.bin.c +demo_static: $(build_dir)/demo_static $(build_dir)/cat.bin + TVM_NUM_THREADS=1 $(build_dir)/demo_static $(build_dir)/cat.bin + +test_static: $(build_dir)/test_static $(build_dir)/test_data.bin $(build_dir)/test_output.bin + TVM_NUM_THREADS=1 $(build_dir)/test_static $(build_dir)/test_data.bin $(build_dir)/test_output.bin $(build_dir)/test_graph.json $(build_dir)/test_params.bin + +$(build_dir)/demo_dynamic: demo.cc ${build_dir}/graph.json.c ${build_dir}/params.bin.c @mkdir -p $(@D) g++ $(PKG_CXXFLAGS) -o $@ demo.cc -ldl -$(build_dir)/test: test.cc ${build_dir}/test_graph.json ${build_dir}/test_params.bin +$(build_dir)/test_dynamic: test.cc ${build_dir}/test_graph.json ${build_dir}/test_params.bin @mkdir -p $(@D) g++ $(PKG_CXXFLAGS) -o $@ test.cc -ldl +$(build_dir)/demo_static: demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o ${build_dir}/graph.json.c ${build_dir}/params.bin.c + @mkdir -p $(@D) + gcc $(PKG_CXXFLAGS) -o $@ demo_static.c ${build_dir}/bundle_static.o ${build_dir}/model.o -lm + +$(build_dir)/test_static: test_static.c ${build_dir}/bundle_static.o ${build_dir}/test_model.o + @mkdir -p $(@D) + gcc $(PKG_CXXFLAGS) -o $@ $^ + # Serialize our graph.json file. $(build_dir)/graph.json.c: $(build_dir)/graph.json xxd -i $^ > $@ @@ -89,6 +103,10 @@ $(build_dir)/test_bundle_c.so: bundle.c runtime.c $(build_dir)/test_model.o @mkdir -p $(@D) gcc -shared $(PKG_CFLAGS) -fvisibility=hidden -o $@ $^ $(PKG_LDFLAGS) +$(build_dir)/bundle_static.o: bundle_static.c + @mkdir -p $(@D) + gcc -c $(PKG_CFLAGS) -o $@ $^ + clean: rm -rf $(build_dir)/bundle.so $(build_dir)/bundle_c.so $(build_dir)/test_bundle.so $(build_dir)/test_bundle_c.so diff --git a/apps/bundle_deploy/README.md b/apps/bundle_deploy/README.md index 676ae7d9e6c9..a52d3a78f9c9 100644 --- a/apps/bundle_deploy/README.md +++ b/apps/bundle_deploy/README.md @@ -39,7 +39,7 @@ Type the following command to run the sample code under the current folder, after building TVM first. ```bash -make demo +make demo_dynamic ``` This will: @@ -48,7 +48,20 @@ This will: - Compile the model with Relay - Build a `bundle.so` shared object containing the model specification and parameters -- Build a `demo` executable that `dlopen`'s `bundle.so` (or `bundle_c.so` in +- Build a `demo_dynamic` executable that `dlopen`'s `bundle.so` (or `bundle_c.so` in terms of the MISRA-C runtime), instantiates the contained graph runtime, and invokes the `GraphRuntime::Run` function on a cat image, then prints the output results. + +Type the following command to run the sample code with static linking. + +```bash +make demo_static +``` + +This will: +- Download the mobilenet0.25 model from the MXNet Gluon Model Zoo +- Compile the model with Relay and outputs `model.o` +- Build a `bundle_static.o` object containing the runtime functions +- Build a `demo_static` executable which has static link to `bundle_static.o` and + `model.o`, functions on a cat image, then prints the output results. diff --git a/apps/bundle_deploy/bundle.h b/apps/bundle_deploy/bundle.h new file mode 100644 index 000000000000..aa57faa38666 --- /dev/null +++ b/apps/bundle_deploy/bundle.h @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef TVM_APPS_BUNDLE_DEPLOY_BUNDLE_H_ +#define TVM_APPS_BUNDLE_DEPLOY_BUNDLE_H_ + +#include + +TVM_DLL void * tvm_runtime_create(const char * json_data, + const char * params_data, + const uint64_t params_size); + +TVM_DLL void tvm_runtime_destroy(void * runtime); + +TVM_DLL void tvm_runtime_set_input(void * runtime, + const char * name, + DLTensor * tensor); + +TVM_DLL void tvm_runtime_run(void * runtime); + +TVM_DLL void tvm_runtime_get_output(void * runtime, + int32_t index, + DLTensor * tensor); + +#endif /* TVM_APPS_BUNDLE_DEPLOY_BUNDLE_H_ */ diff --git a/apps/bundle_deploy/bundle_static.c b/apps/bundle_deploy/bundle_static.c new file mode 100644 index 000000000000..c7eb9352652b --- /dev/null +++ b/apps/bundle_deploy/bundle_static.c @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include "bundle.h" +#include "runtime.c" + +TVM_DLL void * tvm_runtime_create(const char * json_data, + const char * params_data, + const uint64_t params_size) { + int64_t device_type = kDLCPU; + int64_t device_id = 0; + + TVMByteArray params; + params.data = params_data; + params.size = params_size; + + TVMContext ctx; + ctx.device_type = (DLDeviceType)device_type; + ctx.device_id = device_id; + + // declare pointers + void * (*SystemLibraryCreate)(); + TVMGraphRuntime * (*TVMGraphRuntimeCreate)(const char *, const TVMModuleHandle, const TVMContext *); + int (*TVMGraphRuntime_LoadParams)(TVMModuleHandle, const char *, const uint32_t); + + // get pointers + TVMFuncGetGlobal("runtime.SystemLib", (TVMFunctionHandle*)&SystemLibraryCreate); + TVMFuncGetGlobal("tvm.graph_runtime.create", (TVMFunctionHandle*)&TVMGraphRuntimeCreate); + + // run modules + TVMModuleHandle mod_syslib = SystemLibraryCreate(); + TVMModuleHandle mod = TVMGraphRuntimeCreate(json_data, mod_syslib, &ctx); + TVMModGetFunction(mod, "load_params", 0, (TVMFunctionHandle*)&TVMGraphRuntime_LoadParams); + TVMGraphRuntime_LoadParams(mod, params.data, params.size); + + return mod; +} + +TVM_DLL void tvm_runtime_destroy(void * runtime) { + void (*TVMGraphRuntimeRelease)(TVMModuleHandle *); + TVMFuncGetGlobal("tvm.graph_runtime.release", (TVMFunctionHandle*)&TVMGraphRuntimeRelease); + TVMGraphRuntimeRelease(&runtime); +} + +TVM_DLL void tvm_runtime_set_input(void * runtime, const char * name, DLTensor * tensor) { + void (*TVMGraphRuntime_SetInput)(TVMModuleHandle, const char *, DLTensor*); + TVMFuncGetGlobal("tvm.graph_runtime.set_input", (TVMFunctionHandle*)&TVMGraphRuntime_SetInput); + TVMGraphRuntime_SetInput(runtime, name, tensor); +} + +TVM_DLL void tvm_runtime_run(void * runtime) { + void (*TVMGraphRuntime_Run)(TVMModuleHandle runtime); + TVMFuncGetGlobal("tvm.graph_runtime.run", (TVMFunctionHandle*)&TVMGraphRuntime_Run); + TVMGraphRuntime_Run(runtime); +} + +TVM_DLL void tvm_runtime_get_output(void * runtime, int32_t index, DLTensor * tensor) { + int (*TVMGraphRuntime_GetOutput)(TVMModuleHandle, const int32_t, DLTensor *); + TVMFuncGetGlobal("tvm.graph_runtime.get_output", (TVMFunctionHandle*)&TVMGraphRuntime_GetOutput); + TVMGraphRuntime_GetOutput(runtime, index, tensor); +} \ No newline at end of file diff --git a/apps/bundle_deploy/demo_static.c b/apps/bundle_deploy/demo_static.c new file mode 100644 index 000000000000..7a7554523284 --- /dev/null +++ b/apps/bundle_deploy/demo_static.c @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include + +#include "bundle.h" +#include "build/graph.json.c" +#include "build/params.bin.c" + +#define OUTPUT_LEN 1000 + +int main(int argc, char **argv) { + assert(argc == 2 && "Usage: demo_static "); + + char * json_data = (char *)(build_graph_json); + char * params_data = (char *)(build_params_bin); + uint64_t params_size = build_params_bin_len; + + struct timeval t0, t1, t2, t3, t4, t5; + gettimeofday(&t0, 0); + + auto *handle = tvm_runtime_create(json_data, params_data, params_size); + gettimeofday(&t1, 0); + + float input_storage[1 * 3 * 224 * 224]; + FILE * fp = fopen(argv[1], "rb"); + fread(input_storage, 3 * 224 * 224, 4, fp); + fclose(fp); + + DLTensor input; + input.data = input_storage; + DLContext ctx = {kDLCPU, 0}; + input.ctx = ctx; + input.ndim = 4; + DLDataType dtype = {kDLFloat, 32, 1}; + input.dtype = dtype; + int64_t shape [4] = {1, 3, 224, 224}; + input.shape = &shape; + input.strides = NULL; + input.byte_offset = 0; + + tvm_runtime_set_input(handle, "data", &input); + gettimeofday(&t2, 0); + + tvm_runtime_run(handle); + gettimeofday(&t3, 0); + + float output_storage[OUTPUT_LEN]; + DLTensor output; + output.data = output_storage; + DLContext out_ctx = {kDLCPU, 0}; + output.ctx = out_ctx; + output.ndim = 2; + DLDataType out_dtype = {kDLFloat, 32, 1}; + output.dtype = out_dtype; + int64_t out_shape [2] = {1, OUTPUT_LEN}; + output.shape = &out_shape; + output.strides = NULL; + output.byte_offset = 0; + + tvm_runtime_get_output(handle, 0, &output); + gettimeofday(&t4, 0); + + float max_iter = -FLT_MAX; + int32_t max_index = -1; + for (auto i = 0; i < OUTPUT_LEN; ++i) { + if (output_storage[i] > max_iter) { + max_iter = output_storage[i]; + max_index = i; + } + } + + tvm_runtime_destroy(handle); + gettimeofday(&t5, 0); + + printf("The maximum position in output vector is: %d, with max-value %f.\n", + max_index, max_iter); + printf("timing: %.2f ms (create), %.2f ms (set_input), %.2f ms (run), " + "%.2f ms (get_output), %.2f ms (destroy)\n", + (t1.tv_sec-t0.tv_sec)*1000000 + (t1.tv_usec-t0.tv_usec)/1000.f, + (t2.tv_sec-t1.tv_sec)*1000000 + (t2.tv_usec-t1.tv_usec)/1000.f, + (t3.tv_sec-t2.tv_sec)*1000000 + (t3.tv_usec-t2.tv_usec)/1000.f, + (t4.tv_sec-t3.tv_sec)*1000000 + (t4.tv_usec-t3.tv_usec)/1000.f, + (t5.tv_sec-t4.tv_sec)*1000000 + (t5.tv_usec-t4.tv_usec)/1000.f); + + return 0; +} diff --git a/apps/bundle_deploy/test_static.c b/apps/bundle_deploy/test_static.c new file mode 100644 index 000000000000..3a8669699e69 --- /dev/null +++ b/apps/bundle_deploy/test_static.c @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include + +#include "bundle.h" + + +int main(int argc, char **argv) { + assert(argc == 5 && "Usage: test_static "); + + struct stat st; + char * json_data; + char * params_data; + uint64_t params_size; + + FILE * fp = fopen(argv[3], "rb"); + stat(argv[3], &st); + json_data = (char*)malloc(st.st_size); + fread(json_data, st.st_size, 1, fp); + fclose(fp); + + fp = fopen(argv[4], "rb"); + stat(argv[4], &st); + params_data = (char*)malloc(st.st_size); + fread(params_data, st.st_size, 1, fp); + params_size = st.st_size; + fclose(fp); + + struct timeval t0, t1, t2, t3, t4, t5; + gettimeofday(&t0, 0); + + auto *handle = tvm_runtime_create(json_data, params_data, params_size); + gettimeofday(&t1, 0); + + float input_storage[10 * 5]; + fp = fopen(argv[1], "rb"); + fread(input_storage, 10 * 5, 4, fp); + fclose(fp); + + float result_storage[10 * 5]; + fp = fopen(argv[2], "rb"); + fread(result_storage, 10 * 5, 4, fp); + fclose(fp); + + DLTensor input; + input.data = input_storage; + DLContext ctx = {kDLCPU, 0}; + input.ctx = ctx; + input.ndim = 2; + DLDataType dtype = {kDLFloat, 32, 1}; + input.dtype = dtype; + int64_t shape [2] = {10, 5}; + input.shape = &shape; + input.strides = NULL; + input.byte_offset = 0; + + tvm_runtime_set_input(handle, "x", &input); + gettimeofday(&t2, 0); + + tvm_runtime_run(handle); + gettimeofday(&t3, 0); + + float output_storage[10 * 5]; + DLTensor output; + output.data = output_storage; + DLContext out_ctx = {kDLCPU, 0}; + output.ctx = out_ctx; + output.ndim = 2; + DLDataType out_dtype = {kDLFloat, 32, 1}; + output.dtype = out_dtype; + int64_t out_shape [2] = {10, 5}; + output.shape = &out_shape; + output.strides = NULL; + output.byte_offset = 0; + + tvm_runtime_get_output(handle, 0, &output); + gettimeofday(&t4, 0); + + for (auto i = 0; i < 10 * 5; ++i) { + assert(fabs(output_storage[i] - result_storage[i]) < 1e-5f); + if (fabs(output_storage[i] - result_storage[i]) >= 1e-5f) { + printf("got %f, expected %f\n", output_storage[i], result_storage[i]); + } + } + + tvm_runtime_destroy(handle); + gettimeofday(&t5, 0); + + printf("timing: %.2f ms (create), %.2f ms (set_input), %.2f ms (run), " + "%.2f ms (get_output), %.2f ms (destroy)\n", + (t1.tv_sec-t0.tv_sec)*1000000 + (t1.tv_usec-t0.tv_usec)/1000.f, + (t2.tv_sec-t1.tv_sec)*1000000 + (t2.tv_usec-t1.tv_usec)/1000.f, + (t3.tv_sec-t2.tv_sec)*1000000 + (t3.tv_usec-t2.tv_usec)/1000.f, + (t4.tv_sec-t3.tv_sec)*1000000 + (t4.tv_usec-t3.tv_usec)/1000.f, + (t5.tv_sec-t4.tv_sec)*1000000 + (t5.tv_usec-t4.tv_usec)/1000.f); + + free(json_data); + free(params_data); + + return 0; +} diff --git a/tests/scripts/task_python_integration.sh b/tests/scripts/task_python_integration.sh index 5c00fd9c8896..d9cc0eccf3c5 100755 --- a/tests/scripts/task_python_integration.sh +++ b/tests/scripts/task_python_integration.sh @@ -33,7 +33,7 @@ make cython3 # Test MISRA-C runtime cd apps/bundle_deploy rm -rf build -make test +make test_dynamic test_static cd ../.. # Test extern package