From 89223ce0d9e14fb08d0c362bc203ee0c8dfa42b6 Mon Sep 17 00:00:00 2001 From: Vladimir Kuzmin Date: Thu, 6 Aug 2020 03:50:15 -0700 Subject: [PATCH] [C API] Continue implementation Continue with basic implementation of a C API for Neuropod. Issue #407 Added a test that loads a model and fails, runs inference successfully and fails, and verifies the name and platform. --- source/neuropod/bindings/c/README.md | 7 ++ source/neuropod/bindings/c/c_api.cc | 114 +++++++++++++++++++++--- source/neuropod/bindings/c/c_api.h | 65 ++++++++++---- source/neuropod/bindings/c/np_tensor.cc | 16 ++-- source/neuropod/bindings/c/np_tensor.h | 3 + source/neuropod/tests/test_c_api.cc | 59 ++++++++++-- 6 files changed, 224 insertions(+), 40 deletions(-) create mode 100644 source/neuropod/bindings/c/README.md diff --git a/source/neuropod/bindings/c/README.md b/source/neuropod/bindings/c/README.md new file mode 100644 index 00000000..800b67fb --- /dev/null +++ b/source/neuropod/bindings/c/README.md @@ -0,0 +1,7 @@ +# The C api interface: + +1. To build &to unit test : +``` +bazel build // neuropod/bindings/c:c_api +bazel test --test_output=errors // neuropod/tests:test_c_api +``` diff --git a/source/neuropod/bindings/c/c_api.cc b/source/neuropod/bindings/c/c_api.cc index dee44588..b062aa82 100644 --- a/source/neuropod/bindings/c/c_api.cc +++ b/source/neuropod/bindings/c/c_api.cc @@ -18,32 +18,126 @@ limitations under the License. #include "neuropod/bindings/c/c_api.h" #include "neuropod/bindings/c/c_api_internal.h" +#include "neuropod/bindings/c/np_status_internal.h" #include "neuropod/bindings/c/np_tensor_allocator_internal.h" #include "neuropod/bindings/c/np_valuemap_internal.h" -// Load a model given a path -void NP_LoadNeuropod(const char *neuropod_path, NP_Neuropod **model, NP_Status *status) +#include +#include +#include + +void NP_LoadNeuropodWithOpts(const char * neuropod_path, + const NP_RuntimeOptions *options, + NP_Neuropod ** model, + NP_Status * status) { - *model = new NP_Neuropod(); + try + { + *model = new NP_Neuropod(); + (*model)->model = std::make_unique(neuropod_path); + if (status != nullptr) + { + NP_ClearStatus(status); + } + } + catch (std::exception &e) + { + if (status != nullptr) + { + status->code = NEUROPOD_ERROR; + status->message = e.what(); + } + } +} + +NP_RuntimeOptions NP_DefaultRuntimeOptions() +{ + // Use C++ runtime options object to set default values. + neuropod::RuntimeOptions default_options; + NP_RuntimeOptions options; + + options.use_ope = default_options.use_ope; + options.visible_device = static_cast(default_options.visible_device); + options.load_model_at_construction = default_options.load_model_at_construction; + options.disable_shape_and_type_checking = default_options.disable_shape_and_type_checking; + + auto ope_options = &options.ope_options; + ope_options->free_memory_every_cycle = default_options.ope_options.free_memory_every_cycle; + // Control queue name is empty, a new worker will be started. + // This is what is expected by default. + ope_options->control_queue_name[0] = '\0'; - (*model)->model = std::make_unique(neuropod_path); + return options; +} + +void NP_LoadNeuropod(const char *neuropod_path, NP_Neuropod **model, NP_Status *status) +{ + const auto &options = NP_DefaultRuntimeOptions(); + NP_LoadNeuropodWithOpts(neuropod_path, &options, model, status); } -// Free a model void NP_FreeNeuropod(NP_Neuropod *model) { delete model; } -// Run inference -// Note: The caller is responsible for freeing the returned NP_NeuropodValueMap void NP_Infer(NP_Neuropod *model, const NP_NeuropodValueMap *inputs, NP_NeuropodValueMap **outputs, NP_Status *status) { - *outputs = new NP_NeuropodValueMap(); - (*outputs)->data = std::move(*model->model->infer(inputs->data)); + NP_InferWithRequestedOutputs(model, inputs, 0, nullptr, outputs, status); +} + +void NP_InferWithRequestedOutputs(NP_Neuropod * model, + const NP_NeuropodValueMap *inputs, + size_t noutputs, + const char ** requested_outputs, + NP_NeuropodValueMap ** outputs, + NP_Status * status) +{ + try + { + // By default empty collection of requested_ouputs expected. + std::vector rout; + for (size_t i = 0; i < noutputs; ++i) + { + rout.push_back(requested_outputs[i]); + } + *outputs = new NP_NeuropodValueMap(); + (*outputs)->data = std::move(*model->model->infer(inputs->data, rout)); + if (status != nullptr) + { + NP_ClearStatus(status); + } + } + catch (std::exception &e) + { + if (status != nullptr) + { + status->code = NEUROPOD_ERROR; + status->message = e.what(); + } + } +} + +const char *NP_GetName(NP_Neuropod *model) +{ + return model->model->get_name().c_str(); +} + +const char *NP_GetPlatform(NP_Neuropod *model) +{ + return model->model->get_platform().c_str(); +} + +size_t NP_GetNumInputs(NP_Neuropod *model) +{ + return model->model->get_inputs().size(); +} + +size_t NP_GetNumOutputs(NP_Neuropod *model) +{ + return model->model->get_outputs().size(); } -// Get an allocator for a model NP_TensorAllocator *NP_GetAllocator(NP_Neuropod *model) { auto out = new NP_TensorAllocator(); diff --git a/source/neuropod/bindings/c/c_api.h b/source/neuropod/bindings/c/c_api.h index 3ac765ca..db26186b 100644 --- a/source/neuropod/bindings/c/c_api.h +++ b/source/neuropod/bindings/c/c_api.h @@ -29,19 +29,44 @@ extern "C" { typedef struct NP_Neuropod NP_Neuropod; -// Load a model given a path +// Load a model given a path. +// status is set if user provided a valid pointer. void NP_LoadNeuropod(const char *neuropod_path, NP_Neuropod **model, NP_Status *status); -// TODO: Add more options +// Runtime Options. See description of options at neuropod/options.hh typedef struct NP_RuntimeOptions { - // Whether or not to use out-of-process execution - // (using shared memory to communicate between the processes) + // Whether or not to use out-of-process execution. bool use_ope; + // These options are only used if use_ope is set to true. + struct NP_OPEOptions + { + bool free_memory_every_cycle; + char control_queue_name[256]; + } ope_options; + + // C enum that matches neuropod::NeuropodDevice + typedef enum NP_Device + { + CPU = -1, + GPU0 = 0, + GPU1 = 1, + GPU2 = 2, + GPU3 = 3, + GPU4 = 4, + GPU5 = 5, + GPU6 = 6, + GPU7 = 7 + } NP_Device; + NP_Device visible_device = GPU0; + + bool load_model_at_construction; + bool disable_shape_and_type_checking; } NP_RuntimeOptions; -// Load a model given a path and options +// Load a model given a path and options. +// status is set if user provided a valid pointer. void NP_LoadNeuropodWithOpts(const char * neuropod_path, const NP_RuntimeOptions *options, NP_Neuropod ** model, @@ -50,21 +75,29 @@ void NP_LoadNeuropodWithOpts(const char * neuropod_path, // Free a model void NP_FreeNeuropod(NP_Neuropod *model); -// Run inference -// Note: The caller is responsible for freeing the returned NP_NeuropodValueMap +// Run inference. +// status is set if user provided a valid pointer. +// Note: The caller is responsible for freeing the returned NP_NeuropodValueMap. +// Number of elements in outputs is equal to NP_GetNumOutputs. void NP_Infer(NP_Neuropod *model, const NP_NeuropodValueMap *inputs, NP_NeuropodValueMap **outputs, NP_Status *status); -// Run inference with a set of requested outputs -// requested_outputs should be a null terminated array of char ptr containing the names of requested outputs -// Note: The caller is responsible for freeing the returned NP_NeuropodValueMap +// Run inference with a set of requested outputs. +// requested_outputs should be array of char ptr containing the names of requested outputs. +// noutputs is in/out param that specifies number of elements in requested_outputs and +// number of elements in outputs after execution. +// status is set if user provided a valid pointer. +// Note: The caller is responsible for freeing the returned NP_NeuropodValueMap. void NP_InferWithRequestedOutputs(NP_Neuropod * model, const NP_NeuropodValueMap *inputs, + size_t noutput, const char ** requested_outputs, NP_NeuropodValueMap ** outputs, NP_Status * status); -// Get information about the model +// Get name of the model. const char *NP_GetName(NP_Neuropod *model); + +// Get platform identfier of the model. const char *NP_GetPlatform(NP_Neuropod *model); // Get the number of items in the input spec of a model @@ -73,17 +106,17 @@ size_t NP_GetNumInputs(NP_Neuropod *model); // Get the number of items in the output spec of a model size_t NP_GetNumOutputs(NP_Neuropod *model); -// Get an item from the input spec of a model -// Returns nullptr if index is out of range +// Get an item from the input spec of a model. +// Returns nullptr if index is out of range. // Note: The caller is responsible for freeing the returned TensorSpec NP_TensorSpec *NP_GetInputSpec(NP_Neuropod *model, size_t index); -// Get an item from the output spec of a model -// Returns nullptr if index is out of range +// Get an item from the output spec of a model. +// Returns nullptr if index is out of range. // Note: The caller is responsible for freeing the returned TensorSpec NP_TensorSpec *NP_GetOutputSpec(NP_Neuropod *model, size_t index); -// Get an allocator for a model +// Get an allocator for a model. // Note: The caller is responsible for freeing the returned TensorAllocator NP_TensorAllocator *NP_GetAllocator(NP_Neuropod *model); diff --git a/source/neuropod/bindings/c/np_tensor.cc b/source/neuropod/bindings/c/np_tensor.cc index 26915a34..37844a56 100644 --- a/source/neuropod/bindings/c/np_tensor.cc +++ b/source/neuropod/bindings/c/np_tensor.cc @@ -18,17 +18,21 @@ limitations under the License. #include "neuropod/bindings/c/np_tensor_internal.h" #include "neuropod/internal/neuropod_tensor_raw_data_access.hh" -// For non-string tensors, get a pointer to the underlying data -// Returns nullptr if called on a string tensor void *NP_GetData(NP_NeuropodTensor *tensor) { return neuropod::internal::NeuropodTensorRawDataAccess::get_untyped_data_ptr(*tensor->tensor->as_tensor()); } -// Releases a tensor. The memory might not be deallocated immediately if the tensor is still -// referenced by a value-map object or used by an infer operation. -// This should be called on every tensor returned by the C API. -// See the notes in `np_valuemap.h` and `np_tensor_allocator.h` for more detail. +const void *NP_GetDataReadOnly(const NP_NeuropodTensor *tensor) +{ + return neuropod::internal::NeuropodTensorRawDataAccess::get_untyped_data_ptr(*tensor->tensor->as_tensor()); +} + +size_t NP_GetNumElements(const NP_NeuropodTensor *tensor) +{ + return tensor->tensor->as_tensor()->get_num_elements(); +} + void NP_FreeTensor(NP_NeuropodTensor *tensor) { delete tensor; diff --git a/source/neuropod/bindings/c/np_tensor.h b/source/neuropod/bindings/c/np_tensor.h index 358b9c54..90bf0282 100644 --- a/source/neuropod/bindings/c/np_tensor.h +++ b/source/neuropod/bindings/c/np_tensor.h @@ -61,6 +61,9 @@ void *NP_GetData(NP_NeuropodTensor *tensor); // Returns nullptr if called on a string tensor const void *NP_GetDataReadOnly(const NP_NeuropodTensor *tensor); +// Returns number of elements in tensor. +size_t NP_GetNumElements(const NP_NeuropodTensor *tensor); + // For string tensors, set the value of a specified element in the flattened tensor void NP_SetStringElement(NP_NeuropodTensor *tensor, size_t index, const char *item, NP_Status *status); diff --git a/source/neuropod/tests/test_c_api.cc b/source/neuropod/tests/test_c_api.cc index 047683f1..c7f04aa2 100644 --- a/source/neuropod/tests/test_c_api.cc +++ b/source/neuropod/tests/test_c_api.cc @@ -16,19 +16,41 @@ limitations under the License. #include "gtest/gtest.h" #include "neuropod/bindings/c/c_api.h" +#include +#include + TEST(test_c_api, basic) { NP_Neuropod *model; NP_Status * status = NP_NewStatus(); - // Load a model and get an allocator + // Load a model from a wrong path. + NP_LoadNeuropod("wrong_path", &model, status); + if (NP_GetCode(status) != NEUROPOD_ERROR) + { + // Throw an error here + FAIL() << "Error from C API expeted error during model loading: " << NP_GetMessage(status); + } + + // Load a model and get an allocator. NP_LoadNeuropod("neuropod/tests/test_data/tf_addition_model/", &model, status); if (NP_GetCode(status) != NEUROPOD_OK) { - // Throw an error here FAIL() << "Error from C API during model loading: " << NP_GetMessage(status); } + // Test model name. + if (NP_GetName(model) != std::string("addition_model")) + { + FAIL() << "Error from C API unexpected model name: " << NP_GetName(model); + } + + // Test model platform. + if (NP_GetPlatform(model) != std::string("tensorflow")) + { + FAIL() << "Error from C API unexpected model name: " << NP_GetPlatform(model); + } + NP_TensorAllocator *allocator = NP_GetAllocator(model); // Create tensors @@ -45,6 +67,16 @@ TEST(test_c_api, basic) // Create the input NP_NeuropodValueMap *inputs = NP_NewValueMap(); + NP_NeuropodValueMap *outputs; + + // Run inference with empty input that should fail. + NP_Infer(model, inputs, &outputs, status); + if (NP_GetCode(status) != NEUROPOD_ERROR) + { + FAIL() << "Error from C API error is expected during inference: " << NP_GetMessage(status); + } + + // Insert tensors into inputs NP_InsertTensor(inputs, "x", x); NP_InsertTensor(inputs, "y", y); @@ -55,19 +87,30 @@ TEST(test_c_api, basic) // Free the allocator NP_FreeAllocator(allocator); - // Run inference - NP_NeuropodValueMap *outputs; + // Run succcessful inference NP_Infer(model, inputs, &outputs, status); if (NP_GetCode(status) != NEUROPOD_OK) { - // Throw an error here FAIL() << "Error from C API during inference: " << NP_GetMessage(status); } + // The same inference but specify requested output. + const char *requested_output[] = {"out"}; + NP_InferWithRequestedOutputs(model, inputs, 1, static_cast(requested_output), &outputs, status); + if (NP_GetCode(status) != NEUROPOD_OK) + { + FAIL() << "Error from C API during inference: " << NP_GetMessage(status); + } + + // Test model input and output configuration. + EXPECT_EQ(2, NP_GetNumInputs(model)); + EXPECT_EQ(1, NP_GetNumOutputs(model)); + // Get the output and compare to the expected value - NP_NeuropodTensor *out = NP_GetTensor(outputs, "out"); - float * out_data = reinterpret_cast(NP_GetData(out)); - EXPECT_TRUE(std::equal(target, target + 4, out_data)); + NP_NeuropodTensor *out = NP_GetTensor(outputs, "out"); + float * out_data = reinterpret_cast(NP_GetData(out)); + size_t nout_data = NP_GetNumElements(out); + EXPECT_TRUE(std::equal(out_data, out_data + nout_data, target)); // Free the input and output maps NP_FreeValueMap(inputs);