diff --git a/src/apidata.cc b/src/apidata.cc index 97fba182f..124283fd9 100644 --- a/src/apidata.cc +++ b/src/apidata.cc @@ -21,6 +21,8 @@ #include "apidata.h" +#include "utils/utils.hpp" + namespace dd { /*- visitor_vad -*/ @@ -244,4 +246,11 @@ namespace dd } } + std::string APIData::toJSONString() const + { + JDoc jd; + jd.SetObject(); + toJDoc(jd); + return dd_utils::jrender(jd); + } } diff --git a/src/apidata.h b/src/apidata.h index 34924e4db..e5ef389b6 100644 --- a/src/apidata.h +++ b/src/apidata.h @@ -285,6 +285,11 @@ namespace dd */ void toJVal(JDoc &jd, JVal &jv) const; + /** + * \brief converts APIData to json string + */ + std::string toJSONString() const; + /** * \brief converts APIData to oat++ DTO */ diff --git a/src/backends/tensorrt/tensorrtlib.cc b/src/backends/tensorrt/tensorrtlib.cc index 5fd9d1826..f829a890b 100644 --- a/src/backends/tensorrt/tensorrtlib.cc +++ b/src/backends/tensorrt/tensorrtlib.cc @@ -139,6 +139,12 @@ namespace dd _floatOut = tl._floatOut; _keepCount = tl._keepCount; _dims = tl._dims; + _error_recorder = tl._error_recorder; + _calibrator = tl._calibrator; + _engine = tl._engine; + _builder = tl._builder; + _context = tl._context; + _builderc = tl._builderc; _runtime = tl._runtime; } @@ -147,6 +153,12 @@ namespace dd TensorRTLib::~TensorRTLib() { + // Delete objects in the correct order + _calibrator = nullptr; + _context = nullptr; + _engine = nullptr; + _builderc = nullptr; + _builder = nullptr; } template ( nvinfer1::createInferRuntime(trtLogger)); - _runtime->setErrorRecorder(new TRTErrorRecorder(this->_logger)); + _error_recorder.reset(new TRTErrorRecorder(this->_logger)); + _runtime->setErrorRecorder(_error_recorder.get()); if (ad.has("tensorRTEngineFile")) _engineFileName = ad.get("tensorRTEngineFile").get(); @@ -377,7 +390,8 @@ namespace dd break; } - nvinfer1::INetworkDefinition *network = _builder->createNetworkV2(0U); + std::unique_ptr network( + _builder->createNetworkV2(0U)); nvcaffeparser1::ICaffeParser *caffeParser = nvcaffeparser1::createCaffeParser(); @@ -426,7 +440,6 @@ namespace dd outl->setPrecision(nvinfer1::DataType::kFLOAT); nvinfer1::IHostMemory *n = _builder->buildSerializedNetwork(*network, *_builderc); - return _runtime->deserializeCudaEngine(n->data(), n->size()); } @@ -439,8 +452,8 @@ namespace dd const auto explicitBatch = 1U << static_cast( nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH); - nvinfer1::INetworkDefinition *network - = _builder->createNetworkV2(explicitBatch); + std::unique_ptr network( + _builder->createNetworkV2(explicitBatch)); _explicit_batch = true; nvonnxparser::IParser *onnxParser @@ -473,7 +486,6 @@ namespace dd if (n == nullptr) throw MLLibInternalException("Could not build model: " + this->_mlmodel._model); - return _runtime->deserializeCudaEngine(n->data(), n->size()); } @@ -506,6 +518,12 @@ namespace dd if (ad.has("data_raw_img")) predict_dto->_data_raw_img = ad.get("data_raw_img").get>(); +#ifdef USE_CUDA_CV + if (ad.has("data_raw_img_cuda")) + predict_dto->_data_raw_img_cuda + = ad.get("data_raw_img_cuda") + .get>(); +#endif if (ad.has("ids")) predict_dto->_ids = ad.get("ids").get>(); if (ad.has("meta_uris")) diff --git a/src/backends/tensorrt/tensorrtlib.h b/src/backends/tensorrt/tensorrtlib.h index e1c6f5ad0..52ceae2f7 100644 --- a/src/backends/tensorrt/tensorrtlib.h +++ b/src/backends/tensorrt/tensorrtlib.h @@ -20,11 +20,11 @@ #ifndef TENSORRTLIB_H #define TENSORRTLIB_H -#include "tensorrtmodel.h" -#include "apidata.h" #include "NvCaffeParser.h" #include "NvInfer.h" +#include "apidata.h" +#include "tensorrtmodel.h" #include "error_recorder.hpp" namespace dd @@ -127,6 +127,7 @@ namespace dd _template; /**< template for models that require specific treatment */ //!< The TensorRT engine used to run the network + std::shared_ptr _error_recorder = nullptr; std::shared_ptr _calibrator = nullptr; std::shared_ptr _engine = nullptr; std::shared_ptr _builder = nullptr; diff --git a/src/chain.cc b/src/chain.cc index 9e887488d..f3bc538be 100644 --- a/src/chain.cc +++ b/src/chain.cc @@ -96,7 +96,7 @@ namespace dd for (auto p : *body->predictions) { std::string uri = p->uri; - p->uri = model_name.c_str(); + p->uri = model_name; other_models_out.insert( std::pair>(uri, p)); @@ -123,7 +123,7 @@ namespace dd for (auto p : *out_body->predictions) { std::string uri = p->uri; - p->uri = action_id.c_str(); + p->uri = action_id; other_models_out.insert( std::pair>(uri, p)); diff --git a/src/chain_actions.cc b/src/chain_actions.cc index b7c45686c..c8e1e0cf9 100644 --- a/src/chain_actions.cc +++ b/src/chain_actions.cc @@ -185,7 +185,6 @@ namespace dd } cv::Rect roi(cxmin, cymin, cxmax - cxmin, cymax - cymin); - #ifdef USE_CUDA_CV if (!cuda_imgs.empty()) { @@ -235,11 +234,16 @@ namespace dd } // store crops into action output store APIData action_out; - action_out.add("data_raw_img", cropped_imgs); #ifdef USE_CUDA_CV if (!cropped_cuda_imgs.empty()) - action_out.add("data_cuda_img", cropped_cuda_imgs); + { + action_out.add("data_cuda_img", cropped_cuda_imgs); + } + else #endif + { + action_out.add("data_raw_img", cropped_imgs); + } action_out.add("cids", bbox_ids); cdata.add_action_data(_action_id, action_out); @@ -326,6 +330,205 @@ namespace dd cdata.add_action_data(_action_id, action_out); } + void make_even(cv::Mat &mat, int &width, int &height) + { + if (width % 2 != 0) + width -= 1; + if (height % 2 != 0) + height -= 1; + + if (width != mat.cols || height != mat.rows) + { + cv::Rect roi{ 0, 0, width, height }; + mat = mat(roi); + } + } + + void ImgsCropRecomposeAction::apply(APIData &model_out, ChainData &cdata) + { + APIData first_model = cdata.get_model_data("0"); + APIData input_ad = first_model.getobj("input"); + // image + std::vector imgs; +#ifdef USE_CUDA_CV + std::vector cuda_imgs; + std::vector cropped_cuda_imgs; + + if (input_ad.has("cuda_imgs")) + { + cuda_imgs + = input_ad.get("cuda_imgs").get>(); + } + else +#endif + { + imgs = input_ad.get("imgs").get>(); + } + std::vector> imgs_size + = input_ad.get("imgs_size").get>>(); + + // bbox + std::vector vad = first_model.getv("predictions"); + + // generated images + // XXX: Images always are written on RAM first. + // This may change in the future + std::map gen_imgs; + if (model_out.has("dto")) + { + auto dto = model_out.get("dto") + .get() + .retrieve>(); + for (size_t i = 0; i < dto->predictions->size(); ++i) + { + auto images = dto->predictions->at(i)->images; + if (images->size() == 0) + throw ActionBadParamException( + "Recompose requires output.image = true in previous model"); + gen_imgs.insert( + { *dto->predictions->at(i)->uri, images->at(0)->get_img() }); + } + } + else + { + throw ActionBadParamException("Recompose action requires GAN output"); + } + + std::vector rimgs; + std::vector uris; + + bool save_img = _params->save_img; + std::string save_path = _params->save_path; + if (!save_path.empty()) + save_path += "/"; + + auto pred_body = DTO::PredictBody::createShared(); + + // need: original image, bbox coordinates, new image + for (size_t i = 0; i < vad.size(); i++) + { + std::string uri = vad.at(i).get("uri").get(); + uris.push_back(uri); + + cv::Mat input_img; + int input_width, input_height; + cv::Mat rimg; +#ifdef USE_CUDA_CV + cv::cuda::GpuMat cuda_input_img; + cv::cuda::GpuMat cuda_rimg; + if (!cuda_imgs.empty()) + { + cuda_input_img = cuda_imgs.at(i); + input_width = cuda_input_img.cols; + input_height = cuda_input_img.rows; + cuda_rimg = cuda_input_img.clone(); + } + else +#endif + { + input_img = imgs.at(i); + input_width = input_img.cols; + input_height = input_img.rows; + rimg = input_img.clone(); + } + int orig_width = imgs_size.at(i).second; + int orig_height = imgs_size.at(i).first; + + std::vector ad_cls = vad.at(i).getv("classes"); + APIData bbox; + + for (size_t j = 0; j < ad_cls.size(); j++) + { + bbox = ad_cls.at(j).getobj("bbox"); + std::string cls_id + = ad_cls.at(j).get("class_id").get(); + + cv::Mat gen_img = gen_imgs.at(cls_id); + int gen_width = gen_img.cols; + int gen_height = gen_img.rows; + + // support odd width & height + make_even(gen_img, gen_width, gen_height); + + if (gen_width > input_width || gen_height > input_height) + { + throw ActionBadParamException( + "Recomposing image is impossible, crop is too big: " + + std::to_string(gen_width) + "," + + std::to_string(gen_height) + "/" + + std::to_string(orig_width) + "," + + std::to_string(orig_height)); + } + + double xmin + = bbox.get("xmin").get() / orig_width * input_width; + double ymin + = bbox.get("ymin").get() / orig_height * input_height; + double xmax + = bbox.get("xmax").get() / orig_width * input_width; + double ymax + = bbox.get("ymax").get() / orig_height * input_height; + + int cx = static_cast((xmin + xmax) / 2); + int cy = static_cast((ymin + ymax) / 2); + cx = std::min(std::max(cx, gen_width / 2), + input_width - gen_width / 2); + cy = std::min(std::max(cy, gen_height / 2), + input_height - gen_height / 2); + + cv::Rect roi{ cx - gen_width / 2, cy - gen_height / 2, gen_width, + gen_height }; +#ifdef USE_CUDA_CV + if (cuda_rimg.cols != 0) + { + cv::cuda::GpuMat cuda_gen_img; + cuda_gen_img.upload(gen_img); + cuda_gen_img.copyTo(cuda_rimg(roi)); + } + else +#endif + { + gen_img.copyTo(rimg(roi)); + } + } + + rimgs.push_back(rimg); + + auto action_pred = DTO::Prediction::createShared(); + action_pred->uri = uri.c_str(); + action_pred->images = oatpp::Vector::createShared(); +#ifdef USE_CUDA_CV + if (cuda_rimg.cols != 0) + { + action_pred->images->push_back({ cuda_rimg }); + } + else +#endif + { + action_pred->images->push_back({ rimg }); + } + pred_body->predictions->push_back(action_pred); + + // save image if requested + if (save_img) + { + std::string puri = dd_utils::split(uri, '/').back(); +#ifdef USE_CUDA_CV + if (cuda_rimg.cols != 0) + cuda_rimg.download(rimg); +#endif + cv::imwrite(save_path + "recompose_" + puri + ".png", rimg); + } + } + + // Output: new image -> only works in "image" output mode + APIData action_out; + action_out.add("data_raw_img", rimgs); + action_out.add("cids", uris); + action_out.add("output", pred_body); + cdata.add_action_data(_action_id, action_out); + } + cv::Scalar bbox_palette[] = { { 82, 188, 227 }, { 196, 110, 49 }, { 39, 54, 227 }, { 68, 227, 81 }, { 77, 157, 255 }, { 255, 112, 207 }, @@ -533,6 +736,7 @@ namespace dd CHAIN_ACTION("crop", ImgsCropAction) CHAIN_ACTION("rotate", ImgsRotateAction) + CHAIN_ACTION("recompose", ImgsCropRecomposeAction) CHAIN_ACTION("draw_bbox", ImgsDrawBBoxAction) CHAIN_ACTION("filter", ClassFilter) #ifdef USE_DLIB diff --git a/src/chain_actions.h b/src/chain_actions.h index 132d34679..1e20f8d8c 100644 --- a/src/chain_actions.h +++ b/src/chain_actions.h @@ -88,6 +88,15 @@ namespace dd return std::to_string(std::hash{}(str)); } + /** Apply an action to a model output. + * \param model_out Output of the previous model. + * \param cdata Chain data object containing all the output of previous + * models and actions. This method should add the action result to cdata + * using the `add_action_data()` method. If the action creates data that + * should appear in the result, they are stored in the "output" field. If + * the action creates data that should be used by the next model, they are + * stored in other fields, as handled in `services.h:chain_service()`. + * */ void apply(APIData &model_out, ChainData &cdata); std::string _action_id; @@ -129,6 +138,24 @@ namespace dd void apply(APIData &model_out, ChainData &cdata); }; + /** Recompose origin image with inference result, for example in crop + GAN + * pipeline */ + class ImgsCropRecomposeAction : public ChainAction + { + public: + ImgsCropRecomposeAction(oatpp::Object call_dto, + const std::shared_ptr chain_logger) + : ChainAction(call_dto, chain_logger) + { + } + + ~ImgsCropRecomposeAction() + { + } + + void apply(APIData &model_out, ChainData &cdata); + }; + class ImgsDrawBBoxAction : public ChainAction { public: @@ -206,6 +233,9 @@ namespace dd oatpp::Object _call_dto; }; + /** Add an action type to DeepDetect chains. The action transforms the output + * of a model. This output can then be used by another model or embedded in + * the API result. */ #define CHAIN_ACTION(ActionName, ActionType) \ namespace \ { \ diff --git a/src/dd_types.h b/src/dd_types.h index c1e20e30a..1c94ef78a 100644 --- a/src/dd_types.h +++ b/src/dd_types.h @@ -23,6 +23,7 @@ #define DDTYPES_H #include + class RapidjsonException : public std::exception { public: diff --git a/src/dto/predict_out.hpp b/src/dto/predict_out.hpp index fa8b7313d..48aef6d98 100644 --- a/src/dto/predict_out.hpp +++ b/src/dto/predict_out.hpp @@ -128,7 +128,7 @@ namespace dd info->description = "[Unsupervised] Array of images returned by the model"; } - DTO_FIELD(Vector, images); + DTO_FIELD(Vector, images) = Vector::createShared(); DTO_FIELD_INFO(imgsize) { diff --git a/src/jsonapi.cc b/src/jsonapi.cc index 2689848c3..a34c6fece 100644 --- a/src/jsonapi.cc +++ b/src/jsonapi.cc @@ -349,30 +349,15 @@ namespace dd return jd; } + // XXX: legacy methods, remove in favor of dd_utils::jrender? std::string JsonAPI::jrender(const JDoc &jst) const { - rapidjson::StringBuffer buffer; - rapidjson::Writer, - rapidjson::UTF8<>, rapidjson::CrtAllocator, - rapidjson::kWriteNanAndInfFlag> - writer(buffer); - bool done = jst.Accept(writer); - if (!done) - throw DataConversionException("JSON rendering failed"); - return buffer.GetString(); + return dd_utils::jrender(jst); } std::string JsonAPI::jrender(const JVal &jval) const { - rapidjson::StringBuffer buffer; - rapidjson::Writer, - rapidjson::UTF8<>, rapidjson::CrtAllocator, - rapidjson::kWriteNanAndInfFlag> - writer(buffer); - bool done = jval.Accept(writer); - if (!done) - throw DataConversionException("JSON rendering failed"); - return buffer.GetString(); + return dd_utils::jrender(jval); } JDoc JsonAPI::info(const std::string &jstr) const diff --git a/src/services.h b/src/services.h index b0d6e9984..8b2173b8e 100644 --- a/src/services.h +++ b/src/services.h @@ -764,6 +764,14 @@ namespace dd // be supported / // auto-detected) } +#ifdef USE_CUDA_CV + else if (act_data.has("data_cuda_img")) + { + adc.add("data_raw_img_cuda", + act_data.get("data_cuda_img") + .get>()); + } +#endif else if (act_data.has("data_raw_img")) // raw images { adc.add( @@ -819,7 +827,8 @@ namespace dd size_t npred_classes = pred->classes != nullptr ? pred->classes->size() : 0; classes_size += npred_classes; - vals_size += static_cast(pred->vals != nullptr); + vals_size += static_cast(pred->vals != nullptr) + + static_cast(pred->images->size()); if (chain_pos == 0) // first call's response contains uniformized // top level URIs. @@ -913,7 +922,8 @@ namespace dd = adc.getobj("action").get("type").get(); APIData prev_data = cdata.get_model_data(prec_pred_id); - if (!prev_data.getv("predictions").size()) + // XXX: no prediction not checked in case of dto + if (prev_data.getv("predictions").empty() && !prev_data.has("dto")) { // no prediction to work from chain_logger->info("no prediction to act on"); @@ -930,7 +940,7 @@ namespace dd cdata.add_model_data(prec_pred_id, prev_data); std::vector vad = prev_data.getv("predictions"); - if (vad.empty()) + if (vad.empty() && !prev_data.has("dto")) { // no prediction to work from chain_logger->info("no prediction to act on after applying action " @@ -938,6 +948,7 @@ namespace dd return 1; } + // check that there are predictions int classes_size = 0; int vals_size = 0; for (size_t i = 0; i < vad.size(); i++) @@ -948,7 +959,7 @@ namespace dd vals_size += static_cast(vad.at(i).has("vals")); } - if (!classes_size && !vals_size) + if (classes_size == 0 && vals_size == 0 && !prev_data.has("dto")) { chain_logger->info("[" + std::to_string(chain_pos) + "] / no result after applying action " diff --git a/src/utils/oatpp.cc b/src/utils/oatpp.cc index 22dd1844d..903e077e6 100644 --- a/src/utils/oatpp.cc +++ b/src/utils/oatpp.cc @@ -23,6 +23,7 @@ #include #include "dto/ddtypes.hpp" +#include "utils/utils.hpp" namespace dd { @@ -294,5 +295,13 @@ namespace dd + "\": type not recognised"); } } + + std::string dtoToJSONString(const oatpp::Void &polymorph, bool ignore_null) + { + JDoc jd; + jd.SetObject(); + dtoToJDoc(polymorph, jd, ignore_null); + return dd_utils::jrender(jd); + } } } diff --git a/src/utils/oatpp.hpp b/src/utils/oatpp.hpp index 7b9159341..28f5dd9f1 100644 --- a/src/utils/oatpp.hpp +++ b/src/utils/oatpp.hpp @@ -69,6 +69,9 @@ namespace dd bool ignore_null = true); void dtoToJVal(const oatpp::Void &polymorph, JDoc &jdoc, JVal &jval, bool ignore_null = true); + + std::string dtoToJSONString(const oatpp::Void &polymorph, + bool ignore_null = true); } } diff --git a/src/utils/utils.hpp b/src/utils/utils.hpp index dcedf5986..32f3e79dd 100644 --- a/src/utils/utils.hpp +++ b/src/utils/utils.hpp @@ -27,6 +27,13 @@ #include #include +#include +#include +#include +#include + +#include "apidata.h" +#include "dd_types.h" namespace dd { @@ -82,6 +89,32 @@ namespace dd : item.substr(start, end - start + 1); } + inline std::string jrender(const JDoc &jst) + { + rapidjson::StringBuffer buffer; + rapidjson::Writer, + rapidjson::UTF8<>, rapidjson::CrtAllocator, + rapidjson::kWriteNanAndInfFlag> + writer(buffer); + bool done = jst.Accept(writer); + if (!done) + throw DataConversionException("JSON rendering failed"); + return buffer.GetString(); + } + + inline std::string jrender(const JVal &jval) + { + rapidjson::StringBuffer buffer; + rapidjson::Writer, + rapidjson::UTF8<>, rapidjson::CrtAllocator, + rapidjson::kWriteNanAndInfFlag> + writer(buffer); + bool done = jval.Accept(writer); + if (!done) + throw DataConversionException("JSON rendering failed"); + return buffer.GetString(); + } + inline bool iequals(const std::string &a, const std::string &b) { unsigned int sz = a.size(); diff --git a/tests/ut-chain.cc b/tests/ut-chain.cc index 5fb6dcbb8..12f75462b 100644 --- a/tests/ut-chain.cc +++ b/tests/ut-chain.cc @@ -377,7 +377,7 @@ TEST(chain, chain_trt_detection_gan) + trt_detect_repo + "\"},\"parameters\":{\"input\":{\"connector\":" "\"image\",\"height\":640,\"width\":640},\"mllib\":{" - "\"maxBatchSize\":2,\"maxWorkspaceSize\":256,\"gpuid\":0," + "\"maxWorkspaceSize\":256,\"gpuid\":0," "\"template\":\"yolox\",\"nclasses\":81,\"datatype\":\"fp16\"}}}"; std::string joutstr = japi.jrender(japi.service_create(detect_sname, jstr)); ASSERT_EQ(created_str, joutstr); @@ -401,9 +401,9 @@ TEST(chain, chain_trt_detection_gan) + "\",\"parameters\":{\"input\":{\"keep_orig\":true},\"output\":{" "\"bbox\":true,\"best_bbox\":1}},\"data\":[\"" + trt_gan_repo - + "/horse.jpg\"]}," + + "/horse_1024.jpg\"]}," "{\"id\":\"crop\",\"action\":{\"type\":\"crop\",\"parameters\":{" - "\"fixed_size\":360}}},{\"service\":\"" + "\"fixed_width\":360,\"fixed_height\":360}}},{\"service\":\"" + gan_sname + "\",\"parent_id\":\"crop\",\"parameters\":{\"mllib\":{\"extract_" "layer\":\"last\"},\"output\":{}}}" @@ -431,6 +431,50 @@ TEST(chain, chain_trt_detection_gan) ASSERT_TRUE(gan_pred["vals"].IsArray()); ASSERT_EQ(gan_pred["vals"].Size(), 360 * 360 * 3); + // image recompose + // XXX: keep_orig = false doesn't work on CUDA images! + jchainstr + = "{\"chain\":{\"name\":\"chain\",\"calls\":[{\"service\":\"" + + detect_sname + + "\",\"parameters\":{\"input\":{\"keep_orig\":true," +#ifdef USE_CUDA_CV + "\"cuda\":true" +#else + "\"cuda\":false" +#endif + "},\"output\":{\"bbox\":true,\"best_bbox\":2}},\"data\":[\"" + + trt_gan_repo + + "/horse_1024.jpg\"]}," + "{\"id\":\"crop\",\"action\":{\"type\":\"crop\",\"parameters\":{" + "\"fixed_width\":360,\"fixed_height\":360}}},{\"service\":\"" + + gan_sname + + "\",\"parent_id\":\"crop\",\"parameters\":{\"input\":{" +#ifdef USE_CUDA_CV + "\"cuda\":true" +#else + "\"cuda\":false" +#endif + "},\"mllib\":{\"extract_layer\":\"last\"},\"output\":{\"image\":" + "true}}},{\"id\":\"recompose\",\"action\":{\"type\":\"recompose\"," + "\"parameters\":{\"save_img\":true,\"save_path\":\".\"}}}]}}"; + joutstr = japi.jrender(japi.service_chain("chain", jchainstr)); + std::cout << "joutstr=" << joutstr.substr(0, 500) + << (joutstr.size() > 500 + ? " ... " + joutstr.substr(joutstr.size() - 500) + : "") + << std::endl; + jd = JDoc(); + jd.Parse(joutstr.c_str()); + ASSERT_TRUE(!jd.HasParseError()); + ASSERT_EQ(200, jd["status"]["code"]); + ASSERT_TRUE(jd["body"]["predictions"].IsArray()); + ASSERT_TRUE(jd["body"]["predictions"][0]["classes"].IsArray()); + ASSERT_EQ(2, jd["body"]["predictions"][0]["classes"].Size()); + ASSERT_TRUE(jd["body"]["predictions"][0]["recompose"].IsObject()); + + auto &recompose_pred = jd["body"]["predictions"][0]["recompose"]; + ASSERT_TRUE(recompose_pred["images"].IsArray()); + jstr = "{\"clear\":\"lib\"}"; joutstr = japi.jrender(japi.service_delete(detect_sname, jstr)); ASSERT_EQ(ok_str, joutstr);