From efc1c0596e2550d6e0c134ca2d94b0e5a0fd78cb Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 14 Jul 2022 09:32:15 -0700 Subject: [PATCH 01/41] Copy srf changes as-is, need to update regexes --- .clang-format | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/.clang-format b/.clang-format index bda649666d..d9320c1d8c 100644 --- a/.clang-format +++ b/.clang-format @@ -66,20 +66,42 @@ ForEachMacros: - foreach - Q_FOREACH - BOOST_FOREACH -IncludeBlocks: Preserve +IncludeBlocks: Regroup IncludeCategories: - - Regex: '^' + # The order of the groups is + # 0 - Main include file for .cpp + # 1 - source relative files `#include "./some_header.hpp"` (Grouped with 2) + # 2 - source relative files starting with internal/public `#include "internal/some_header.hpp"` + # 3 - Python SRF public API files `#include "pysrf/srf_header.hpp"` + # 4 - SRF public API files `#include "srf/srf_header.hpp"` + # 5 - NVRPC public API files `#include "nvrpc/some_header.hpp"` + # 6 - External installed libraries `#include ` + # 7 - System includes `#include ` + # First match any Python SRF public API headers with quotes + - Regex: '^"pysrf\/.*\.(h|hpp)"' + Priority: 3 + # Next match any SRF public API headers with quotes + - Regex: '^"srf\/.*\.(h|hpp)"' + Priority: 4 + # Next match public NVRPC headers with quotes + - Regex: '^' + Priority: 5 + # Next find any headers in internal or public + - Regex: '^"(internal|public)\/.*\.(h|hpp)"' Priority: 2 - - Regex: '^<.*\.h>' + # Any other quoted includes need to be with internal/public but on top (Thats why this group is last) + - Regex: '^".*\.(h|hpp)"' Priority: 1 + # Last is system includes which dont have a '/' like or + - Regex: '<([a-z_])+>' + Priority: 7 + # Finally, put all 3rd party includes before the system includes - Regex: '^<.*' - Priority: 2 - - Regex: '.*' - Priority: 3 + Priority: 6 IncludeIsMainRegex: '([-_](test|unittest))?$' IndentCaseBlocks: false IndentCaseLabels: false -IndentPPDirectives: None +IndentPPDirectives: BeforeHash IndentWidth: 4 IndentWrappedFunctionNames: false JavaScriptQuotes: Leave @@ -148,7 +170,7 @@ SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false -Standard: Cpp11 +Standard: c++17 StatementMacros: - Q_UNUSED - QT_REQUIRE_VERSION From c220350e7186e9a66726dd505da459551ed48429 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 14 Jul 2022 15:03:35 -0700 Subject: [PATCH 02/41] Update clang-format for morpheus --- .clang-format | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/.clang-format b/.clang-format index d9320c1d8c..10b2fb7992 100644 --- a/.clang-format +++ b/.clang-format @@ -71,33 +71,21 @@ IncludeCategories: # The order of the groups is # 0 - Main include file for .cpp # 1 - source relative files `#include "./some_header.hpp"` (Grouped with 2) - # 2 - source relative files starting with internal/public `#include "internal/some_header.hpp"` - # 3 - Python SRF public API files `#include "pysrf/srf_header.hpp"` - # 4 - SRF public API files `#include "srf/srf_header.hpp"` - # 5 - NVRPC public API files `#include "nvrpc/some_header.hpp"` - # 6 - External installed libraries `#include ` - # 7 - System includes `#include ` - # First match any Python SRF public API headers with quotes - - Regex: '^"pysrf\/.*\.(h|hpp)"' - Priority: 3 - # Next match any SRF public API headers with quotes - - Regex: '^"srf\/.*\.(h|hpp)"' - Priority: 4 - # Next match public NVRPC headers with quotes - - Regex: '^' - Priority: 5 - # Next find any headers in internal or public - - Regex: '^"(internal|public)\/.*\.(h|hpp)"' + # 2 - Morpheus API files `#include "morpheus/messages/meta.hpp` + # 3 - External installed libraries `#include ` + # 4 - System includes `#include ` + # First match any Morpheus headers + - Regex: '^"morpheus\/.*\.(h|hpp)"' Priority: 2 - # Any other quoted includes need to be with internal/public but on top (Thats why this group is last) + # Any relative quoted includes need to be with internal/public but on top (Thats why this group is last) - Regex: '^".*\.(h|hpp)"' Priority: 1 # Last is system includes which dont have a '/' like or - Regex: '<([a-z_])+>' - Priority: 7 + Priority: 4 # Finally, put all 3rd party includes before the system includes - Regex: '^<.*' - Priority: 6 + Priority: 3 IncludeIsMainRegex: '([-_](test|unittest))?$' IndentCaseBlocks: false IndentCaseLabels: false From cb38ea3183988b0c4adbda1722cc6a05a1aa92c7 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 14 Jul 2022 15:07:34 -0700 Subject: [PATCH 03/41] Replace internal angle bracket includes with quoted includes. Also applies any clang-formatting rules to any files not matching --- .../_lib/include/morpheus/io/serializers.hpp | 2 +- .../messages/memory/inference_memory.hpp | 4 +- .../messages/memory/inference_memory_fil.hpp | 9 +- .../messages/memory/inference_memory_nlp.hpp | 6 +- .../messages/memory/response_memory.hpp | 4 +- .../messages/memory/response_memory_probs.hpp | 7 +- .../_lib/include/morpheus/messages/meta.hpp | 137 +++++----- .../_lib/include/morpheus/messages/multi.hpp | 9 +- .../morpheus/messages/multi_inference.hpp | 12 +- .../morpheus/messages/multi_inference_fil.hpp | 6 +- .../morpheus/messages/multi_inference_nlp.hpp | 7 +- .../morpheus/messages/multi_response.hpp | 15 +- .../messages/multi_response_probs.hpp | 14 +- .../include/morpheus/objects/dev_mem_info.hpp | 2 +- .../morpheus/objects/python_data_table.hpp | 53 ++-- .../include/morpheus/objects/rmm_tensor.hpp | 5 +- .../include/morpheus/objects/table_info.hpp | 4 +- .../_lib/include/morpheus/objects/tensor.hpp | 17 +- .../morpheus/objects/tensor_object.hpp | 12 +- .../morpheus/objects/triton_in_out.hpp | 2 +- .../morpheus/objects/wrapped_tensor.hpp | 3 +- .../morpheus/stages/add_classification.hpp | 2 +- .../include/morpheus/stages/add_scores.hpp | 2 +- .../morpheus/stages/deserialization.hpp | 4 +- .../include/morpheus/stages/file_source.hpp | 2 +- .../morpheus/stages/filter_detection.hpp | 2 +- .../include/morpheus/stages/kafka_source.hpp | 7 +- .../morpheus/stages/preprocess_fil.hpp | 4 +- .../morpheus/stages/preprocess_nlp.hpp | 4 +- .../include/morpheus/stages/serialize.hpp | 2 +- .../morpheus/stages/triton_inference.hpp | 9 +- .../include/morpheus/stages/write_to_file.hpp | 8 +- .../include/morpheus/utilities/cudf_util.hpp | 42 ++- .../include/morpheus/utilities/cupy_util.hpp | 2 +- .../include/morpheus/utilities/matx_util.hpp | 8 +- .../morpheus/utilities/tensor_util.hpp | 2 +- .../include/morpheus/utilities/type_util.hpp | 2 +- morpheus/_lib/src/io/serializers.cpp | 6 +- .../src/messages/memory/inference_memory.cpp | 26 +- .../messages/memory/inference_memory_fil.cpp | 10 +- .../messages/memory/inference_memory_nlp.cpp | 9 +- .../src/messages/memory/response_memory.cpp | 8 +- .../messages/memory/response_memory_probs.cpp | 10 +- morpheus/_lib/src/messages/meta.cpp | 141 +++++----- morpheus/_lib/src/messages/multi.cpp | 6 +- .../_lib/src/messages/multi_inference.cpp | 14 +- .../_lib/src/messages/multi_inference_fil.cpp | 10 +- .../_lib/src/messages/multi_inference_nlp.cpp | 13 +- morpheus/_lib/src/messages/multi_response.cpp | 13 +- .../src/messages/multi_response_probs.cpp | 10 +- morpheus/_lib/src/objects/dev_mem_info.cpp | 13 +- morpheus/_lib/src/objects/fiber_queue.cpp | 241 ++++++++++-------- morpheus/_lib/src/objects/file_types.cpp | 4 +- .../_lib/src/objects/python_data_table.cpp | 52 ++-- morpheus/_lib/src/objects/rmm_tensor.cpp | 10 +- morpheus/_lib/src/objects/table_info.cpp | 5 +- morpheus/_lib/src/objects/tensor.cpp | 8 +- morpheus/_lib/src/objects/tensor_object.cpp | 4 +- morpheus/_lib/src/objects/wrapped_tensor.cpp | 4 +- morpheus/_lib/src/python_modules/common.cpp | 6 +- .../_lib/src/python_modules/file_types.cpp | 5 +- morpheus/_lib/src/python_modules/messages.cpp | 31 ++- morpheus/_lib/src/python_modules/stages.cpp | 24 +- .../_lib/src/stages/add_classification.cpp | 4 +- morpheus/_lib/src/stages/add_scores.cpp | 4 +- morpheus/_lib/src/stages/deserialize.cpp | 2 +- morpheus/_lib/src/stages/file_source.cpp | 10 +- morpheus/_lib/src/stages/filter_detection.cpp | 4 +- morpheus/_lib/src/stages/kafka_source.cpp | 21 +- morpheus/_lib/src/stages/preprocess_fil.cpp | 23 +- morpheus/_lib/src/stages/preprocess_nlp.cpp | 13 +- morpheus/_lib/src/stages/serialize.cpp | 2 +- morpheus/_lib/src/stages/triton_inference.cpp | 15 +- morpheus/_lib/src/stages/write_to_file.cpp | 4 +- morpheus/_lib/src/utilities/cudf_util.cpp | 6 +- morpheus/_lib/src/utilities/cupy_util.cpp | 4 +- morpheus/_lib/src/utilities/matx_util.cu | 8 +- morpheus/_lib/src/utilities/string_util.cpp | 11 +- morpheus/_lib/src/utilities/table_util.cpp | 19 +- morpheus/_lib/src/utilities/tensor_util.cpp | 9 +- morpheus/_lib/src/utilities/type_util.cu | 2 +- .../_lib/src/utilities/type_util_detail.cpp | 4 +- morpheus/_lib/tests/test_cuda.cu | 2 +- morpheus/_lib/tests/test_type_util_detail.cpp | 2 +- 84 files changed, 648 insertions(+), 636 deletions(-) diff --git a/morpheus/_lib/include/morpheus/io/serializers.hpp b/morpheus/_lib/include/morpheus/io/serializers.hpp index e5b297b896..de5c7cb83a 100644 --- a/morpheus/_lib/include/morpheus/io/serializers.hpp +++ b/morpheus/_lib/include/morpheus/io/serializers.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/objects/table_info.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/messages/memory/inference_memory.hpp b/morpheus/_lib/include/morpheus/messages/memory/inference_memory.hpp index c1be524e73..a8bd3e08f2 100644 --- a/morpheus/_lib/include/morpheus/messages/memory/inference_memory.hpp +++ b/morpheus/_lib/include/morpheus/messages/memory/inference_memory.hpp @@ -17,8 +17,8 @@ #pragma once -#include -#include +#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/messages/memory/inference_memory_fil.hpp b/morpheus/_lib/include/morpheus/messages/memory/inference_memory_fil.hpp index c7cf6f3833..73618942e9 100644 --- a/morpheus/_lib/include/morpheus/messages/memory/inference_memory_fil.hpp +++ b/morpheus/_lib/include/morpheus/messages/memory/inference_memory_fil.hpp @@ -17,13 +17,12 @@ #pragma once -#include -#include -#include -#include +#include "morpheus/messages/memory/inference_memory.hpp" +#include "morpheus/objects/python_data_table.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" #include - #include #include diff --git a/morpheus/_lib/include/morpheus/messages/memory/inference_memory_nlp.hpp b/morpheus/_lib/include/morpheus/messages/memory/inference_memory_nlp.hpp index 282b0c2c91..eb3713903f 100644 --- a/morpheus/_lib/include/morpheus/messages/memory/inference_memory_nlp.hpp +++ b/morpheus/_lib/include/morpheus/messages/memory/inference_memory_nlp.hpp @@ -17,9 +17,9 @@ #pragma once -#include -#include -#include +#include "morpheus/messages/memory/inference_memory.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" #include diff --git a/morpheus/_lib/include/morpheus/messages/memory/response_memory.hpp b/morpheus/_lib/include/morpheus/messages/memory/response_memory.hpp index f4736b7017..20cb00173e 100644 --- a/morpheus/_lib/include/morpheus/messages/memory/response_memory.hpp +++ b/morpheus/_lib/include/morpheus/messages/memory/response_memory.hpp @@ -17,10 +17,10 @@ #pragma once -#include +#include "morpheus/objects/tensor.hpp" -#include #include +#include #include #include diff --git a/morpheus/_lib/include/morpheus/messages/memory/response_memory_probs.hpp b/morpheus/_lib/include/morpheus/messages/memory/response_memory_probs.hpp index c726888695..765ce5479b 100644 --- a/morpheus/_lib/include/morpheus/messages/memory/response_memory_probs.hpp +++ b/morpheus/_lib/include/morpheus/messages/memory/response_memory_probs.hpp @@ -17,13 +17,12 @@ #pragma once -#include -#include -#include +#include "morpheus/messages/memory/response_memory.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" #include #include - #include #include diff --git a/morpheus/_lib/include/morpheus/messages/meta.hpp b/morpheus/_lib/include/morpheus/messages/meta.hpp index dfef434d82..2ff91ce371 100644 --- a/morpheus/_lib/include/morpheus/messages/meta.hpp +++ b/morpheus/_lib/include/morpheus/messages/meta.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/objects/table_info.hpp" #include #include @@ -26,76 +26,77 @@ #include namespace morpheus { - /****** Component public implementations ******************/ - /****** MessageMeta****************************************/ +/****** Component public implementations ******************/ +/****** MessageMeta****************************************/ +/** + * TODO(Documentation) + */ +#pragma GCC visibility push(default) +class MessageMeta +{ + public: /** * TODO(Documentation) */ -#pragma GCC visibility push(default) - class MessageMeta { - public: - /** - * TODO(Documentation) - */ - pybind11::object get_py_table() const; - - /** - * TODO(Documentation) - */ - size_t count() const; - - /** - * TODO(Documentation) - */ - TableInfo get_info() const; - - /** - * TODO(Documentation) - */ - static std::shared_ptr create_from_python(pybind11::object &&data_table); - - /** - * TODO(Documentation) - */ - static std::shared_ptr create_from_cpp(cudf::io::table_with_metadata &&data_table, - int index_col_count = 0); - - private: - MessageMeta(std::shared_ptr data); - - /** - * TODO(Documentation) - */ - static pybind11::object cpp_to_py(cudf::io::table_with_metadata &&table, int index_col_count = 0); - - std::shared_ptr m_data; - }; - - - /****** MessageMetaInterfaceProxy**************************/ + pybind11::object get_py_table() const; + + /** + * TODO(Documentation) + */ + size_t count() const; + + /** + * TODO(Documentation) + */ + TableInfo get_info() const; + + /** + * TODO(Documentation) + */ + static std::shared_ptr create_from_python(pybind11::object&& data_table); + + /** + * TODO(Documentation) + */ + static std::shared_ptr create_from_cpp(cudf::io::table_with_metadata&& data_table, + int index_col_count = 0); + + private: + MessageMeta(std::shared_ptr data); + /** - * @brief Interface proxy, used to insulate python bindings. + * TODO(Documentation) + */ + static pybind11::object cpp_to_py(cudf::io::table_with_metadata&& table, int index_col_count = 0); + + std::shared_ptr m_data; +}; + +/****** MessageMetaInterfaceProxy**************************/ +/** + * @brief Interface proxy, used to insulate python bindings. + */ +struct MessageMetaInterfaceProxy +{ + /** + * TODO(Documentation) + */ + static std::shared_ptr init_cpp(const std::string& filename); + + /** + * TODO(Documentation) + */ + static std::shared_ptr init_python(pybind11::object&& data_frame); + + /** + * TODO(Documentation) + */ + static cudf::size_type count(MessageMeta& self); + + /** + * TODO(Documentation) */ - struct MessageMetaInterfaceProxy { - /** - * TODO(Documentation) - */ - static std::shared_ptr init_cpp(const std::string& filename); - - /** - * TODO(Documentation) - */ - static std::shared_ptr init_python(pybind11::object&& data_frame); - - /** - * TODO(Documentation) - */ - static cudf::size_type count(MessageMeta& self); - - /** - * TODO(Documentation) - */ - static pybind11::object get_data_frame(MessageMeta& self); - }; + static pybind11::object get_data_frame(MessageMeta& self); +}; #pragma GCC visibility pop -} \ No newline at end of file +} // namespace morpheus diff --git a/morpheus/_lib/include/morpheus/messages/multi.hpp b/morpheus/_lib/include/morpheus/messages/multi.hpp index 341aaad19d..1216a906d5 100644 --- a/morpheus/_lib/include/morpheus/messages/multi.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi.hpp @@ -17,16 +17,15 @@ #pragma once -#include -#include -#include -#include +#include "morpheus/messages/meta.hpp" +#include "morpheus/objects/table_info.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" #include #include #include #include - #include #include #include diff --git a/morpheus/_lib/include/morpheus/messages/multi_inference.hpp b/morpheus/_lib/include/morpheus/messages/multi_inference.hpp index eee29f3108..0a6e1bf655 100644 --- a/morpheus/_lib/include/morpheus/messages/multi_inference.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi_inference.hpp @@ -17,14 +17,14 @@ #pragma once -#include -#include -#include -#include -#include +#include "morpheus/messages/memory/inference_memory.hpp" +#include "morpheus/messages/meta.hpp" +#include "morpheus/messages/multi.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" -#include #include +#include #include #include diff --git a/morpheus/_lib/include/morpheus/messages/multi_inference_fil.hpp b/morpheus/_lib/include/morpheus/messages/multi_inference_fil.hpp index 7fe7507bbc..7468d82a32 100644 --- a/morpheus/_lib/include/morpheus/messages/multi_inference_fil.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi_inference_fil.hpp @@ -17,9 +17,9 @@ #pragma once -#include -#include -#include +#include "morpheus/messages/multi_inference.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/messages/multi_inference_nlp.hpp b/morpheus/_lib/include/morpheus/messages/multi_inference_nlp.hpp index fa3649bb60..4d09cef3dd 100644 --- a/morpheus/_lib/include/morpheus/messages/multi_inference_nlp.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi_inference_nlp.hpp @@ -17,12 +17,11 @@ #pragma once -#include -#include -#include +#include "morpheus/messages/multi_inference.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" #include - #include #include diff --git a/morpheus/_lib/include/morpheus/messages/multi_response.hpp b/morpheus/_lib/include/morpheus/messages/multi_response.hpp index 41ad40b078..e8ea8c6e7c 100644 --- a/morpheus/_lib/include/morpheus/messages/multi_response.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi_response.hpp @@ -17,16 +17,15 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include +#include "morpheus/messages/memory/response_memory.hpp" +#include "morpheus/messages/meta.hpp" +#include "morpheus/messages/multi.hpp" +#include "morpheus/objects/table_info.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" +#include "morpheus/utilities/table_util.hpp" #include - #include #include diff --git a/morpheus/_lib/include/morpheus/messages/multi_response_probs.hpp b/morpheus/_lib/include/morpheus/messages/multi_response_probs.hpp index 7c64dae511..8c0271b053 100644 --- a/morpheus/_lib/include/morpheus/messages/multi_response_probs.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi_response_probs.hpp @@ -17,15 +17,15 @@ #pragma once -#include -#include -#include -#include -#include -#include +#include "morpheus/messages/memory/response_memory.hpp" +#include "morpheus/messages/meta.hpp" +#include "morpheus/messages/multi.hpp" +#include "morpheus/messages/multi_response.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" -#include #include +#include #include diff --git a/morpheus/_lib/include/morpheus/objects/dev_mem_info.hpp b/morpheus/_lib/include/morpheus/objects/dev_mem_info.hpp index 9b37e95d33..e72d86680c 100644 --- a/morpheus/_lib/include/morpheus/objects/dev_mem_info.hpp +++ b/morpheus/_lib/include/morpheus/objects/dev_mem_info.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/utilities/type_util_detail.hpp" #include diff --git a/morpheus/_lib/include/morpheus/objects/python_data_table.hpp b/morpheus/_lib/include/morpheus/objects/python_data_table.hpp index de9c7e515b..bbe395b285 100644 --- a/morpheus/_lib/include/morpheus/objects/python_data_table.hpp +++ b/morpheus/_lib/include/morpheus/objects/python_data_table.hpp @@ -17,36 +17,37 @@ #pragma once -#include +#include "morpheus/objects/table_info.hpp" #include namespace morpheus { - /****** Component public implementations *******************/ - /****** PyDataTable****************************************/ +/****** Component public implementations *******************/ +/****** PyDataTable****************************************/ +/** + * TODO(Documentation) + */ +struct PyDataTable : public IDataTable +{ + PyDataTable(pybind11::object &&py_table); + ~PyDataTable(); + + /** + * TODO(Documentation) + */ + cudf::size_type count() const override; + + /** + * TODO(Documentation) + */ + TableInfo get_info() const override; + /** * TODO(Documentation) */ - struct PyDataTable : public IDataTable { - PyDataTable(pybind11::object &&py_table); - ~PyDataTable(); - - /** - * TODO(Documentation) - */ - cudf::size_type count() const override; - - /** - * TODO(Documentation) - */ - TableInfo get_info() const override; - - /** - * TODO(Documentation) - */ - const pybind11::object &get_py_object() const override; - - private: - pybind11::object m_py_table; - }; -} \ No newline at end of file + const pybind11::object &get_py_object() const override; + + private: + pybind11::object m_py_table; +}; +} // namespace morpheus diff --git a/morpheus/_lib/include/morpheus/objects/rmm_tensor.hpp b/morpheus/_lib/include/morpheus/objects/rmm_tensor.hpp index 2f31640715..b3b96448a8 100644 --- a/morpheus/_lib/include/morpheus/objects/rmm_tensor.hpp +++ b/morpheus/_lib/include/morpheus/objects/rmm_tensor.hpp @@ -17,11 +17,10 @@ #pragma once -#include -#include +#include "morpheus/objects/tensor_object.hpp" +#include "morpheus/utilities/type_util.hpp" #include - #include #include diff --git a/morpheus/_lib/include/morpheus/objects/table_info.hpp b/morpheus/_lib/include/morpheus/objects/table_info.hpp index 627a9ba98d..14329af4e3 100644 --- a/morpheus/_lib/include/morpheus/objects/table_info.hpp +++ b/morpheus/_lib/include/morpheus/objects/table_info.hpp @@ -17,8 +17,8 @@ #pragma once -#include -#include +#include "morpheus/objects/data_table.hpp" +#include "morpheus/utilities/type_util_detail.hpp" #include diff --git a/morpheus/_lib/include/morpheus/objects/tensor.hpp b/morpheus/_lib/include/morpheus/objects/tensor.hpp index 038c110510..ee73ce3f3e 100644 --- a/morpheus/_lib/include/morpheus/objects/tensor.hpp +++ b/morpheus/_lib/include/morpheus/objects/tensor.hpp @@ -17,20 +17,17 @@ #pragma once -#include -#include -#include -#include - -#include +#include "morpheus/objects/rmm_tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" +#include "morpheus/utilities/matx_util.hpp" +#include "morpheus/utilities/type_util.hpp" #include - -#include -#include - #include #include +#include +#include +#include #include #include diff --git a/morpheus/_lib/include/morpheus/objects/tensor_object.hpp b/morpheus/_lib/include/morpheus/objects/tensor_object.hpp index 8d26f4ee90..68faf6b28b 100644 --- a/morpheus/_lib/include/morpheus/objects/tensor_object.hpp +++ b/morpheus/_lib/include/morpheus/objects/tensor_object.hpp @@ -17,19 +17,17 @@ #pragma once -#include - -#include +#include "morpheus/utilities/string_util.hpp" +#include "morpheus/utilities/type_util_detail.hpp" +#include // for cudaMemcpyDeviceToHost & cudaMemcpy +#include // for CHECK +#include #include #include #include #include // for memory_kind_type -#include // for cudaMemcpyDeviceToHost & cudaMemcpy -#include // for CHECK -#include - #include #include #include // for size_t, byte diff --git a/morpheus/_lib/include/morpheus/objects/triton_in_out.hpp b/morpheus/_lib/include/morpheus/objects/triton_in_out.hpp index de6c0dd8d6..eddbc63b87 100644 --- a/morpheus/_lib/include/morpheus/objects/triton_in_out.hpp +++ b/morpheus/_lib/include/morpheus/objects/triton_in_out.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/utilities/type_util.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/objects/wrapped_tensor.hpp b/morpheus/_lib/include/morpheus/objects/wrapped_tensor.hpp index d76a82c803..fa845dbe38 100644 --- a/morpheus/_lib/include/morpheus/objects/wrapped_tensor.hpp +++ b/morpheus/_lib/include/morpheus/objects/wrapped_tensor.hpp @@ -18,11 +18,10 @@ #pragma once -#include +#include "morpheus/objects/tensor_object.hpp" #include #include - #include #include diff --git a/morpheus/_lib/include/morpheus/stages/add_classification.hpp b/morpheus/_lib/include/morpheus/stages/add_classification.hpp index b430314560..850ff726c4 100644 --- a/morpheus/_lib/include/morpheus/stages/add_classification.hpp +++ b/morpheus/_lib/include/morpheus/stages/add_classification.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/messages/multi_response_probs.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/stages/add_scores.hpp b/morpheus/_lib/include/morpheus/stages/add_scores.hpp index 63f1b6a23e..02ba811bab 100644 --- a/morpheus/_lib/include/morpheus/stages/add_scores.hpp +++ b/morpheus/_lib/include/morpheus/stages/add_scores.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/messages/multi_response_probs.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/stages/deserialization.hpp b/morpheus/_lib/include/morpheus/stages/deserialization.hpp index 25c1a1d46a..6b4cf821da 100644 --- a/morpheus/_lib/include/morpheus/stages/deserialization.hpp +++ b/morpheus/_lib/include/morpheus/stages/deserialization.hpp @@ -17,8 +17,8 @@ #pragma once -#include -#include +#include "morpheus/messages/meta.hpp" +#include "morpheus/messages/multi.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/stages/file_source.hpp b/morpheus/_lib/include/morpheus/stages/file_source.hpp index 6d2426cbf5..4eb27ffcfe 100644 --- a/morpheus/_lib/include/morpheus/stages/file_source.hpp +++ b/morpheus/_lib/include/morpheus/stages/file_source.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/messages/meta.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/stages/filter_detection.hpp b/morpheus/_lib/include/morpheus/stages/filter_detection.hpp index e3b1e53e0f..1ccd0ea079 100644 --- a/morpheus/_lib/include/morpheus/stages/filter_detection.hpp +++ b/morpheus/_lib/include/morpheus/stages/filter_detection.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/messages/multi_response_probs.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/stages/kafka_source.hpp b/morpheus/_lib/include/morpheus/stages/kafka_source.hpp index eb1eb8e4ee..c3370cf485 100644 --- a/morpheus/_lib/include/morpheus/stages/kafka_source.hpp +++ b/morpheus/_lib/include/morpheus/stages/kafka_source.hpp @@ -17,16 +17,15 @@ #pragma once -#include +#include "morpheus/messages/meta.hpp" +#include +#include #include #include #include #include -#include -#include - #include #include #include diff --git a/morpheus/_lib/include/morpheus/stages/preprocess_fil.hpp b/morpheus/_lib/include/morpheus/stages/preprocess_fil.hpp index 481a47a4c4..6b19073ba1 100644 --- a/morpheus/_lib/include/morpheus/stages/preprocess_fil.hpp +++ b/morpheus/_lib/include/morpheus/stages/preprocess_fil.hpp @@ -17,8 +17,8 @@ #pragma once -#include -#include +#include "morpheus/messages/multi.hpp" +#include "morpheus/messages/multi_inference.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/stages/preprocess_nlp.hpp b/morpheus/_lib/include/morpheus/stages/preprocess_nlp.hpp index 7b0c6c2682..977dadb1b9 100644 --- a/morpheus/_lib/include/morpheus/stages/preprocess_nlp.hpp +++ b/morpheus/_lib/include/morpheus/stages/preprocess_nlp.hpp @@ -17,8 +17,8 @@ #pragma once -#include -#include +#include "morpheus/messages/multi.hpp" +#include "morpheus/messages/multi_inference.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/stages/serialize.hpp b/morpheus/_lib/include/morpheus/stages/serialize.hpp index 0072645b48..038ced9ce1 100644 --- a/morpheus/_lib/include/morpheus/stages/serialize.hpp +++ b/morpheus/_lib/include/morpheus/stages/serialize.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/messages/multi.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/stages/triton_inference.hpp b/morpheus/_lib/include/morpheus/stages/triton_inference.hpp index fe1c5db068..14d99d338a 100644 --- a/morpheus/_lib/include/morpheus/stages/triton_inference.hpp +++ b/morpheus/_lib/include/morpheus/stages/triton_inference.hpp @@ -17,15 +17,14 @@ #pragma once -#include -#include -#include +#include "morpheus/messages/multi_inference.hpp" +#include "morpheus/messages/multi_response_probs.hpp" +#include "morpheus/objects/triton_in_out.hpp" +#include #include #include -#include - #include #include diff --git a/morpheus/_lib/include/morpheus/stages/write_to_file.hpp b/morpheus/_lib/include/morpheus/stages/write_to_file.hpp index 71b1b9b12f..e0bb5a8598 100644 --- a/morpheus/_lib/include/morpheus/stages/write_to_file.hpp +++ b/morpheus/_lib/include/morpheus/stages/write_to_file.hpp @@ -17,10 +17,10 @@ #pragma once -#include -#include -#include -#include +#include "morpheus/io/serializers.hpp" +#include "morpheus/messages/meta.hpp" +#include "morpheus/objects/file_types.hpp" +#include "morpheus/utilities/string_util.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp b/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp index 037309cb88..4c8cada7b1 100644 --- a/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp +++ b/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp @@ -17,36 +17,34 @@ #pragma once -#include +#include "morpheus/objects/table_info.hpp" #include - #include - namespace morpheus { - /****** Component public free function implementations******/ - /** - * TODO(Documentation) - */ +/****** Component public free function implementations******/ +/** + * TODO(Documentation) + */ #pragma GCC visibility push(default) - void load_cudf_helpers(); +void load_cudf_helpers(); #pragma GCC visibility pop - /** - * @brief These proxy functions allow us to have a shared set of cudf_helpers interfaces declarations, which proxy - * the actual generated cython calls. The cython implementation in 'cudf_helpers_api.h' can only appear in the - * translation unit for the pybind module declaration. - */ - pybind11::object proxy_table_from_table_with_metadata(cudf::io::table_with_metadata&&, int); - TableInfo proxy_table_info_from_table(pybind11::object table, std::shared_ptr idata_table); +/** + * @brief These proxy functions allow us to have a shared set of cudf_helpers interfaces declarations, which proxy + * the actual generated cython calls. The cython implementation in 'cudf_helpers_api.h' can only appear in the + * translation unit for the pybind module declaration. + */ +pybind11::object proxy_table_from_table_with_metadata(cudf::io::table_with_metadata &&, int); +TableInfo proxy_table_info_from_table(pybind11::object table, std::shared_ptr idata_table); - /** - * @brief cudf_helper stubs -- currently not used anywhere - */ - pybind11::object /*PyColumn*/ proxy_column_from_view(cudf::column_view view); - cudf::column_view proxy_view_from_column(pybind11::object *column /*PyColumn**/); - pybind11::object /*PyTable*/ proxy_table_from_table_info(morpheus::TableInfo table_info, pybind11::object *object); - pybind11::object /*PyTable*/ proxy_series_from_table_info(morpheus::TableInfo table_info, pybind11::object *object); +/** + * @brief cudf_helper stubs -- currently not used anywhere + */ +pybind11::object /*PyColumn*/ proxy_column_from_view(cudf::column_view view); +cudf::column_view proxy_view_from_column(pybind11::object *column /*PyColumn**/); +pybind11::object /*PyTable*/ proxy_table_from_table_info(morpheus::TableInfo table_info, pybind11::object *object); +pybind11::object /*PyTable*/ proxy_series_from_table_info(morpheus::TableInfo table_info, pybind11::object *object); } // namespace morpheus diff --git a/morpheus/_lib/include/morpheus/utilities/cupy_util.hpp b/morpheus/_lib/include/morpheus/utilities/cupy_util.hpp index 058901a1fa..630f7902c4 100644 --- a/morpheus/_lib/include/morpheus/utilities/cupy_util.hpp +++ b/morpheus/_lib/include/morpheus/utilities/cupy_util.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/objects/tensor_object.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/utilities/matx_util.hpp b/morpheus/_lib/include/morpheus/utilities/matx_util.hpp index 12f2050d69..05e559ba27 100644 --- a/morpheus/_lib/include/morpheus/utilities/matx_util.hpp +++ b/morpheus/_lib/include/morpheus/utilities/matx_util.hpp @@ -17,10 +17,10 @@ #pragma once -#include -#include -#include -#include +#include "morpheus/objects/dev_mem_info.hpp" +#include "morpheus/objects/rmm_tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" +#include "morpheus/utilities/type_util_detail.hpp" #include #include diff --git a/morpheus/_lib/include/morpheus/utilities/tensor_util.hpp b/morpheus/_lib/include/morpheus/utilities/tensor_util.hpp index 0574ac5a0f..f523ea61fd 100644 --- a/morpheus/_lib/include/morpheus/utilities/tensor_util.hpp +++ b/morpheus/_lib/include/morpheus/utilities/tensor_util.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/objects/tensor_object.hpp" #include // for ostream #include // for string diff --git a/morpheus/_lib/include/morpheus/utilities/type_util.hpp b/morpheus/_lib/include/morpheus/utilities/type_util.hpp index 501577e8e8..3cde701579 100644 --- a/morpheus/_lib/include/morpheus/utilities/type_util.hpp +++ b/morpheus/_lib/include/morpheus/utilities/type_util.hpp @@ -17,7 +17,7 @@ #pragma once -#include +#include "morpheus/utilities/type_util_detail.hpp" #include #include diff --git a/morpheus/_lib/src/io/serializers.cpp b/morpheus/_lib/src/io/serializers.cpp index 51781dc8bd..3a5ceeeb55 100644 --- a/morpheus/_lib/src/io/serializers.cpp +++ b/morpheus/_lib/src/io/serializers.cpp @@ -15,13 +15,13 @@ * limitations under the License. */ -#include +#include "morpheus/io/serializers.hpp" +#include +#include #include #include #include -#include -#include #include #include diff --git a/morpheus/_lib/src/messages/memory/inference_memory.cpp b/morpheus/_lib/src/messages/memory/inference_memory.cpp index 5df5132c58..4012924546 100644 --- a/morpheus/_lib/src/messages/memory/inference_memory.cpp +++ b/morpheus/_lib/src/messages/memory/inference_memory.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include +#include "morpheus/messages/memory/inference_memory.hpp" #include @@ -23,16 +23,18 @@ #include namespace morpheus { - /****** Component public implementations *******************/ - /****** InferenceMemory****************************************/ - InferenceMemory::InferenceMemory(size_t count) : count(count) {} +/****** Component public implementations *******************/ +/****** InferenceMemory****************************************/ +InferenceMemory::InferenceMemory(size_t count) : count(count) {} - bool InferenceMemory::has_input(const std::string &name) const { - return this->inputs.find(name) != this->inputs.end(); - } +bool InferenceMemory::has_input(const std::string& name) const +{ + return this->inputs.find(name) != this->inputs.end(); +} - /****** InferenceMemoryInterfaceProxy *************************/ - std::size_t InferenceMemoryInterfaceProxy::get_count(InferenceMemory& self) { - return self.count; - } -} \ No newline at end of file +/****** InferenceMemoryInterfaceProxy *************************/ +std::size_t InferenceMemoryInterfaceProxy::get_count(InferenceMemory& self) +{ + return self.count; +} +} // namespace morpheus diff --git a/morpheus/_lib/src/messages/memory/inference_memory_fil.cpp b/morpheus/_lib/src/messages/memory/inference_memory_fil.cpp index 37fc21407f..6102724e0b 100644 --- a/morpheus/_lib/src/messages/memory/inference_memory_fil.cpp +++ b/morpheus/_lib/src/messages/memory/inference_memory_fil.cpp @@ -15,14 +15,14 @@ * limitations under the License. */ -#include -#include -#include -#include +#include "morpheus/messages/memory/inference_memory_fil.hpp" + +#include "morpheus/messages/memory/inference_memory.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/utilities/cupy_util.hpp" #include #include - #include #include diff --git a/morpheus/_lib/src/messages/memory/inference_memory_nlp.cpp b/morpheus/_lib/src/messages/memory/inference_memory_nlp.cpp index d262f7c45b..07cf2e264b 100644 --- a/morpheus/_lib/src/messages/memory/inference_memory_nlp.cpp +++ b/morpheus/_lib/src/messages/memory/inference_memory_nlp.cpp @@ -15,10 +15,11 @@ * limitations under the License. */ -#include -#include -#include -#include +#include "morpheus/messages/memory/inference_memory_nlp.hpp" + +#include "morpheus/messages/memory/inference_memory.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/utilities/cupy_util.hpp" #include #include diff --git a/morpheus/_lib/src/messages/memory/response_memory.cpp b/morpheus/_lib/src/messages/memory/response_memory.cpp index a7d027e92c..db04c08eda 100644 --- a/morpheus/_lib/src/messages/memory/response_memory.cpp +++ b/morpheus/_lib/src/messages/memory/response_memory.cpp @@ -15,12 +15,12 @@ * limitations under the License. */ -#include -#include -#include +#include "morpheus/messages/memory/response_memory.hpp" -#include +#include "morpheus/objects/tensor.hpp" +#include "morpheus/utilities/cupy_util.hpp" +#include #include #include diff --git a/morpheus/_lib/src/messages/memory/response_memory_probs.cpp b/morpheus/_lib/src/messages/memory/response_memory_probs.cpp index 617fdfcee4..2efcd99560 100644 --- a/morpheus/_lib/src/messages/memory/response_memory_probs.cpp +++ b/morpheus/_lib/src/messages/memory/response_memory_probs.cpp @@ -15,14 +15,14 @@ * limitations under the License. */ -#include -#include -#include -#include +#include "morpheus/messages/memory/response_memory_probs.hpp" + +#include "morpheus/messages/memory/response_memory.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/utilities/cupy_util.hpp" #include #include - #include #include diff --git a/morpheus/_lib/src/messages/meta.cpp b/morpheus/_lib/src/messages/meta.cpp index 3b2dc53f13..5ae9a7aebb 100644 --- a/morpheus/_lib/src/messages/meta.cpp +++ b/morpheus/_lib/src/messages/meta.cpp @@ -15,15 +15,14 @@ * limitations under the License. */ -#include +#include "morpheus/messages/meta.hpp" -#include -#include -#include -#include +#include "morpheus/objects/python_data_table.hpp" +#include "morpheus/objects/table_info.hpp" +#include "morpheus/utilities/cudf_util.hpp" +#include "morpheus/utilities/table_util.hpp" #include - #include #include @@ -93,78 +92,88 @@ struct MessageMetaImpl { /****** Component public implementations *******************/ /****** MessageMeta ****************************************/ - pybind11::object MessageMeta::get_py_table() const { - return m_data->get_py_object(); - } +pybind11::object MessageMeta::get_py_table() const +{ + return m_data->get_py_object(); +} - size_t MessageMeta::count() const { - return m_data->count(); - } +size_t MessageMeta::count() const +{ + return m_data->count(); +} - TableInfo MessageMeta::get_info() const { - return this->m_data->get_info(); - } +TableInfo MessageMeta::get_info() const +{ + return this->m_data->get_info(); +} - std::shared_ptr MessageMeta::create_from_python(pybind11::object &&data_table) { - auto data = std::make_unique(std::move(data_table)); +std::shared_ptr MessageMeta::create_from_python(pybind11::object &&data_table) +{ + auto data = std::make_unique(std::move(data_table)); - return std::shared_ptr(new MessageMeta(std::move(data))); - } + return std::shared_ptr(new MessageMeta(std::move(data))); +} - std::shared_ptr MessageMeta::create_from_cpp(cudf::io::table_with_metadata &&data_table, - int index_col_count) { - // Convert to py first - pybind11::object py_dt = cpp_to_py(std::move(data_table), index_col_count); +std::shared_ptr MessageMeta::create_from_cpp(cudf::io::table_with_metadata &&data_table, + int index_col_count) +{ + // Convert to py first + pybind11::object py_dt = cpp_to_py(std::move(data_table), index_col_count); - auto data = std::make_unique(std::move(py_dt)); + auto data = std::make_unique(std::move(py_dt)); - return std::shared_ptr(new MessageMeta(std::move(data))); - } + return std::shared_ptr(new MessageMeta(std::move(data))); +} - MessageMeta::MessageMeta(std::shared_ptr data) : m_data(std::move(data)) {} +MessageMeta::MessageMeta(std::shared_ptr data) : m_data(std::move(data)) {} - pybind11::object MessageMeta::cpp_to_py(cudf::io::table_with_metadata &&table, int index_col_count) { - pybind11::gil_scoped_acquire gil; +pybind11::object MessageMeta::cpp_to_py(cudf::io::table_with_metadata &&table, int index_col_count) +{ + pybind11::gil_scoped_acquire gil; - // Now convert to a python TableInfo object - auto converted_table = proxy_table_from_table_with_metadata(std::move(table), index_col_count); + // Now convert to a python TableInfo object + auto converted_table = proxy_table_from_table_with_metadata(std::move(table), index_col_count); - // VLOG(10) << "Table. Num Col: " << converted_table.attr("_num_columns").str().cast() - // << ", Num Ind: " << converted_table.attr("_num_columns").cast() - // << ", Rows: " << converted_table.attr("_num_rows").cast(); - // pybind11::print("Table Created. Num Rows: {}, Num Cols: {}, Num Ind: {}", - // converted_table.attr("_num_rows"), - // converted_table.attr("_num_columns"), - // converted_table.attr("_num_indices")); + // VLOG(10) << "Table. Num Col: " << converted_table.attr("_num_columns").str().cast() + // << ", Num Ind: " << converted_table.attr("_num_columns").cast() + // << ", Rows: " << converted_table.attr("_num_rows").cast(); + // pybind11::print("Table Created. Num Rows: {}, Num Cols: {}, Num Ind: {}", + // converted_table.attr("_num_rows"), + // converted_table.attr("_num_columns"), + // converted_table.attr("_num_indices")); - return converted_table; - } + return converted_table; +} /********** MessageMetaInterfaceProxy **********/ - std::shared_ptr MessageMetaInterfaceProxy::init_python(pybind11::object &&data_frame) { - return MessageMeta::create_from_python(std::move(data_frame)); - } - - cudf::size_type MessageMetaInterfaceProxy::count(MessageMeta &self) { - return self.count(); - } - - pybind11::object MessageMetaInterfaceProxy::get_data_frame(MessageMeta &self) { - // // Get the column and convert to cudf - // auto py_table_struct = make_table_from_view_and_meta(self.m_pydf;.tbl->view(), - // self.m_pydf;.metadata); py::object py_table = - // py::reinterpret_steal((PyObject*)py_table_struct); - - // // py_col.inc_ref(); - - // return py_table; - return self.get_py_table(); - } - - std::shared_ptr MessageMetaInterfaceProxy::init_cpp(const std::string &filename) { - // Load the file - auto df_with_meta = CuDFTableUtil::load_table(filename); - - return MessageMeta::create_from_cpp(std::move(df_with_meta)); - } +std::shared_ptr MessageMetaInterfaceProxy::init_python(pybind11::object &&data_frame) +{ + return MessageMeta::create_from_python(std::move(data_frame)); +} + +cudf::size_type MessageMetaInterfaceProxy::count(MessageMeta &self) +{ + return self.count(); +} + +pybind11::object MessageMetaInterfaceProxy::get_data_frame(MessageMeta &self) +{ + // // Get the column and convert to cudf + // auto py_table_struct = make_table_from_view_and_meta(self.m_pydf;.tbl->view(), + // self.m_pydf;.metadata); py::object py_table = + // py::reinterpret_steal((PyObject*)py_table_struct); + + // // py_col.inc_ref(); + + // return py_table; + return self.get_py_table(); +} + +std::shared_ptr MessageMetaInterfaceProxy::init_cpp(const std::string &filename) +{ + // Load the file + auto df_with_meta = CuDFTableUtil::load_table(filename); + + return MessageMeta::create_from_cpp(std::move(df_with_meta)); +} } // namespace morpheus diff --git a/morpheus/_lib/src/messages/multi.cpp b/morpheus/_lib/src/messages/multi.cpp index 61e1d7a24b..757d855e04 100644 --- a/morpheus/_lib/src/messages/multi.cpp +++ b/morpheus/_lib/src/messages/multi.cpp @@ -15,10 +15,10 @@ * limitations under the License. */ -#include +#include "morpheus/messages/multi.hpp" -#include -#include +#include "morpheus/messages/meta.hpp" +#include "morpheus/objects/table_info.hpp" #include diff --git a/morpheus/_lib/src/messages/multi_inference.cpp b/morpheus/_lib/src/messages/multi_inference.cpp index 022b80d114..5d0489ca2c 100644 --- a/morpheus/_lib/src/messages/multi_inference.cpp +++ b/morpheus/_lib/src/messages/multi_inference.cpp @@ -15,16 +15,16 @@ * limitations under the License. */ -#include +#include "morpheus/messages/multi_inference.hpp" -#include -#include -#include -#include -#include +#include "morpheus/messages/memory/inference_memory.hpp" +#include "morpheus/messages/meta.hpp" +#include "morpheus/messages/multi.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/utilities/cupy_util.hpp" -#include #include +#include #include #include diff --git a/morpheus/_lib/src/messages/multi_inference_fil.cpp b/morpheus/_lib/src/messages/multi_inference_fil.cpp index 356a9d7bbd..676258a8b9 100644 --- a/morpheus/_lib/src/messages/multi_inference_fil.cpp +++ b/morpheus/_lib/src/messages/multi_inference_fil.cpp @@ -15,12 +15,12 @@ * limitations under the License. */ -#include +#include "morpheus/messages/multi_inference_fil.hpp" -#include -#include -#include -#include +#include "morpheus/messages/memory/inference_memory.hpp" +#include "morpheus/messages/meta.hpp" +#include "morpheus/messages/multi_inference.hpp" +#include "morpheus/objects/tensor.hpp" #include diff --git a/morpheus/_lib/src/messages/multi_inference_nlp.cpp b/morpheus/_lib/src/messages/multi_inference_nlp.cpp index d1b02bae4e..e737ca4e0a 100644 --- a/morpheus/_lib/src/messages/multi_inference_nlp.cpp +++ b/morpheus/_lib/src/messages/multi_inference_nlp.cpp @@ -15,16 +15,15 @@ * limitations under the License. */ -#include +#include "morpheus/messages/multi_inference_nlp.hpp" -#include -#include -#include -#include -#include +#include "morpheus/messages/memory/inference_memory.hpp" +#include "morpheus/messages/meta.hpp" +#include "morpheus/messages/multi_inference.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/utilities/cupy_util.hpp" #include - #include #include diff --git a/morpheus/_lib/src/messages/multi_response.cpp b/morpheus/_lib/src/messages/multi_response.cpp index b5f416313f..f23986dcf5 100644 --- a/morpheus/_lib/src/messages/multi_response.cpp +++ b/morpheus/_lib/src/messages/multi_response.cpp @@ -15,16 +15,15 @@ * limitations under the License. */ -#include +#include "morpheus/messages/multi_response.hpp" -#include -#include -#include -#include -#include +#include "morpheus/messages/memory/response_memory.hpp" +#include "morpheus/messages/meta.hpp" +#include "morpheus/messages/multi.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/utilities/cupy_util.hpp" #include - #include #include diff --git a/morpheus/_lib/src/messages/multi_response_probs.cpp b/morpheus/_lib/src/messages/multi_response_probs.cpp index 9c029b137f..b3fdf50c2c 100644 --- a/morpheus/_lib/src/messages/multi_response_probs.cpp +++ b/morpheus/_lib/src/messages/multi_response_probs.cpp @@ -15,14 +15,14 @@ * limitations under the License. */ -#include +#include "morpheus/messages/multi_response_probs.hpp" -#include -#include -#include +#include "morpheus/messages/meta.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/utilities/cupy_util.hpp" -#include #include +#include #include #include diff --git a/morpheus/_lib/src/objects/dev_mem_info.cpp b/morpheus/_lib/src/objects/dev_mem_info.cpp index 350799a425..08d53642aa 100644 --- a/morpheus/_lib/src/objects/dev_mem_info.cpp +++ b/morpheus/_lib/src/objects/dev_mem_info.cpp @@ -15,12 +15,13 @@ * limitations under the License. */ -#include +#include "morpheus/objects/dev_mem_info.hpp" namespace morpheus { - // Component public implementations - // ************ DevMemInfo************************* // - void *DevMemInfo::data() const { - return static_cast(buffer->data()) + offset; - } +// Component public implementations +// ************ DevMemInfo************************* // +void *DevMemInfo::data() const +{ + return static_cast(buffer->data()) + offset; } +} // namespace morpheus diff --git a/morpheus/_lib/src/objects/fiber_queue.cpp b/morpheus/_lib/src/objects/fiber_queue.cpp index 081f3cdb8a..1eca4dd748 100644 --- a/morpheus/_lib/src/objects/fiber_queue.cpp +++ b/morpheus/_lib/src/objects/fiber_queue.cpp @@ -15,160 +15,175 @@ * limitations under the License. */ -#include +#include "morpheus/objects/fiber_queue.hpp" #include - #include #include #include #include - namespace morpheus { - /****** Component public implementations *******************/ - /****** FiberQueue****************************************/ - FiberQueue::FiberQueue(size_t max_size) : m_queue(max_size) {} - - boost::fibers::channel_op_status FiberQueue::put(pybind11::object &&item, bool block, float timeout) { - if (!block) { - return m_queue.try_push(std::move(item)); - } else if (timeout > 0.0) { - return m_queue.push_wait_for( - std::move(item), - std::chrono::duration_cast(std::chrono::duration(timeout))); - } else { - // Blocking no timeout - return m_queue.push(std::move(item)); - } +/****** Component public implementations *******************/ +/****** FiberQueue****************************************/ +FiberQueue::FiberQueue(size_t max_size) : m_queue(max_size) {} + +boost::fibers::channel_op_status FiberQueue::put(pybind11::object &&item, bool block, float timeout) +{ + if (!block) + { + return m_queue.try_push(std::move(item)); } - - boost::fibers::channel_op_status FiberQueue::get(pybind11::object &item, bool block, float timeout) { - if (!block) { - return m_queue.try_pop(std::ref(item)); - } else if (timeout > 0.0) { - return m_queue.pop_wait_for( - std::ref(item), - std::chrono::duration_cast(std::chrono::duration(timeout))); - } else { - // Blocking no timeout - return m_queue.pop(std::ref(item)); - } + else if (timeout > 0.0) + { + return m_queue.push_wait_for( + std::move(item), std::chrono::duration_cast(std::chrono::duration(timeout))); } - - void FiberQueue::close() { - m_queue.close(); + else + { + // Blocking no timeout + return m_queue.push(std::move(item)); } +} - bool FiberQueue::is_closed() { - return m_queue.is_closed(); +boost::fibers::channel_op_status FiberQueue::get(pybind11::object &item, bool block, float timeout) +{ + if (!block) + { + return m_queue.try_pop(std::ref(item)); } - - void FiberQueue::join() { - // TODO(MDD): Not sure how to join a buffered channel + else if (timeout > 0.0) + { + return m_queue.pop_wait_for( + std::ref(item), std::chrono::duration_cast(std::chrono::duration(timeout))); } + else + { + // Blocking no timeout + return m_queue.pop(std::ref(item)); + } +} - /****** FiberQueueInterfaceProxy *************************/ - std::shared_ptr FiberQueueInterfaceProxy::init(std::size_t max_size) { - if (max_size < 2 || ((max_size & (max_size - 1)) != 0)) { - throw std::invalid_argument("max_size must be greater than 1 and a power of 2."); - } +void FiberQueue::close() +{ + m_queue.close(); +} - // Create a new shared_ptr - return std::make_shared(max_size); - } +bool FiberQueue::is_closed() +{ + return m_queue.is_closed(); +} - void FiberQueueInterfaceProxy::put(morpheus::FiberQueue &self, pybind11::object item, bool block, float timeout) { - boost::fibers::channel_op_status status; +void FiberQueue::join() +{ + // TODO(MDD): Not sure how to join a buffered channel +} - // Release the GIL and try to move it - { - pybind11::gil_scoped_release nogil; +/****** FiberQueueInterfaceProxy *************************/ +std::shared_ptr FiberQueueInterfaceProxy::init(std::size_t max_size) +{ + if (max_size < 2 || ((max_size & (max_size - 1)) != 0)) + { + throw std::invalid_argument("max_size must be greater than 1 and a power of 2."); + } - status = self.put(std::move(item), block, timeout); - } + // Create a new shared_ptr + return std::make_shared(max_size); +} - switch (status) { - case boost::fibers::channel_op_status::success: - return; - case boost::fibers::channel_op_status::empty: { - // Raise queue.Empty - pybind11::object exc_class = pybind11::module_::import("queue").attr("Empty"); +void FiberQueueInterfaceProxy::put(morpheus::FiberQueue &self, pybind11::object item, bool block, float timeout) +{ + boost::fibers::channel_op_status status; - PyErr_SetNone(exc_class.ptr()); + // Release the GIL and try to move it + { + pybind11::gil_scoped_release nogil; - throw pybind11::error_already_set(); - } - case boost::fibers::channel_op_status::full: - case boost::fibers::channel_op_status::timeout: { - // Raise queue.Full - pybind11::object exc_class = pybind11::module_::import("queue").attr("Empty"); + status = self.put(std::move(item), block, timeout); + } - PyErr_SetNone(exc_class.ptr()); + switch (status) + { + case boost::fibers::channel_op_status::success: + return; + case boost::fibers::channel_op_status::empty: { + // Raise queue.Empty + pybind11::object exc_class = pybind11::module_::import("queue").attr("Empty"); - throw pybind11::error_already_set(); - } - case boost::fibers::channel_op_status::closed: { - // Raise queue.Full - pybind11::object exc_class = pybind11::module_::import( - "morpheus.utils.producer_consumer_queue").attr("Closed"); + PyErr_SetNone(exc_class.ptr()); - PyErr_SetNone(exc_class.ptr()); + throw pybind11::error_already_set(); + } + case boost::fibers::channel_op_status::full: + case boost::fibers::channel_op_status::timeout: { + // Raise queue.Full + pybind11::object exc_class = pybind11::module_::import("queue").attr("Empty"); - throw pybind11::error_already_set(); - } - } + PyErr_SetNone(exc_class.ptr()); + throw pybind11::error_already_set(); } + case boost::fibers::channel_op_status::closed: { + // Raise queue.Full + pybind11::object exc_class = pybind11::module_::import("morpheus.utils.producer_consumer_queue").attr("Closed"); - pybind11::object FiberQueueInterfaceProxy::get(morpheus::FiberQueue &self, bool block, float timeout) { - boost::fibers::channel_op_status status; + PyErr_SetNone(exc_class.ptr()); - pybind11::object item; + throw pybind11::error_already_set(); + } + } +} - // Release the GIL and try to move it - { - pybind11::gil_scoped_release nogil; +pybind11::object FiberQueueInterfaceProxy::get(morpheus::FiberQueue &self, bool block, float timeout) +{ + boost::fibers::channel_op_status status; - status = self.get(std::ref(item), block, timeout); - } + pybind11::object item; - switch (status) { - case boost::fibers::channel_op_status::success: - return item; - case boost::fibers::channel_op_status::empty: { - // Raise queue.Empty - pybind11::object exc_class = pybind11::module_::import("queue").attr("Empty"); + // Release the GIL and try to move it + { + pybind11::gil_scoped_release nogil; - PyErr_SetNone(exc_class.ptr()); + status = self.get(std::ref(item), block, timeout); + } - throw pybind11::error_already_set(); - } - case boost::fibers::channel_op_status::full: - case boost::fibers::channel_op_status::timeout: { - // Raise queue.Full - pybind11::object exc_class = pybind11::module_::import("queue").attr("Empty"); + switch (status) + { + case boost::fibers::channel_op_status::success: + return item; + case boost::fibers::channel_op_status::empty: { + // Raise queue.Empty + pybind11::object exc_class = pybind11::module_::import("queue").attr("Empty"); - PyErr_SetNone(exc_class.ptr()); + PyErr_SetNone(exc_class.ptr()); - throw pybind11::error_already_set(); - } - case boost::fibers::channel_op_status::closed: { - // Raise queue.Full - pybind11::object exc_class = pybind11::module_::import( - "morpheus.utils.producer_consumer_queue").attr("Closed"); + throw pybind11::error_already_set(); + } + case boost::fibers::channel_op_status::full: + case boost::fibers::channel_op_status::timeout: { + // Raise queue.Full + pybind11::object exc_class = pybind11::module_::import("queue").attr("Empty"); - PyErr_SetNone(exc_class.ptr()); + PyErr_SetNone(exc_class.ptr()); - throw pybind11::error_already_set(); - } - default: - throw std::runtime_error("Unknown channel status"); - } + throw pybind11::error_already_set(); } + case boost::fibers::channel_op_status::closed: { + // Raise queue.Full + pybind11::object exc_class = pybind11::module_::import("morpheus.utils.producer_consumer_queue").attr("Closed"); + + PyErr_SetNone(exc_class.ptr()); - void FiberQueueInterfaceProxy::close(morpheus::FiberQueue &self) { - self.close(); + throw pybind11::error_already_set(); } + default: + throw std::runtime_error("Unknown channel status"); + } +} + +void FiberQueueInterfaceProxy::close(morpheus::FiberQueue &self) +{ + self.close(); } +} // namespace morpheus diff --git a/morpheus/_lib/src/objects/file_types.cpp b/morpheus/_lib/src/objects/file_types.cpp index f9e37d53d4..e180755613 100644 --- a/morpheus/_lib/src/objects/file_types.cpp +++ b/morpheus/_lib/src/objects/file_types.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include +#include "morpheus/objects/file_types.hpp" -#include +#include "morpheus/utilities/string_util.hpp" #include #include diff --git a/morpheus/_lib/src/objects/python_data_table.cpp b/morpheus/_lib/src/objects/python_data_table.cpp index 1cf1c752a8..12087ca685 100644 --- a/morpheus/_lib/src/objects/python_data_table.cpp +++ b/morpheus/_lib/src/objects/python_data_table.cpp @@ -15,13 +15,12 @@ * limitations under the License. */ -#include +#include "morpheus/objects/python_data_table.hpp" -#include -#include +#include "morpheus/objects/table_info.hpp" +#include "morpheus/utilities/cudf_util.hpp" #include - #include #include @@ -30,32 +29,37 @@ namespace morpheus { /****** Component public implementations *******************/ /****** PyDataTable****************************************/ - PyDataTable::PyDataTable(pybind11::object &&py_table) : m_py_table(std::move(py_table)) {} +PyDataTable::PyDataTable(pybind11::object &&py_table) : m_py_table(std::move(py_table)) {} - PyDataTable::~PyDataTable() { - if (m_py_table) { - pybind11::gil_scoped_acquire gil; +PyDataTable::~PyDataTable() +{ + if (m_py_table) + { + pybind11::gil_scoped_acquire gil; - // Clear out the python object - m_py_table = pybind11::object(); - } + // Clear out the python object + m_py_table = pybind11::object(); } +} - cudf::size_type PyDataTable::count() const { - pybind11::gil_scoped_acquire gil; - return m_py_table.attr("_num_rows").cast(); - } +cudf::size_type PyDataTable::count() const +{ + pybind11::gil_scoped_acquire gil; + return m_py_table.attr("_num_rows").cast(); +} - TableInfo PyDataTable::get_info() const { - pybind11::gil_scoped_acquire gil; +TableInfo PyDataTable::get_info() const +{ + pybind11::gil_scoped_acquire gil; - // auto info = proxy_table_info_from_table((PyTable *) m_py_table.ptr(), this->shared_from_this()); - auto info = proxy_table_info_from_table(m_py_table, this->shared_from_this()); + // auto info = proxy_table_info_from_table((PyTable *) m_py_table.ptr(), this->shared_from_this()); + auto info = proxy_table_info_from_table(m_py_table, this->shared_from_this()); - return info; - } + return info; +} - const pybind11::object &PyDataTable::get_py_object() const { - return m_py_table; - } +const pybind11::object &PyDataTable::get_py_object() const +{ + return m_py_table; +} } // namespace morpheus diff --git a/morpheus/_lib/src/objects/rmm_tensor.cpp b/morpheus/_lib/src/objects/rmm_tensor.cpp index 0895854afa..fe4a939b42 100644 --- a/morpheus/_lib/src/objects/rmm_tensor.cpp +++ b/morpheus/_lib/src/objects/rmm_tensor.cpp @@ -15,14 +15,14 @@ * limitations under the License. */ -#include +#include "morpheus/objects/rmm_tensor.hpp" -#include -#include -#include +#include "morpheus/objects/tensor_object.hpp" +#include "morpheus/utilities/matx_util.hpp" +#include "morpheus/utilities/type_util.hpp" -#include #include +#include #include #include diff --git a/morpheus/_lib/src/objects/table_info.cpp b/morpheus/_lib/src/objects/table_info.cpp index c5c819be68..0dc7ebd98f 100644 --- a/morpheus/_lib/src/objects/table_info.cpp +++ b/morpheus/_lib/src/objects/table_info.cpp @@ -15,14 +15,13 @@ * limitations under the License. */ -#include +#include "morpheus/objects/table_info.hpp" -#include +#include "morpheus/utilities/type_util_detail.hpp" #include #include #include - #include #include #include diff --git a/morpheus/_lib/src/objects/tensor.cpp b/morpheus/_lib/src/objects/tensor.cpp index e5b1d9670f..632977665d 100644 --- a/morpheus/_lib/src/objects/tensor.cpp +++ b/morpheus/_lib/src/objects/tensor.cpp @@ -15,11 +15,11 @@ * limitations under the License. */ -#include +#include "morpheus/objects/tensor.hpp" -#include -#include -#include +#include "morpheus/objects/rmm_tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" +#include "morpheus/utilities/type_util.hpp" #include diff --git a/morpheus/_lib/src/objects/tensor_object.cpp b/morpheus/_lib/src/objects/tensor_object.cpp index 8a70f2c138..9641416a37 100644 --- a/morpheus/_lib/src/objects/tensor_object.cpp +++ b/morpheus/_lib/src/objects/tensor_object.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include +#include "morpheus/objects/tensor_object.hpp" -#include +#include "morpheus/utilities/tensor_util.hpp" #include // for blob #include // for DataType diff --git a/morpheus/_lib/src/objects/wrapped_tensor.cpp b/morpheus/_lib/src/objects/wrapped_tensor.cpp index f7fc1fba2a..7b314c4eb5 100644 --- a/morpheus/_lib/src/objects/wrapped_tensor.cpp +++ b/morpheus/_lib/src/objects/wrapped_tensor.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include +#include "morpheus/objects/wrapped_tensor.hpp" -#include +#include "morpheus/objects/tensor_object.hpp" #include #include diff --git a/morpheus/_lib/src/python_modules/common.cpp b/morpheus/_lib/src/python_modules/common.cpp index 4c9d714038..ca2ab266fe 100644 --- a/morpheus/_lib/src/python_modules/common.cpp +++ b/morpheus/_lib/src/python_modules/common.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include -#include -#include +#include "morpheus/objects/fiber_queue.hpp" +#include "morpheus/objects/wrapped_tensor.hpp" +#include "morpheus/utilities/cudf_util.hpp" #include diff --git a/morpheus/_lib/src/python_modules/file_types.cpp b/morpheus/_lib/src/python_modules/file_types.cpp index aa5f7a1551..f72cc17c40 100644 --- a/morpheus/_lib/src/python_modules/file_types.cpp +++ b/morpheus/_lib/src/python_modules/file_types.cpp @@ -15,8 +15,9 @@ * limitations under the License. */ -#include -#include +#include "morpheus/objects/file_types.hpp" + +#include "morpheus/utilities/cudf_util.hpp" #include diff --git a/morpheus/_lib/src/python_modules/messages.cpp b/morpheus/_lib/src/python_modules/messages.cpp index 4252f22118..9e2921fef4 100644 --- a/morpheus/_lib/src/python_modules/messages.cpp +++ b/morpheus/_lib/src/python_modules/messages.cpp @@ -15,28 +15,27 @@ * limitations under the License. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include +#include "morpheus/messages/memory/inference_memory.hpp" +#include "morpheus/messages/memory/inference_memory_fil.hpp" +#include "morpheus/messages/memory/inference_memory_nlp.hpp" +#include "morpheus/messages/memory/response_memory.hpp" +#include "morpheus/messages/memory/response_memory_probs.hpp" +#include "morpheus/messages/meta.hpp" +#include "morpheus/messages/multi.hpp" +#include "morpheus/messages/multi_inference.hpp" +#include "morpheus/messages/multi_inference_fil.hpp" +#include "morpheus/messages/multi_inference_nlp.hpp" +#include "morpheus/messages/multi_response.hpp" +#include "morpheus/messages/multi_response_probs.hpp" +#include "morpheus/objects/tensor.hpp" +#include "morpheus/utilities/cudf_util.hpp" #include #include // IWYU pragma: keep #include #include #include // IWYU pragma: keep +#include #include #include diff --git a/morpheus/_lib/src/python_modules/stages.cpp b/morpheus/_lib/src/python_modules/stages.cpp index f5a9cb71a6..b463645070 100644 --- a/morpheus/_lib/src/python_modules/stages.cpp +++ b/morpheus/_lib/src/python_modules/stages.cpp @@ -15,18 +15,18 @@ * limitations under the License. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "morpheus/stages/add_classification.hpp" +#include "morpheus/stages/add_scores.hpp" +#include "morpheus/stages/deserialization.hpp" +#include "morpheus/stages/file_source.hpp" +#include "morpheus/stages/filter_detection.hpp" +#include "morpheus/stages/kafka_source.hpp" +#include "morpheus/stages/preprocess_fil.hpp" +#include "morpheus/stages/preprocess_nlp.hpp" +#include "morpheus/stages/serialize.hpp" +#include "morpheus/stages/triton_inference.hpp" +#include "morpheus/stages/write_to_file.hpp" +#include "morpheus/utilities/cudf_util.hpp" #include diff --git a/morpheus/_lib/src/stages/add_classification.cpp b/morpheus/_lib/src/stages/add_classification.cpp index 5ff733989e..f67bb6b71c 100644 --- a/morpheus/_lib/src/stages/add_classification.cpp +++ b/morpheus/_lib/src/stages/add_classification.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include +#include "morpheus/stages/add_classification.hpp" -#include +#include "morpheus/utilities/matx_util.hpp" #include #include diff --git a/morpheus/_lib/src/stages/add_scores.cpp b/morpheus/_lib/src/stages/add_scores.cpp index fd7ef90dc8..0b23890382 100644 --- a/morpheus/_lib/src/stages/add_scores.cpp +++ b/morpheus/_lib/src/stages/add_scores.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include +#include "morpheus/stages/add_scores.hpp" -#include +#include "morpheus/utilities/matx_util.hpp" #include #include diff --git a/morpheus/_lib/src/stages/deserialize.cpp b/morpheus/_lib/src/stages/deserialize.cpp index b3a18c717c..e98c8e8477 100644 --- a/morpheus/_lib/src/stages/deserialize.cpp +++ b/morpheus/_lib/src/stages/deserialize.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include +#include "morpheus/stages/deserialization.hpp" #include #include diff --git a/morpheus/_lib/src/stages/file_source.cpp b/morpheus/_lib/src/stages/file_source.cpp index 67df61d83e..825f10c6c5 100644 --- a/morpheus/_lib/src/stages/file_source.cpp +++ b/morpheus/_lib/src/stages/file_source.cpp @@ -15,20 +15,18 @@ * limitations under the License. */ -#include - -#include +#include "morpheus/stages/file_source.hpp" #include #include #include #include -#include - #include +#include +#include #include #include -#include +#include #include #include diff --git a/morpheus/_lib/src/stages/filter_detection.cpp b/morpheus/_lib/src/stages/filter_detection.cpp index 53b71dd295..c5e43d04d2 100644 --- a/morpheus/_lib/src/stages/filter_detection.cpp +++ b/morpheus/_lib/src/stages/filter_detection.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include +#include "morpheus/stages/filter_detection.hpp" -#include +#include "morpheus/utilities/matx_util.hpp" #include #include diff --git a/morpheus/_lib/src/stages/kafka_source.cpp b/morpheus/_lib/src/stages/kafka_source.cpp index 4227701e44..e8eeb4da1e 100644 --- a/morpheus/_lib/src/stages/kafka_source.cpp +++ b/morpheus/_lib/src/stages/kafka_source.cpp @@ -15,25 +15,24 @@ * limitations under the License. */ -#include +#include "morpheus/stages/kafka_source.hpp" -#include -#include -#include +#include "morpheus/messages/meta.hpp" +#include "morpheus/utilities/stage_util.hpp" +#include "morpheus/utilities/string_util.hpp" -#include -#include -#include - -#include -#include -#include #include #include #include #include +#include +#include +#include #include #include +#include +#include +#include #include #include diff --git a/morpheus/_lib/src/stages/preprocess_fil.cpp b/morpheus/_lib/src/stages/preprocess_fil.cpp index 3a5a1a1058..fc33c9d559 100644 --- a/morpheus/_lib/src/stages/preprocess_fil.cpp +++ b/morpheus/_lib/src/stages/preprocess_fil.cpp @@ -15,25 +15,24 @@ * limitations under the License. */ -#include +#include "morpheus/stages/preprocess_fil.hpp" -#include -#include -#include -#include +#include "morpheus/messages/memory/inference_memory_fil.hpp" +#include "morpheus/utilities/matx_util.hpp" +#include "morpheus/utilities/type_util.hpp" +#include "morpheus/utilities/type_util_detail.hpp" -#include -#include - -#include -#include -#include -#include #include #include #include #include +#include +#include +#include +#include +#include #include +#include #include #include diff --git a/morpheus/_lib/src/stages/preprocess_nlp.cpp b/morpheus/_lib/src/stages/preprocess_nlp.cpp index 6db54444ba..c2f10bdf24 100644 --- a/morpheus/_lib/src/stages/preprocess_nlp.cpp +++ b/morpheus/_lib/src/stages/preprocess_nlp.cpp @@ -15,18 +15,17 @@ * limitations under the License. */ -#include +#include "morpheus/stages/preprocess_nlp.hpp" -#include -#include +#include "morpheus/messages/multi_inference.hpp" +#include "morpheus/utilities/type_util.hpp" -#include -#include - -#include #include #include +#include #include +#include +#include #include #include diff --git a/morpheus/_lib/src/stages/serialize.cpp b/morpheus/_lib/src/stages/serialize.cpp index 6b86f7fd43..55319862b5 100644 --- a/morpheus/_lib/src/stages/serialize.cpp +++ b/morpheus/_lib/src/stages/serialize.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include +#include "morpheus/stages/serialize.hpp" #include #include diff --git a/morpheus/_lib/src/stages/triton_inference.cpp b/morpheus/_lib/src/stages/triton_inference.cpp index 92bc160615..c80bdaece2 100644 --- a/morpheus/_lib/src/stages/triton_inference.cpp +++ b/morpheus/_lib/src/stages/triton_inference.cpp @@ -15,19 +15,18 @@ * limitations under the License. */ -#include +#include "morpheus/stages/triton_inference.hpp" -#include -#include -#include -#include -#include - -#include +#include "morpheus/messages/multi_response_probs.hpp" +#include "morpheus/objects/triton_in_out.hpp" +#include "morpheus/utilities/matx_util.hpp" +#include "morpheus/utilities/stage_util.hpp" +#include "morpheus/utilities/type_util.hpp" #include #include #include +#include #include #include diff --git a/morpheus/_lib/src/stages/write_to_file.cpp b/morpheus/_lib/src/stages/write_to_file.cpp index 45f3fb40c8..4ed12456db 100644 --- a/morpheus/_lib/src/stages/write_to_file.cpp +++ b/morpheus/_lib/src/stages/write_to_file.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include +#include "morpheus/stages/write_to_file.hpp" -#include +#include "morpheus/utilities/matx_util.hpp" #include #include diff --git a/morpheus/_lib/src/utilities/cudf_util.cpp b/morpheus/_lib/src/utilities/cudf_util.cpp index 51c2126aab..42dc1a94c9 100644 --- a/morpheus/_lib/src/utilities/cudf_util.cpp +++ b/morpheus/_lib/src/utilities/cudf_util.cpp @@ -15,13 +15,13 @@ * limitations under the License. */ -#include +#include "morpheus/utilities/cudf_util.hpp" -#include +#include "morpheus/objects/table_info.hpp" +#include #include #include -#include /** * **************This needs to come last.******************** diff --git a/morpheus/_lib/src/utilities/cupy_util.cpp b/morpheus/_lib/src/utilities/cupy_util.cpp index 3a8473d805..e449de825a 100644 --- a/morpheus/_lib/src/utilities/cupy_util.cpp +++ b/morpheus/_lib/src/utilities/cupy_util.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include +#include "morpheus/utilities/cupy_util.hpp" -#include +#include "morpheus/objects/tensor.hpp" #include #include // IWYU pragma: keep diff --git a/morpheus/_lib/src/utilities/matx_util.cu b/morpheus/_lib/src/utilities/matx_util.cu index 09657ef49d..5edabbfba8 100644 --- a/morpheus/_lib/src/utilities/matx_util.cu +++ b/morpheus/_lib/src/utilities/matx_util.cu @@ -15,11 +15,11 @@ * limitations under the License. */ -#include +#include "morpheus/utilities/matx_util.hpp" -#include -#include -#include +#include "morpheus/objects/dev_mem_info.hpp" +#include "morpheus/utilities/type_util.hpp" +#include "morpheus/objects/tensor_object.hpp" #include diff --git a/morpheus/_lib/src/utilities/string_util.cpp b/morpheus/_lib/src/utilities/string_util.cpp index b6ca53971a..e9fa187243 100644 --- a/morpheus/_lib/src/utilities/string_util.cpp +++ b/morpheus/_lib/src/utilities/string_util.cpp @@ -15,12 +15,11 @@ * limitations under the License. */ -#include +#include "morpheus/utilities/string_util.hpp" namespace morpheus { - bool StringUtil::str_contains(const std::string &str, const std::string &search_str) - { - return str.find(search_str) != std::string::npos; - } +bool StringUtil::str_contains(const std::string &str, const std::string &search_str) +{ + return str.find(search_str) != std::string::npos; } - +} // namespace morpheus diff --git a/morpheus/_lib/src/utilities/table_util.cpp b/morpheus/_lib/src/utilities/table_util.cpp index 82dbed070d..854a338117 100644 --- a/morpheus/_lib/src/utilities/table_util.cpp +++ b/morpheus/_lib/src/utilities/table_util.cpp @@ -15,11 +15,10 @@ * limitations under the License. */ -#include +#include "morpheus/utilities/table_util.hpp" #include #include - #include #include @@ -28,20 +27,26 @@ namespace fs = std::filesystem; namespace py = pybind11; -cudf::io::table_with_metadata morpheus::CuDFTableUtil::load_table(const std::string &filename) { +cudf::io::table_with_metadata morpheus::CuDFTableUtil::load_table(const std::string &filename) +{ auto file_path = fs::path(filename); - if (file_path.extension() == ".json" || file_path.extension() == ".jsonlines") { + if (file_path.extension() == ".json" || file_path.extension() == ".jsonlines") + { // First, load the file into json auto options = cudf::io::json_reader_options::builder(cudf::io::source_info{filename}).lines(true); return cudf::io::read_json(options.build()); - } else if (file_path.extension() == ".csv") { + } + else if (file_path.extension() == ".csv") + { auto options = cudf::io::csv_reader_options::builder(cudf::io::source_info{filename}); return cudf::io::read_csv(options.build()); - } else { + } + else + { LOG(FATAL) << "Unknown extension for file: " << filename; throw std::runtime_error("Unknown extension"); } -} \ No newline at end of file +} diff --git a/morpheus/_lib/src/utilities/tensor_util.cpp b/morpheus/_lib/src/utilities/tensor_util.cpp index 9e62a39dc5..a7fb3ab27f 100644 --- a/morpheus/_lib/src/utilities/tensor_util.cpp +++ b/morpheus/_lib/src/utilities/tensor_util.cpp @@ -15,14 +15,13 @@ * limitations under the License. */ -#include +#include "morpheus/utilities/tensor_util.hpp" +#include +#include // for DCHECK_EQ #include // for sort_indexes -#include // for DCHECK_EQ - -#include // for copy -#include +#include // for copy #include // for multiplies #include // for begin, end #include // for accumulate diff --git a/morpheus/_lib/src/utilities/type_util.cu b/morpheus/_lib/src/utilities/type_util.cu index 8fc4697178..40b1c5686f 100644 --- a/morpheus/_lib/src/utilities/type_util.cu +++ b/morpheus/_lib/src/utilities/type_util.cu @@ -15,7 +15,7 @@ * limitations under the License. */ -#include +#include "morpheus/utilities/type_util.hpp" #include #include diff --git a/morpheus/_lib/src/utilities/type_util_detail.cpp b/morpheus/_lib/src/utilities/type_util_detail.cpp index f89cd0f907..670d89da19 100644 --- a/morpheus/_lib/src/utilities/type_util_detail.cpp +++ b/morpheus/_lib/src/utilities/type_util_detail.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include +#include "morpheus/utilities/type_util_detail.hpp" -#include +#include "morpheus/utilities/string_util.hpp" #include diff --git a/morpheus/_lib/tests/test_cuda.cu b/morpheus/_lib/tests/test_cuda.cu index 34e5114df9..5370c1895a 100644 --- a/morpheus/_lib/tests/test_cuda.cu +++ b/morpheus/_lib/tests/test_cuda.cu @@ -17,7 +17,7 @@ #include "test_morpheus.hpp" -#include +#include "morpheus/objects/tensor_object.hpp" #include // for SRF_CHECK_CUDA #include // for enqueue_stream_sync_event diff --git a/morpheus/_lib/tests/test_type_util_detail.cpp b/morpheus/_lib/tests/test_type_util_detail.cpp index 17aafd5760..ddf1186d44 100644 --- a/morpheus/_lib/tests/test_type_util_detail.cpp +++ b/morpheus/_lib/tests/test_type_util_detail.cpp @@ -17,7 +17,7 @@ #include "./test_morpheus.hpp" // IWYU pragma: associated -#include +#include "morpheus/utilities/type_util_detail.hpp" #include // for EXPECT_EQ From 532d8baf33bb0a65f4938c8d616a571f558e34a1 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 14 Jul 2022 15:12:18 -0700 Subject: [PATCH 04/41] Add srf's pragma once check --- ci/scripts/cpp_checks.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ci/scripts/cpp_checks.sh b/ci/scripts/cpp_checks.sh index f396c69c51..e8ca347c38 100755 --- a/ci/scripts/cpp_checks.sh +++ b/ci/scripts/cpp_checks.sh @@ -27,6 +27,7 @@ if [[ "${SKIP_IWYU}" == "" && -z "${IWYU_TOOL}" ]]; then fi # Pre-populate the return values in case they are skipped +PRAGMA_CHECK_RETVAL=0 CLANG_TIDY_RETVAL=0 CLANG_FORMAT_RETVAL=0 IWYU_RETVAL=0 @@ -42,6 +43,16 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then echo " $f" done + # Check for `#pragma once` in any .hpp or .h file + if [[ "${SKIP_PRAGMA_CHECK}" == "" ]]; then + + PRAGMA_CHECK_OUTPUT=`grep -rL --include=\*.{h,hpp} --exclude-dir={build,.cache} -e '#pragma once' $(get_modified_files $CPP_FILE_REGEX)` + + if [[ -n "${PRAGMA_CHECK_OUTPUT}" ]]; then + PRAGMA_CHECK_RETVAL=1 + fi + fi + # Clang-tidy if [[ "${SKIP_CLANG_TIDY}" == "" ]]; then @@ -79,6 +90,19 @@ else fi # Now check the values +if [[ "${SKIP_PRAGMA_CHECK}" != "" ]]; then + echo -e "\n\n>>>> SKIPPED: pragma check\n\n" +elif [[ "${PRAGMA_CHECK_RETVAL}" != "0" ]]; then + echo -e "\n\n>>>> FAILED: pragma check; begin output\n\n" + echo -e "Missing \`#pragma once\` in the following files:" + echo -e "${PRAGMA_CHECK_OUTPUT}" + echo -e "\n\n>>>> FAILED: pragma check; end output\n\n" \ + "To auto-fix many issues (not all) run:\n" \ + " ./ci/scripts/fix_all.sh\n\n" +else + echo -e "\n\n>>>> PASSED: pragma check\n\n" +fi + if [[ "${SKIP_CLANG_TIDY}" != "" ]]; then echo -e "\n\n>>>> SKIPPED: clang-tidy check\n\n" elif [[ "${CLANG_TIDY_RETVAL}" != "0" ]]; then From a0b33f1d7160ddb361baa57f334ac9d8b50a769c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 14 Jul 2022 15:25:49 -0700 Subject: [PATCH 05/41] Add cpp checks to ci --- ci/scripts/jenkins/checks.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index 47de55ca16..29183d69af 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -30,5 +30,8 @@ show_conda_info gpuci_logger "Runing Python style checks" ${MORPHEUS_ROOT}/ci/scripts/python_checks.sh +gpuci_logger "Runing C++ style checks" +${MORPHEUS_ROOT}/ci/scripts/cpp_checks.sh + gpuci_logger "Checking copyright headers" python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT} From cd01f6c4bb69c4134da486d8d7ec7190fa90e40f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 14 Jul 2022 15:51:52 -0700 Subject: [PATCH 06/41] Install clang-tools into checks stage --- ci/scripts/jenkins/checks.sh | 2 +- docker/conda/environments/cuda11.5_ci.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index 29183d69af..6b10bca6fe 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -23,7 +23,7 @@ fetch_base_branch conda activate rapids gpuci_logger "Installing CI dependencies" -mamba install -q -y -c conda-forge "yapf=0.32" +mamba env update -q -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_ci.yml show_conda_info diff --git a/docker/conda/environments/cuda11.5_ci.yml b/docker/conda/environments/cuda11.5_ci.yml index 4dede667c8..c44b7000b5 100644 --- a/docker/conda/environments/cuda11.5_ci.yml +++ b/docker/conda/environments/cuda11.5_ci.yml @@ -19,8 +19,11 @@ channels: - conda-forge dependencies: - bash=5.1.16 # bash >=4.4 is necessary for morpheus cli auto-complete tests to pass. + - clang-tools=12 - conda-pack=0.7 - git-lfs=3.2 - git>=2.35.3 # Needed for wildcards on safe.directory - pkg-config=0.29 - sccache=0.3 + - pip: + yapf=0.32 From 2f9f2efe5ffb66be4634cdd5f94ea5c918fa8173 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 14 Jul 2022 16:00:23 -0700 Subject: [PATCH 07/41] wip --- docker/conda/environments/cuda11.5_ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/conda/environments/cuda11.5_ci.yml b/docker/conda/environments/cuda11.5_ci.yml index c44b7000b5..901e715cd9 100644 --- a/docker/conda/environments/cuda11.5_ci.yml +++ b/docker/conda/environments/cuda11.5_ci.yml @@ -23,7 +23,8 @@ dependencies: - conda-pack=0.7 - git-lfs=3.2 - git>=2.35.3 # Needed for wildcards on safe.directory + - pip - pkg-config=0.29 - sccache=0.3 - pip: - yapf=0.32 + yapf==0.32 From 27dae7b8481937f032e58d3d8c970e21f04df340 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Thu, 14 Jul 2022 16:06:22 -0700 Subject: [PATCH 08/41] Fix formatting --- docker/conda/environments/cuda11.5_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/conda/environments/cuda11.5_ci.yml b/docker/conda/environments/cuda11.5_ci.yml index 901e715cd9..cf7e8720c0 100644 --- a/docker/conda/environments/cuda11.5_ci.yml +++ b/docker/conda/environments/cuda11.5_ci.yml @@ -27,4 +27,4 @@ dependencies: - pkg-config=0.29 - sccache=0.3 - pip: - yapf==0.32 + - yapf==0.32 From afcb56b6e420ff2c649825df2670db1cadca7f36 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 Jul 2022 07:58:26 -0700 Subject: [PATCH 09/41] Attempt to config cmake --- ci/scripts/jenkins/checks.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index 6b10bca6fe..bdd5ef1b0f 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -20,13 +20,29 @@ source ${WORKSPACE}/ci/scripts/jenkins/common.sh fetch_base_branch -conda activate rapids +gpuci_logger "Creating conda env" +rm -rf ${MORPHEUS_ROOT}/.cache/ ${MORPHEUS_ROOT}/build/ +conda config --add pkgs_dirs /opt/conda/pkgs +conda config --env --add channels conda-forge +conda config --env --set channel_alias ${CONDA_CHANNEL_ALIAS:-"https://conda.anaconda.org"} +mamba env create -q -n morpheus -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_dev.yml +conda activate morpheus gpuci_logger "Installing CI dependencies" mamba env update -q -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_ci.yml show_conda_info +gpuci_logger "Configuring cmake for Morpheus" +cmake -B build -G Ninja \ + -DCMAKE_MESSAGE_CONTEXT_SHOW=ON \ + -DMORPHEUS_BUILD_BENCHMARKS=ON \ + -DMORPHEUS_BUILD_EXAMPLES=ON \ + -DMORPHEUS_BUILD_TESTS=ON \ + -DMORPHEUS_USE_CONDA=ON \ + -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF \ + . + gpuci_logger "Runing Python style checks" ${MORPHEUS_ROOT}/ci/scripts/python_checks.sh From 255614ba425679689848517b565cffe1a1f55d99 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 Jul 2022 10:36:09 -0700 Subject: [PATCH 10/41] Add clang to ci tools --- ci/scripts/cpp_checks.sh | 2 ++ ci/scripts/jenkins/checks.sh | 8 ++++---- ci/scripts/run_clang_tidy_for_ci.sh | 6 +++++- docker/conda/environments/cuda11.5_ci.yml | 1 + 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ci/scripts/cpp_checks.sh b/ci/scripts/cpp_checks.sh index e8ca347c38..3a6344cf81 100755 --- a/ci/scripts/cpp_checks.sh +++ b/ci/scripts/cpp_checks.sh @@ -58,6 +58,8 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then CLANG_TIDY_DIFF=$(find_clang_tidy_diff) + echo ${BUILD_DIR} + echo ${CLANG_TIDY_DIFF} # Run using a clang-tidy wrapper to allow warnings-as-errors and to eliminate any output except errors (since clang-tidy-diff.py doesnt return the correct error codes) CLANG_TIDY_OUTPUT=`get_unified_diff ${CPP_FILE_REGEX} | ${CLANG_TIDY_DIFF} -j 0 -path ${BUILD_DIR} -p1 -quiet 2>&1` diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index bdd5ef1b0f..5c59739ec1 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -33,6 +33,9 @@ mamba env update -q -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER show_conda_info +gpuci_logger "Runing Python style checks" +${MORPHEUS_ROOT}/ci/scripts/python_checks.sh + gpuci_logger "Configuring cmake for Morpheus" cmake -B build -G Ninja \ -DCMAKE_MESSAGE_CONTEXT_SHOW=ON \ @@ -43,11 +46,8 @@ cmake -B build -G Ninja \ -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF \ . -gpuci_logger "Runing Python style checks" -${MORPHEUS_ROOT}/ci/scripts/python_checks.sh - gpuci_logger "Runing C++ style checks" -${MORPHEUS_ROOT}/ci/scripts/cpp_checks.sh +SKIP_IWYU=1 ${MORPHEUS_ROOT}/ci/scripts/cpp_checks.sh gpuci_logger "Checking copyright headers" python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT} diff --git a/ci/scripts/run_clang_tidy_for_ci.sh b/ci/scripts/run_clang_tidy_for_ci.sh index 54191c68b8..2cd8b22669 100755 --- a/ci/scripts/run_clang_tidy_for_ci.sh +++ b/ci/scripts/run_clang_tidy_for_ci.sh @@ -23,4 +23,8 @@ # Also add -fno-caret-diagnostics to prevent clangs own compiler warnings from # coming through: # https://github.com/llvm/llvm-project/blob/3f3faa36ff3d84af3c3ed84772d7e4278bc44ff1/libc/cmake/modules/LLVMLibCObjectRules.cmake#L226 -${CLANG_TIDY:-clang-tidy} --warnings-as-errors='*' --extra-arg=-fno-caret-diagnostics "$@" +echo ${CLANG_TIDY:-$(which clang-tidy)} +echo ${CONDA_BUILD_SYSROOT}/usr/include +echo "$@" +echo "---------------" +${CLANG_TIDY:-$(which clang-tidy)} --warnings-as-errors='*' --extra-arg=-fno-caret-diagnostics "$@" diff --git a/docker/conda/environments/cuda11.5_ci.yml b/docker/conda/environments/cuda11.5_ci.yml index cf7e8720c0..267453a1d8 100644 --- a/docker/conda/environments/cuda11.5_ci.yml +++ b/docker/conda/environments/cuda11.5_ci.yml @@ -19,6 +19,7 @@ channels: - conda-forge dependencies: - bash=5.1.16 # bash >=4.4 is necessary for morpheus cli auto-complete tests to pass. + - clang=12 - clang-tools=12 - conda-pack=0.7 - git-lfs=3.2 From 3d361c744ca2cc272d2dd3b1751e67222996323d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 Jul 2022 10:55:54 -0700 Subject: [PATCH 11/41] Revert wip changes that were accidentally commited --- ci/scripts/run_clang_tidy_for_ci.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ci/scripts/run_clang_tidy_for_ci.sh b/ci/scripts/run_clang_tidy_for_ci.sh index 2cd8b22669..54191c68b8 100755 --- a/ci/scripts/run_clang_tidy_for_ci.sh +++ b/ci/scripts/run_clang_tidy_for_ci.sh @@ -23,8 +23,4 @@ # Also add -fno-caret-diagnostics to prevent clangs own compiler warnings from # coming through: # https://github.com/llvm/llvm-project/blob/3f3faa36ff3d84af3c3ed84772d7e4278bc44ff1/libc/cmake/modules/LLVMLibCObjectRules.cmake#L226 -echo ${CLANG_TIDY:-$(which clang-tidy)} -echo ${CONDA_BUILD_SYSROOT}/usr/include -echo "$@" -echo "---------------" -${CLANG_TIDY:-$(which clang-tidy)} --warnings-as-errors='*' --extra-arg=-fno-caret-diagnostics "$@" +${CLANG_TIDY:-clang-tidy} --warnings-as-errors='*' --extra-arg=-fno-caret-diagnostics "$@" From 58d0fd89619f6aa0f62f95a087f1da3e0ea91e65 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 Jul 2022 10:58:45 -0700 Subject: [PATCH 12/41] Revert debug code --- ci/scripts/cpp_checks.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/scripts/cpp_checks.sh b/ci/scripts/cpp_checks.sh index 3a6344cf81..e8ca347c38 100755 --- a/ci/scripts/cpp_checks.sh +++ b/ci/scripts/cpp_checks.sh @@ -58,8 +58,6 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then CLANG_TIDY_DIFF=$(find_clang_tidy_diff) - echo ${BUILD_DIR} - echo ${CLANG_TIDY_DIFF} # Run using a clang-tidy wrapper to allow warnings-as-errors and to eliminate any output except errors (since clang-tidy-diff.py doesnt return the correct error codes) CLANG_TIDY_OUTPUT=`get_unified_diff ${CPP_FILE_REGEX} | ${CLANG_TIDY_DIFF} -j 0 -path ${BUILD_DIR} -p1 -quiet 2>&1` From d0ec98a60d985dbf671e9ad3593577fc3dff4ac9 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 Jul 2022 11:06:54 -0700 Subject: [PATCH 13/41] Generate cython source code --- CMakeLists.txt | 6 +++++- morpheus/_lib/cmake/libraries/cudf_helpers.cmake | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b19fddfad8..24ef2b9cf3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,8 +84,12 @@ endif() ##### Morpheus Python Setup ###### ################################## +# Create a custom target to allow preparing for style checks +add_custom_target(style_checks + COMMENT "Building dependencies for style checks" +) -# Include the main trtlab code +# Include the main morpheus code add_subdirectory(morpheus) # Get all of the python files to copy to the build directory diff --git a/morpheus/_lib/cmake/libraries/cudf_helpers.cmake b/morpheus/_lib/cmake/libraries/cudf_helpers.cmake index 7a2486d87a..46a140c2f2 100644 --- a/morpheus/_lib/cmake/libraries/cudf_helpers.cmake +++ b/morpheus/_lib/cmake/libraries/cudf_helpers.cmake @@ -28,6 +28,8 @@ morpheus_add_cython_libraries( ${MORPHEUS_LIB_INSTALL_DIR} ) +add_dependencies(style_checks ${cudf_helpers_target}) + if (MORPHEUS_PYTHON_INPLACE_BUILD) inplace_build_copy(${cudf_helpers_target} ${MORPHEUS_LIB_ROOT}) endif() From fe663d250174bb4f694d425f1d1d8a025229d7bd Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 Jul 2022 11:07:07 -0700 Subject: [PATCH 14/41] Generate cython source code --- ci/scripts/jenkins/checks.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index 5c59739ec1..fec649efce 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -46,6 +46,9 @@ cmake -B build -G Ninja \ -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF \ . +gpuci_logger "Building targets that generate source code" +cmake --build build --target style_checks --parallel ${PARALLEL_LEVEL} + gpuci_logger "Runing C++ style checks" SKIP_IWYU=1 ${MORPHEUS_ROOT}/ci/scripts/cpp_checks.sh From 1c290d65305ea0530dea6ba244e7932a5b8c373d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 Jul 2022 11:29:38 -0700 Subject: [PATCH 15/41] Use sccache for generating cython --- ci/Jenkinsfile | 12 ++++++++++-- ci/scripts/jenkins/build.sh | 11 +---------- ci/scripts/jenkins/checks.sh | 12 ++++-------- ci/scripts/jenkins/common.sh | 2 ++ 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 8bc2bda689..23a9eb6d9a 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -9,6 +9,7 @@ pipeline { PYTHON_VER = '3.8' RAPIDS_VER = '22.06' CUDA_VER = '11.5' + USE_SCCACHE = '1' HOME = "${WORKSPACE}" BUILD_TYPE = sh(returnStdout: true, script: 'rapids-build-type') } @@ -36,7 +37,15 @@ pipeline { externalDelete: 'sudo rm -rf %s' ) checkout scm - sh "${WORKSPACE}/ci/scripts/jenkins/checks.sh" + withCredentials([[ + $class: 'AmazonWebServicesCredentialsBinding', + credentialsId: "aws-s3-gpuci", + accessKeyVariable: 'AWS_ACCESS_KEY_ID', + secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' + ]]) + { + sh "${WORKSPACE}/ci/scripts/jenkins/checks.sh" + } } } stage('Build:linux:x86_64:gcc') { @@ -44,7 +53,6 @@ pipeline { timeout(time: 4, unit: 'HOURS') } environment { - USE_SCCACHE = '1' HOME = "${WORKSPACE}" } agent { diff --git a/ci/scripts/jenkins/build.sh b/ci/scripts/jenkins/build.sh index 34a0fc0a6e..a015e9e608 100755 --- a/ci/scripts/jenkins/build.sh +++ b/ci/scripts/jenkins/build.sh @@ -41,16 +41,7 @@ cmake --version ninja --version gpuci_logger "Configuring cmake for Morpheus" -cmake -B build -G Ninja \ - -DCMAKE_MESSAGE_CONTEXT_SHOW=ON \ - -DMORPHEUS_BUILD_BENCHMARKS=ON \ - -DMORPHEUS_BUILD_EXAMPLES=ON \ - -DMORPHEUS_BUILD_TESTS=ON \ - -DMORPHEUS_USE_CONDA=ON \ - -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF \ - -DMORPHEUS_USE_CCACHE=ON \ - -DCCACHE_PROGRAM_PATH=$(which sccache) \ - . +cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DCCACHE_PROGRAM_PATH=$(which sccache) . gpuci_logger "Building Morpheus" cmake --build build -j --parallel ${PARALLEL_LEVEL} diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index fec649efce..11db13af91 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -37,18 +37,14 @@ gpuci_logger "Runing Python style checks" ${MORPHEUS_ROOT}/ci/scripts/python_checks.sh gpuci_logger "Configuring cmake for Morpheus" -cmake -B build -G Ninja \ - -DCMAKE_MESSAGE_CONTEXT_SHOW=ON \ - -DMORPHEUS_BUILD_BENCHMARKS=ON \ - -DMORPHEUS_BUILD_EXAMPLES=ON \ - -DMORPHEUS_BUILD_TESTS=ON \ - -DMORPHEUS_USE_CONDA=ON \ - -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF \ - . +cmake -B build -G Ninja cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DCCACHE_PROGRAM_PATH=$(which sccache) . gpuci_logger "Building targets that generate source code" cmake --build build --target style_checks --parallel ${PARALLEL_LEVEL} +gpuci_logger "sccache usage for source build:" +sccache --show-stats + gpuci_logger "Runing C++ style checks" SKIP_IWYU=1 ${MORPHEUS_ROOT}/ci/scripts/cpp_checks.sh diff --git a/ci/scripts/jenkins/common.sh b/ci/scripts/jenkins/common.sh index 79ba89cee0..43d0c45c7e 100644 --- a/ci/scripts/jenkins/common.sh +++ b/ci/scripts/jenkins/common.sh @@ -52,6 +52,8 @@ export SCCACHE_REGION=us-west-2 export SCCACHE_IDLE_TIMEOUT=32768 #export SCCACHE_LOG=debug +export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_USE_CCACHE=ON" + export FETCH_STATUS=0 gpuci_logger "Environ:" From 2249d9413a4f4de4526b492bad12c6d887383894 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Fri, 15 Jul 2022 11:29:59 -0700 Subject: [PATCH 16/41] Update copyright years --- morpheus/_lib/include/morpheus/messages/meta.hpp | 2 +- morpheus/_lib/include/morpheus/objects/python_data_table.hpp | 2 +- morpheus/_lib/include/morpheus/utilities/cudf_util.hpp | 2 +- morpheus/_lib/src/messages/memory/inference_memory.cpp | 2 +- morpheus/_lib/src/messages/meta.cpp | 2 +- morpheus/_lib/src/objects/dev_mem_info.cpp | 2 +- morpheus/_lib/src/objects/fiber_queue.cpp | 2 +- morpheus/_lib/src/objects/python_data_table.cpp | 2 +- morpheus/_lib/src/utilities/string_util.cpp | 2 +- morpheus/_lib/src/utilities/table_util.cpp | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/morpheus/_lib/include/morpheus/messages/meta.hpp b/morpheus/_lib/include/morpheus/messages/meta.hpp index 2ff91ce371..b1fd03973f 100644 --- a/morpheus/_lib/include/morpheus/messages/meta.hpp +++ b/morpheus/_lib/include/morpheus/messages/meta.hpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/morpheus/_lib/include/morpheus/objects/python_data_table.hpp b/morpheus/_lib/include/morpheus/objects/python_data_table.hpp index bbe395b285..0e5ec93f16 100644 --- a/morpheus/_lib/include/morpheus/objects/python_data_table.hpp +++ b/morpheus/_lib/include/morpheus/objects/python_data_table.hpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp b/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp index 4c8cada7b1..5b87df29d0 100644 --- a/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp +++ b/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/morpheus/_lib/src/messages/memory/inference_memory.cpp b/morpheus/_lib/src/messages/memory/inference_memory.cpp index 4012924546..740463b2dc 100644 --- a/morpheus/_lib/src/messages/memory/inference_memory.cpp +++ b/morpheus/_lib/src/messages/memory/inference_memory.cpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/morpheus/_lib/src/messages/meta.cpp b/morpheus/_lib/src/messages/meta.cpp index 5ae9a7aebb..a7f3217483 100644 --- a/morpheus/_lib/src/messages/meta.cpp +++ b/morpheus/_lib/src/messages/meta.cpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/morpheus/_lib/src/objects/dev_mem_info.cpp b/morpheus/_lib/src/objects/dev_mem_info.cpp index 08d53642aa..4d92dfe3cc 100644 --- a/morpheus/_lib/src/objects/dev_mem_info.cpp +++ b/morpheus/_lib/src/objects/dev_mem_info.cpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/morpheus/_lib/src/objects/fiber_queue.cpp b/morpheus/_lib/src/objects/fiber_queue.cpp index 1eca4dd748..2fcf783519 100644 --- a/morpheus/_lib/src/objects/fiber_queue.cpp +++ b/morpheus/_lib/src/objects/fiber_queue.cpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/morpheus/_lib/src/objects/python_data_table.cpp b/morpheus/_lib/src/objects/python_data_table.cpp index 12087ca685..7ab871e965 100644 --- a/morpheus/_lib/src/objects/python_data_table.cpp +++ b/morpheus/_lib/src/objects/python_data_table.cpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/morpheus/_lib/src/utilities/string_util.cpp b/morpheus/_lib/src/utilities/string_util.cpp index e9fa187243..bee6bbe59d 100644 --- a/morpheus/_lib/src/utilities/string_util.cpp +++ b/morpheus/_lib/src/utilities/string_util.cpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/morpheus/_lib/src/utilities/table_util.cpp b/morpheus/_lib/src/utilities/table_util.cpp index 854a338117..0cbf327088 100644 --- a/morpheus/_lib/src/utilities/table_util.cpp +++ b/morpheus/_lib/src/utilities/table_util.cpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); From 8552761e27c8a488fcaec604dcc4c897f9d215ec Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 Jul 2022 15:37:25 -0700 Subject: [PATCH 17/41] wip --- ci/scripts/jenkins/checks.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index 11db13af91..eceb93939f 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -33,6 +33,21 @@ mamba env update -q -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER show_conda_info +gpuci_logger "Installing IWYU" +export IWYU_DIR="${WORKSPACE_TMP}/iwyu" +cd ${WORKSPACE_TMP} +git clone https://github.com/include-what-you-use/include-what-you-use.git ${IWYU_DIR} +cd ${IWYU_DIR} +git checkout clang_12 +cmake -G Ninja \ + -DCMAKE_PREFIX_PATH=$(llvm-config --cmakedir) \ + -DCMAKE_C_COMPILER=$(which clang) \ + -DCMAKE_CXX_COMPILER=$(which clang++) \ + -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} \ + . + +cmake --build . --parallel ${PARALLEL_LEVEL} --target install + gpuci_logger "Runing Python style checks" ${MORPHEUS_ROOT}/ci/scripts/python_checks.sh From e3fca5fd2312e1ff6b9f640d178b8fa84890be21 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 Jul 2022 15:38:42 -0700 Subject: [PATCH 18/41] Fix copy/paste error --- ci/scripts/jenkins/checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index 11db13af91..076fadde17 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -37,7 +37,7 @@ gpuci_logger "Runing Python style checks" ${MORPHEUS_ROOT}/ci/scripts/python_checks.sh gpuci_logger "Configuring cmake for Morpheus" -cmake -B build -G Ninja cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DCCACHE_PROGRAM_PATH=$(which sccache) . +cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DCCACHE_PROGRAM_PATH=$(which sccache) . gpuci_logger "Building targets that generate source code" cmake --build build --target style_checks --parallel ${PARALLEL_LEVEL} From 542f7f70fb986b9bae138ee6357ac84288e84995 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 Jul 2022 15:40:53 -0700 Subject: [PATCH 19/41] Enable iwyu --- ci/scripts/jenkins/checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index a1dbbf453c..832aa60c47 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -61,7 +61,7 @@ gpuci_logger "sccache usage for source build:" sccache --show-stats gpuci_logger "Runing C++ style checks" -SKIP_IWYU=1 ${MORPHEUS_ROOT}/ci/scripts/cpp_checks.sh +${MORPHEUS_ROOT}/ci/scripts/cpp_checks.sh gpuci_logger "Checking copyright headers" python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT} From 753f05cc7a980d5c44a9596d9302cd988c497f7f Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 Jul 2022 16:00:24 -0700 Subject: [PATCH 20/41] Add clangxx --- docker/conda/environments/cuda11.5_ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/conda/environments/cuda11.5_ci.yml b/docker/conda/environments/cuda11.5_ci.yml index 267453a1d8..003dff6a5a 100644 --- a/docker/conda/environments/cuda11.5_ci.yml +++ b/docker/conda/environments/cuda11.5_ci.yml @@ -21,6 +21,7 @@ dependencies: - bash=5.1.16 # bash >=4.4 is necessary for morpheus cli auto-complete tests to pass. - clang=12 - clang-tools=12 + - clangxx=12 - conda-pack=0.7 - git-lfs=3.2 - git>=2.35.3 # Needed for wildcards on safe.directory From fee64cde7783900d5241e16a865a54a0eac4a951 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 Jul 2022 16:10:42 -0700 Subject: [PATCH 21/41] One more dir to rm --- ci/scripts/jenkins/checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index 832aa60c47..fd59b578d1 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -21,7 +21,7 @@ source ${WORKSPACE}/ci/scripts/jenkins/common.sh fetch_base_branch gpuci_logger "Creating conda env" -rm -rf ${MORPHEUS_ROOT}/.cache/ ${MORPHEUS_ROOT}/build/ +rm -rf ${MORPHEUS_ROOT}/.cache/ ${MORPHEUS_ROOT}/build/ ${WORKSPACE_TMP}/iwyu conda config --add pkgs_dirs /opt/conda/pkgs conda config --env --add channels conda-forge conda config --env --set channel_alias ${CONDA_CHANNEL_ALIAS:-"https://conda.anaconda.org"} From cc2c546c8367e312845a98df405e1fae8b9fb05a Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 Jul 2022 16:23:10 -0700 Subject: [PATCH 22/41] wip --- docker/conda/environments/cuda11.5_ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/conda/environments/cuda11.5_ci.yml b/docker/conda/environments/cuda11.5_ci.yml index 003dff6a5a..1ac61799bc 100644 --- a/docker/conda/environments/cuda11.5_ci.yml +++ b/docker/conda/environments/cuda11.5_ci.yml @@ -21,10 +21,12 @@ dependencies: - bash=5.1.16 # bash >=4.4 is necessary for morpheus cli auto-complete tests to pass. - clang=12 - clang-tools=12 + - clangdev=12 - clangxx=12 - conda-pack=0.7 - git-lfs=3.2 - git>=2.35.3 # Needed for wildcards on safe.directory + - llvmdev=12 - pip - pkg-config=0.29 - sccache=0.3 From c82b2e3b986b1aba37d795cbdd840e7411c4fc5d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 18 Jul 2022 16:41:08 -0700 Subject: [PATCH 23/41] wip --- ci/scripts/jenkins/checks.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index fd59b578d1..e208fcc550 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -48,6 +48,8 @@ cmake -G Ninja \ cmake --build . --parallel ${PARALLEL_LEVEL} --target install +cd ${MORPHEUS_ROOT} + gpuci_logger "Runing Python style checks" ${MORPHEUS_ROOT}/ci/scripts/python_checks.sh From 78ccf1e2f2592df59f489843605d50efee07218b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 19 Jul 2022 07:48:21 -0700 Subject: [PATCH 24/41] Fix directories to check --- ci/scripts/cpp_checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/cpp_checks.sh b/ci/scripts/cpp_checks.sh index e8ca347c38..4c8bca71e6 100755 --- a/ci/scripts/cpp_checks.sh +++ b/ci/scripts/cpp_checks.sh @@ -80,7 +80,7 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then # Include What You Use if [[ "${SKIP_IWYU}" == "" ]]; then - IWYU_DIRS="benchmarks examples python src tools" + IWYU_DIRS="morpheus" NUM_PROC=$(get_num_proc) IWYU_OUTPUT=`${IWYU_TOOL} -p ${BUILD_DIR} -j ${NUM_PROC} ${IWYU_DIRS} 2>&1` IWYU_RETVAL=$? From e8d1389f89a5ec884fb4e71978cbe49b141af8c4 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 19 Jul 2022 08:14:23 -0700 Subject: [PATCH 25/41] IWYU wip --- ci/scripts/run_iwyu_for_ci.sh | 2 +- .../include/morpheus/objects/fiber_queue.hpp | 107 +++++++++--------- .../morpheus/utilities/type_util_detail.hpp | 11 +- morpheus/_lib/src/objects/fiber_queue.cpp | 1 + morpheus/_lib/src/objects/file_types.cpp | 1 + morpheus/_lib/tests/test_type_util_detail.cpp | 3 + 6 files changed, 65 insertions(+), 60 deletions(-) diff --git a/ci/scripts/run_iwyu_for_ci.sh b/ci/scripts/run_iwyu_for_ci.sh index b32b7798ab..d92a960a10 100755 --- a/ci/scripts/run_iwyu_for_ci.sh +++ b/ci/scripts/run_iwyu_for_ci.sh @@ -25,5 +25,5 @@ ${IWYU_TOOL_PY:-iwyu_tool.py} "$@" -- \ -Xiwyu --no_fwd_decls \ -Xiwyu --quoted_includes_first \ -Xiwyu --cxx17ns \ - -Xiwyu --no_comments \ + -Xiwyu --max_line_length=120 \ --driver-mode=g++ diff --git a/morpheus/_lib/include/morpheus/objects/fiber_queue.hpp b/morpheus/_lib/include/morpheus/objects/fiber_queue.hpp index b68a71a5b7..3ec8959b94 100644 --- a/morpheus/_lib/include/morpheus/objects/fiber_queue.hpp +++ b/morpheus/_lib/include/morpheus/objects/fiber_queue.hpp @@ -19,77 +19,78 @@ #include #include - -#include +#include // IWYU pragma: keep #include -#include #include +#include namespace morpheus { - /****** Component public implementations *******************/ - /****** FiberQueue****************************************/ +/****** Component public implementations *******************/ +/****** FiberQueue****************************************/ +/** + * TODO(Documentation) + */ +class FiberQueue +{ + public: + FiberQueue(std::size_t max_size); + /** * TODO(Documentation) */ - class FiberQueue { - public: - FiberQueue(std::size_t max_size); + boost::fibers::channel_op_status put(pybind11::object &&item, bool block = true, float timeout = 0.0); - /** - * TODO(Documentation) - */ - boost::fibers::channel_op_status put(pybind11::object &&item, bool block = true, float timeout = 0.0); - - /** - * TODO(Documentation) - */ - boost::fibers::channel_op_status get(pybind11::object &item, bool block = true, float timeout = 0.0); + /** + * TODO(Documentation) + */ + boost::fibers::channel_op_status get(pybind11::object &item, bool block = true, float timeout = 0.0); - /** - * TODO(Documentation) - */ - void close(); + /** + * TODO(Documentation) + */ + void close(); - /** - * TODO(Documentation) - */ - bool is_closed(); + /** + * TODO(Documentation) + */ + bool is_closed(); - /** - * TODO(Documentation) - */ - void join(); + /** + * TODO(Documentation) + */ + void join(); - private: - boost::fibers::buffered_channel m_queue; - }; + private: + boost::fibers::buffered_channel m_queue; +}; #pragma GCC visibility push(default) - /****** FiberQueueInterfaceProxy *************************/ +/****** FiberQueueInterfaceProxy *************************/ +/** + * @brief Interface proxy, used to insulate python bindings. + */ +struct FiberQueueInterfaceProxy +{ /** - * @brief Interface proxy, used to insulate python bindings. + * @brief Create and initialize a FIberQueue, and return a shared pointer to the result. */ - struct FiberQueueInterfaceProxy { - /** - * @brief Create and initialize a FIberQueue, and return a shared pointer to the result. - */ - static std::shared_ptr init(std::size_t max_size); + static std::shared_ptr init(std::size_t max_size); - /** - * TODO(Documentation) - */ - static void put(morpheus::FiberQueue &self, pybind11::object item, bool block = true, float timeout = 0.0); + /** + * TODO(Documentation) + */ + static void put(morpheus::FiberQueue &self, pybind11::object item, bool block = true, float timeout = 0.0); - /** - * TODO(Documentation) - */ - static pybind11::object get(morpheus::FiberQueue &self, bool block = true, float timeout = 0.0); + /** + * TODO(Documentation) + */ + static pybind11::object get(morpheus::FiberQueue &self, bool block = true, float timeout = 0.0); - /** - * TODO(Documentation) - */ - static void close(morpheus::FiberQueue &self); - }; + /** + * TODO(Documentation) + */ + static void close(morpheus::FiberQueue &self); +}; #pragma GCC visibility pop -} \ No newline at end of file +} // namespace morpheus diff --git a/morpheus/_lib/include/morpheus/utilities/type_util_detail.hpp b/morpheus/_lib/include/morpheus/utilities/type_util_detail.hpp index 90dfe85486..475f271de1 100644 --- a/morpheus/_lib/include/morpheus/utilities/type_util_detail.hpp +++ b/morpheus/_lib/include/morpheus/utilities/type_util_detail.hpp @@ -17,14 +17,13 @@ #pragma once -#include -#include -#include -#include -#include +#include // for CHAR_BIT +#include // for size_t +#include // for int32_t +#include // for string #ifndef NDEBUG -#include + #include #endif namespace morpheus { diff --git a/morpheus/_lib/src/objects/fiber_queue.cpp b/morpheus/_lib/src/objects/fiber_queue.cpp index 2fcf783519..342227f007 100644 --- a/morpheus/_lib/src/objects/fiber_queue.cpp +++ b/morpheus/_lib/src/objects/fiber_queue.cpp @@ -18,6 +18,7 @@ #include "morpheus/objects/fiber_queue.hpp" #include +#include // for gil_scoped_release #include #include diff --git a/morpheus/_lib/src/objects/file_types.cpp b/morpheus/_lib/src/objects/file_types.cpp index e180755613..c7dfe11ed4 100644 --- a/morpheus/_lib/src/objects/file_types.cpp +++ b/morpheus/_lib/src/objects/file_types.cpp @@ -20,6 +20,7 @@ #include "morpheus/utilities/string_util.hpp" #include +#include // ostringstream needed by MORPHEUS_CONCAT_STR #include namespace morpheus { diff --git a/morpheus/_lib/tests/test_type_util_detail.cpp b/morpheus/_lib/tests/test_type_util_detail.cpp index ddf1186d44..964a975d01 100644 --- a/morpheus/_lib/tests/test_type_util_detail.cpp +++ b/morpheus/_lib/tests/test_type_util_detail.cpp @@ -22,6 +22,9 @@ #include // for EXPECT_EQ #include +// work-around for known iwyu issue +// https://github.com/include-what-you-use/include-what-you-use/issues/908 +// IWYU pragma: no_include TEST_CLASS(TypeUtils); From 31d429805fc13f9673e1303bc2f2a16872eba30b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 10 Aug 2022 16:04:13 -0700 Subject: [PATCH 26/41] Add IWYU to cmake --- CMakeLists.txt | 4 +++ cmake/run_iwyu.sh.in | 19 +++++++++++++ cmake/setup_iwyu.cmake | 62 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100755 cmake/run_iwyu.sh.in create mode 100644 cmake/setup_iwyu.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 24ef2b9cf3..2f0cf4014a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,7 @@ option(MORPHEUS_USE_CCACHE "Enable caching compilation results with ccache" OFF) option(MORPHEUS_USE_CLANG_TIDY "Enable running clang-tidy as part of the build process" OFF) option(MORPHEUS_USE_CONDA "Enables finding dependencies via conda instead of vcpkg. Note: This will disable vcpkg. All dependencies must be installed first in the conda environment" OFF) +option(MORPHEUS_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF) set(MORPHEUS_PY_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/wheel" CACHE STRING "Location to install the python directory") set(MORPHEUS_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data") @@ -74,6 +75,9 @@ include(cmake/dependencies.cmake) # - Post dependencies setup -------- include(cmake/setup_compiler.cmake) +# Setup IWYU if enabled +include(cmake/setup_iwyu.cmake) + # To make it easier for CI to find output files, set the default executable suffix to .x if not set if("${CMAKE_EXECUTABLE_SUFFIX}" STREQUAL "") set(CMAKE_EXECUTABLE_SUFFIX ".x") diff --git a/cmake/run_iwyu.sh.in b/cmake/run_iwyu.sh.in new file mode 100755 index 0000000000..b880aa3f60 --- /dev/null +++ b/cmake/run_iwyu.sh.in @@ -0,0 +1,19 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Allows running ccache with options inside of CMake. CMake does not work well +# with setting variables before calling a command +${MORPHEUS_IWYU_PROGRAM} ${MORPHEUS_IWYU_OPTIONS} "$@" diff --git a/cmake/setup_iwyu.cmake b/cmake/setup_iwyu.cmake new file mode 100644 index 0000000000..ab1d40e7f2 --- /dev/null +++ b/cmake/setup_iwyu.cmake @@ -0,0 +1,62 @@ +#============================================================================= +# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + + +function(configure_iwyu) + list(APPEND CMAKE_MESSAGE_CONTEXT "iwyu") + + set(MORPHEUS_IWYU_VERBOSITY "1" CACHE STRING "Set verbosity level for include-what-you-use, 1 is default, 1 only shows recomendations and 11+ prints everything") + + find_program(MORPHEUS_IWYU_PROGRAM "include-what-you-use") + + if(MORPHEUS_IWYU_PROGRAM) + set(MORPHEUS_IWYU_OPTIONS + -Xiwyu; --mapping_file=${PROJECT_SOURCE_DIR}/ci/iwyu/mappings.imp; + -Xiwyu; --max_line_length=120; + -Xiwyu; --verbose=${MORPHEUS_IWYU_VERBOSITY}; + -Xiwyu; --no_fwd_decls; + -Xiwyu; --quoted_includes_first; + -Xiwyu; --cxx17ns; + -Xiwyu --no_comments) + + # Convert these to space separated arguments + string(REPLACE ";" " " MORPHEUS_IWYU_OPTIONS "${MORPHEUS_IWYU_OPTIONS}") + + message(STATUS "Enabling include-what-you-use for Morpheus targets") + + set(IWYU_WRAPPER "${CMAKE_CURRENT_BINARY_DIR}/run_iwyu.sh") + + # Make a ccache runner file with the necessary settings. ccache must already be configured + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/run_iwyu.sh.in" "${IWYU_WRAPPER}") + + if(MORPHEUS_USE_CCACHE) + set(CMAKE_C_INCLUDE_WHAT_YOU_USE "${CMAKE_CURRENT_BINARY_DIR}/run_ccache_prefix.sh;${IWYU_WRAPPER};${CMAKE_C_COMPILER}" PARENT_SCOPE) + set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "${CMAKE_CURRENT_BINARY_DIR}/run_ccache_prefix.sh;${IWYU_WRAPPER};${CMAKE_CXX_COMPILER}" PARENT_SCOPE) + else() + set(CMAKE_C_INCLUDE_WHAT_YOU_USE "${IWYU_WRAPPER}" PARENT_SCOPE) + set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "${IWYU_WRAPPER}" PARENT_SCOPE) + endif() + + else() + message(WARNING "IWYU option MORPHEUS_USE_IWYU is enabled but the include-what-you-use was not found. Check iwyu installation and add the iwyu bin dir to your PATH variable.") + endif(MORPHEUS_IWYU_PROGRAM) +endfunction() + +# Configure IWYU if requested +if(MORPHEUS_USE_IWYU) + configure_iwyu() +endif(MORPHEUS_USE_IWYU) From c5703ef716c8575db5047a37fb99c0e81bd263a3 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 10 Aug 2022 16:59:02 -0700 Subject: [PATCH 27/41] WIP --- cmake/setup_iwyu.cmake | 3 +-- morpheus/_lib/include/morpheus/objects/table_info.hpp | 3 +++ morpheus/_lib/include/morpheus/objects/tensor_object.hpp | 7 +++---- morpheus/_lib/include/morpheus/objects/wrapped_tensor.hpp | 8 -------- morpheus/_lib/include/morpheus/utilities/cudf_util.hpp | 4 ++++ morpheus/_lib/include/morpheus/utilities/tensor_util.hpp | 3 ++- morpheus/_lib/src/objects/dev_mem_info.cpp | 2 ++ morpheus/_lib/src/objects/fiber_queue.cpp | 4 ++++ morpheus/_lib/src/objects/table_info.cpp | 8 +++++++- morpheus/_lib/src/objects/tensor_object.cpp | 1 - morpheus/_lib/src/utilities/cudf_util.cpp | 4 ++++ morpheus/_lib/src/utilities/tensor_util.cpp | 1 - morpheus/_lib/tests/test_matx_util.cpp | 8 ++++++++ morpheus/_lib/tests/test_morpheus.cpp | 2 +- morpheus/_lib/tests/test_multi_slices.cpp | 3 ++- morpheus/_lib/tests/test_tensor.cpp | 8 ++++++-- 16 files changed, 47 insertions(+), 22 deletions(-) diff --git a/cmake/setup_iwyu.cmake b/cmake/setup_iwyu.cmake index ab1d40e7f2..4844e5828a 100644 --- a/cmake/setup_iwyu.cmake +++ b/cmake/setup_iwyu.cmake @@ -30,8 +30,7 @@ function(configure_iwyu) -Xiwyu; --verbose=${MORPHEUS_IWYU_VERBOSITY}; -Xiwyu; --no_fwd_decls; -Xiwyu; --quoted_includes_first; - -Xiwyu; --cxx17ns; - -Xiwyu --no_comments) + -Xiwyu; --cxx17ns;) # Convert these to space separated arguments string(REPLACE ";" " " MORPHEUS_IWYU_OPTIONS "${MORPHEUS_IWYU_OPTIONS}") diff --git a/morpheus/_lib/include/morpheus/objects/table_info.hpp b/morpheus/_lib/include/morpheus/objects/table_info.hpp index cfbf5c3fef..8e874bd40a 100644 --- a/morpheus/_lib/include/morpheus/objects/table_info.hpp +++ b/morpheus/_lib/include/morpheus/objects/table_info.hpp @@ -20,7 +20,10 @@ #include "morpheus/objects/data_table.hpp" #include "morpheus/utilities/type_util_detail.hpp" +#include // for column_view #include +#include // for size_type +#include // for object #include #include diff --git a/morpheus/_lib/include/morpheus/objects/tensor_object.hpp b/morpheus/_lib/include/morpheus/objects/tensor_object.hpp index 766e4a1cc3..a7f9056e1e 100644 --- a/morpheus/_lib/include/morpheus/objects/tensor_object.hpp +++ b/morpheus/_lib/include/morpheus/objects/tensor_object.hpp @@ -26,16 +26,14 @@ #include #include #include -#include // for memory_kind_type #include #include #include // for size_t, byte #include #include -#include -#include -#include +#include // for shared_ptr +#include // IWYU pragma: keep #include #include // for runtime_error #include @@ -43,6 +41,7 @@ #include // IWYU is confusing std::size_t with __gnu_cxx::size_t for some reason // when we define vector +// The header is needed for transform_reduce but IWYU thinks we don't need it // IWYU pragma: no_include namespace morpheus { diff --git a/morpheus/_lib/include/morpheus/objects/wrapped_tensor.hpp b/morpheus/_lib/include/morpheus/objects/wrapped_tensor.hpp index fa845dbe38..409fb75ff9 100644 --- a/morpheus/_lib/include/morpheus/objects/wrapped_tensor.hpp +++ b/morpheus/_lib/include/morpheus/objects/wrapped_tensor.hpp @@ -20,16 +20,8 @@ #include "morpheus/objects/tensor_object.hpp" -#include -#include #include -#include -#include -#include -#include -#include - namespace morpheus { /****** Component public implementations *******************/ /****** TensorObject****************************************/ diff --git a/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp b/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp index 5b87df29d0..b2515648bb 100644 --- a/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp +++ b/morpheus/_lib/include/morpheus/utilities/cudf_util.hpp @@ -17,11 +17,15 @@ #pragma once +#include "morpheus/objects/data_table.hpp" // for IDataTable #include "morpheus/objects/table_info.hpp" +#include // for column_view #include #include +#include // for shared_ptr + namespace morpheus { /****** Component public free function implementations******/ /** diff --git a/morpheus/_lib/include/morpheus/utilities/tensor_util.hpp b/morpheus/_lib/include/morpheus/utilities/tensor_util.hpp index 903855a6e2..8d31a94e70 100644 --- a/morpheus/_lib/include/morpheus/utilities/tensor_util.hpp +++ b/morpheus/_lib/include/morpheus/utilities/tensor_util.hpp @@ -19,10 +19,11 @@ #include "morpheus/objects/tensor_object.hpp" -#include // for copy, min_element & transform +#include // IWYU pragma: keep #include // for ostream #include // for string #include // for vector +// is needed for copy, min_element & transform namespace morpheus { diff --git a/morpheus/_lib/src/objects/dev_mem_info.cpp b/morpheus/_lib/src/objects/dev_mem_info.cpp index 4d92dfe3cc..8b6c3951fb 100644 --- a/morpheus/_lib/src/objects/dev_mem_info.cpp +++ b/morpheus/_lib/src/objects/dev_mem_info.cpp @@ -17,6 +17,8 @@ #include "morpheus/objects/dev_mem_info.hpp" +#include // for uint8_t + namespace morpheus { // Component public implementations // ************ DevMemInfo************************* // diff --git a/morpheus/_lib/src/objects/fiber_queue.cpp b/morpheus/_lib/src/objects/fiber_queue.cpp index 342227f007..274ca9ae06 100644 --- a/morpheus/_lib/src/objects/fiber_queue.cpp +++ b/morpheus/_lib/src/objects/fiber_queue.cpp @@ -20,9 +20,13 @@ #include #include // for gil_scoped_release #include +#include // for PyErr_SetNone #include +#include // for ref, reference_wrapper #include +#include // for ratio needed for std::chrono::duration +#include // for invalid_argument, runtime_error #include namespace morpheus { diff --git a/morpheus/_lib/src/objects/table_info.cpp b/morpheus/_lib/src/objects/table_info.cpp index 0dc7ebd98f..0f1ce95c3e 100644 --- a/morpheus/_lib/src/objects/table_info.cpp +++ b/morpheus/_lib/src/objects/table_info.cpp @@ -24,12 +24,18 @@ #include #include #include +#include #include -#include +#include // IWYU pragma: keep +#include // for find, transform +#include // needed for pybind11::make_tuple +#include // for size_t +#include // for back_insert_iterator, back_inserter #include #include #include +// IWYU pragma: no_include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/objects/tensor_object.cpp b/morpheus/_lib/src/objects/tensor_object.cpp index 9641416a37..b740222e65 100644 --- a/morpheus/_lib/src/objects/tensor_object.cpp +++ b/morpheus/_lib/src/objects/tensor_object.cpp @@ -20,7 +20,6 @@ #include "morpheus/utilities/tensor_util.hpp" #include // for blob -#include // for DataType #include diff --git a/morpheus/_lib/src/utilities/cudf_util.cpp b/morpheus/_lib/src/utilities/cudf_util.cpp index 42dc1a94c9..f1ed1768a6 100644 --- a/morpheus/_lib/src/utilities/cudf_util.cpp +++ b/morpheus/_lib/src/utilities/cudf_util.cpp @@ -22,6 +22,10 @@ #include #include #include +#include + +#include // Needed for logging +#include // for move /** * **************This needs to come last.******************** diff --git a/morpheus/_lib/src/utilities/tensor_util.cpp b/morpheus/_lib/src/utilities/tensor_util.cpp index e06c963e00..0b11d089d5 100644 --- a/morpheus/_lib/src/utilities/tensor_util.cpp +++ b/morpheus/_lib/src/utilities/tensor_util.cpp @@ -20,7 +20,6 @@ #include // for DCHECK_EQ #include // for sort_indexes -#include // for copy // clang-format off // prevent from moving this into the third-party section #include // for make_ostream_joiner diff --git a/morpheus/_lib/tests/test_matx_util.cpp b/morpheus/_lib/tests/test_matx_util.cpp index 50091f5aaa..53d076085c 100644 --- a/morpheus/_lib/tests/test_matx_util.cpp +++ b/morpheus/_lib/tests/test_matx_util.cpp @@ -17,17 +17,25 @@ #include "./test_morpheus.hpp" // IWYU pragma: associated +#include "morpheus/objects/dev_mem_info.hpp" #include "morpheus/utilities/matx_util.hpp" #include "morpheus/utilities/type_util.hpp" #include "morpheus/utilities/type_util_detail.hpp" +#include // for cudaMemcpy, cudaMemcpyDeviceToHost, cudaMemcpyHostToDevice +#include // for column +#include // for column_view #include +#include // for data_type, size_type #include +#include // for cuda_stream_per_thread #include #include +#include // for int64_t, int32_t, uint8_t #include // for std::getenv #include +#include // for shared_ptr, make_shared, unique_ptr #include #include diff --git a/morpheus/_lib/tests/test_morpheus.cpp b/morpheus/_lib/tests/test_morpheus.cpp index 3c41db3f1e..73da7bbca1 100644 --- a/morpheus/_lib/tests/test_morpheus.cpp +++ b/morpheus/_lib/tests/test_morpheus.cpp @@ -18,7 +18,7 @@ #include "test_morpheus.hpp" #include -#include +#include // IWYU pragma: keep cudf::io::table_with_metadata load_table_from_csv(std::string filename) { diff --git a/morpheus/_lib/tests/test_multi_slices.cpp b/morpheus/_lib/tests/test_multi_slices.cpp index d92bdac887..ee94b80701 100644 --- a/morpheus/_lib/tests/test_multi_slices.cpp +++ b/morpheus/_lib/tests/test_multi_slices.cpp @@ -20,12 +20,13 @@ #include #include #include -#include #include +#include #include #include #include +#include // for unique_ptr #include TEST_CLASS(Masking); diff --git a/morpheus/_lib/tests/test_tensor.cpp b/morpheus/_lib/tests/test_tensor.cpp index 18642e0c4a..195ae7cdaf 100644 --- a/morpheus/_lib/tests/test_tensor.cpp +++ b/morpheus/_lib/tests/test_tensor.cpp @@ -17,11 +17,15 @@ #include "./test_morpheus.hpp" // IWYU pragma: associated -#include // for TensorUtils, TensorUtils::shape_type +#include "morpheus/utilities/tensor_util.hpp" // for TensorUtils, TensorUtils::shape_type +#include "morpheus/objects/tensor_object.hpp" // for TensorIndex #include // for AssertionResult, SuiteApiResolver, TestInfo, EXPECT_TRUE, Message, TEST_F, Test, TestFactoryImpl, TestPartResult -#include // for allocator, operator==, basic_string, string +#include // for size_t +#include // for allocator, operator==, basic_string, string +#include // for vector +// IWYU pragma: no_include "morpheus/utilities/string_util.hpp" using namespace morpheus; From d846ac76c0e7e86623bdd77a9c68c0712c131676 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 15 Aug 2022 10:35:44 -0700 Subject: [PATCH 28/41] WIP --- ci/iwyu/mappings.imp | 18 ++++++++++++------ .../messages/memory/inference_memory.hpp | 3 --- .../messages/memory/inference_memory_fil.hpp | 7 ++----- .../messages/memory/inference_memory_nlp.hpp | 2 +- .../messages/memory/response_memory.hpp | 5 ++--- .../messages/memory/response_memory_probs.hpp | 3 --- .../morpheus/messages/memory/tensor_memory.hpp | 8 ++++---- .../_lib/include/morpheus/messages/meta.hpp | 5 ++++- .../_lib/include/morpheus/messages/multi.hpp | 11 +++-------- .../morpheus/messages/multi_inference.hpp | 3 ++- .../morpheus/messages/multi_inference_fil.hpp | 5 +++-- .../morpheus/messages/multi_inference_nlp.hpp | 6 +++--- .../morpheus/messages/multi_response.hpp | 5 ++--- .../morpheus/messages/multi_response_probs.hpp | 3 +-- .../morpheus/objects/python_data_table.hpp | 4 +++- .../include/morpheus/objects/rmm_tensor.hpp | 6 ++---- .../_lib/include/morpheus/objects/tensor.hpp | 8 +------- .../include/morpheus/utilities/table_util.hpp | 12 ++---------- .../src/messages/memory/inference_memory.cpp | 4 +--- .../messages/memory/inference_memory_fil.cpp | 6 +++--- .../messages/memory/inference_memory_nlp.cpp | 7 +++++-- .../src/messages/memory/response_memory.cpp | 4 +--- .../messages/memory/response_memory_probs.cpp | 5 +++-- .../_lib/src/messages/memory/tensor_memory.cpp | 6 ------ morpheus/_lib/src/messages/meta.cpp | 1 - morpheus/_lib/src/messages/multi.cpp | 1 + morpheus/_lib/src/messages/multi_inference.cpp | 5 ++++- .../_lib/src/messages/multi_inference_fil.cpp | 2 -- .../_lib/src/messages/multi_inference_nlp.cpp | 2 -- morpheus/_lib/src/messages/multi_response.cpp | 3 ++- .../_lib/src/messages/multi_response_probs.cpp | 2 -- morpheus/_lib/src/objects/fiber_queue.cpp | 3 +-- .../_lib/src/objects/python_data_table.cpp | 3 ++- morpheus/_lib/src/objects/rmm_tensor.cpp | 13 ++++++++++--- morpheus/_lib/src/objects/table_info.cpp | 7 +++---- morpheus/_lib/src/objects/tensor.cpp | 3 +++ morpheus/_lib/src/objects/wrapped_tensor.cpp | 4 +++- morpheus/_lib/src/python_modules/common.cpp | 1 + morpheus/_lib/src/python_modules/messages.cpp | 1 + morpheus/_lib/src/stages/add_scores.cpp | 1 + morpheus/_lib/src/utilities/cudf_util.cpp | 5 ++--- morpheus/_lib/src/utilities/table_util.cpp | 4 +++- 42 files changed, 97 insertions(+), 110 deletions(-) diff --git a/ci/iwyu/mappings.imp b/ci/iwyu/mappings.imp index 24caf7c04b..75a77bcf13 100644 --- a/ci/iwyu/mappings.imp +++ b/ci/iwyu/mappings.imp @@ -42,6 +42,7 @@ { "symbol": ["__gnu_cxx::__enable_if::__type", "private", "", "public"] }, { "symbol": ["std::__success_type>>::type" , "private", "", "public"] }, { "symbol": ["__cxxabiv1::__forced_unwind", "private", "", "public"] }, +{ "symbol": ["std::filesystem", "private", "", "public"] }, # boost { "symbol": ["__forced_unwind", "private", "", "public"] }, @@ -66,20 +67,25 @@ { "symbol": ["pybind11", "private", "", "public"] }, { "symbol": ["pybind11", "private", "", "public"] }, +{ "symbol": ["_Py_IsFinalizing", "private", "", "public"] }, { "symbol": ["PYBIND11_MODULE", "private", "", "public"] }, -{ "symbol": ["PySequence_GetItem", "private", "", "public"] }, { "symbol": ["PyExc_ImportError", "private", "", "public"] }, +{ "symbol": ["PyErr_SetNone", "private", "", "public"] }, { "symbol": ["PyErr_SetObject", "private", "", "public"] }, { "symbol": ["PyExc_StopIteration", "private", "", "public"] }, -{ "symbol": ["_Py_IsFinalizing", "private", "", "public"] }, -{ "symbol": ["pybind11::detail::str_attr_accessor", "private", "", "public"] }, +{ "symbol": ["PyGILState_Check", "private", "", "public"] }, +{ "symbol": ["PyObject", "private", "", "public"] }, +{ "symbol": ["PySequence_GetItem", "private", "", "public"] }, +{ "symbol": ["pybind11::arg", "private", "", "public"] }, +{ "symbol": ["pybind11::detail::get_type_info", "private", "", "public"] }, +{ "symbol": ["pybind11::detail::key_error", "private", "", "public"] }, { "symbol": ["pybind11::detail::overload_cast_impl", "private", "", "public"] }, +{ "symbol": ["pybind11::detail::str_attr_accessor", "private", "", "public"] }, +{ "symbol": ["pybind11::key_error", "private", "", "public"] }, { "symbol": ["pybind11::overload_cast", "private", "", "public"] }, -{ "symbol": ["pybind11::stop_iteration", "private", "", "public"] }, { "symbol": ["pybind11::return_value_policy", "private", "", "public"] }, { "symbol": ["pybind11::return_value_policy::reference_internal", "private", "", "public"] }, -{ "symbol": ["pybind11::detail::get_type_info", "private", "", "public"] }, -{ "symbol": ["PyGILState_Check", "private", "", "public"] }, +{ "symbol": ["pybind11::stop_iteration", "private", "", "public"] }, # spdlog { "symbol": ["spdlog::details::file_helper::~file_helper", "private", "", "public"] }, diff --git a/morpheus/_lib/include/morpheus/messages/memory/inference_memory.hpp b/morpheus/_lib/include/morpheus/messages/memory/inference_memory.hpp index 2f91c095fe..966d225c90 100644 --- a/morpheus/_lib/include/morpheus/messages/memory/inference_memory.hpp +++ b/morpheus/_lib/include/morpheus/messages/memory/inference_memory.hpp @@ -18,11 +18,8 @@ #pragma once #include "morpheus/messages/memory/tensor_memory.hpp" -#include "morpheus/objects/tensor.hpp" -#include "morpheus/objects/tensor_object.hpp" #include -#include #include namespace morpheus { diff --git a/morpheus/_lib/include/morpheus/messages/memory/inference_memory_fil.hpp b/morpheus/_lib/include/morpheus/messages/memory/inference_memory_fil.hpp index 73618942e9..8f9b979e3c 100644 --- a/morpheus/_lib/include/morpheus/messages/memory/inference_memory_fil.hpp +++ b/morpheus/_lib/include/morpheus/messages/memory/inference_memory_fil.hpp @@ -18,15 +18,12 @@ #pragma once #include "morpheus/messages/memory/inference_memory.hpp" -#include "morpheus/objects/python_data_table.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/objects/tensor_object.hpp" -#include -#include +#include // for size_type +#include // for object #include -#include #include #include diff --git a/morpheus/_lib/include/morpheus/messages/memory/inference_memory_nlp.hpp b/morpheus/_lib/include/morpheus/messages/memory/inference_memory_nlp.hpp index eb3713903f..522d1bed57 100644 --- a/morpheus/_lib/include/morpheus/messages/memory/inference_memory_nlp.hpp +++ b/morpheus/_lib/include/morpheus/messages/memory/inference_memory_nlp.hpp @@ -18,9 +18,9 @@ #pragma once #include "morpheus/messages/memory/inference_memory.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/objects/tensor_object.hpp" +#include // for size_type #include #include diff --git a/morpheus/_lib/include/morpheus/messages/memory/response_memory.hpp b/morpheus/_lib/include/morpheus/messages/memory/response_memory.hpp index b7a6778763..535130ebc4 100644 --- a/morpheus/_lib/include/morpheus/messages/memory/response_memory.hpp +++ b/morpheus/_lib/include/morpheus/messages/memory/response_memory.hpp @@ -18,13 +18,12 @@ #pragma once #include "morpheus/messages/memory/tensor_memory.hpp" -#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" // for TensorObject -#include #include +#include // for size_t #include -#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/messages/memory/response_memory_probs.hpp b/morpheus/_lib/include/morpheus/messages/memory/response_memory_probs.hpp index 765ce5479b..62c716e952 100644 --- a/morpheus/_lib/include/morpheus/messages/memory/response_memory_probs.hpp +++ b/morpheus/_lib/include/morpheus/messages/memory/response_memory_probs.hpp @@ -18,16 +18,13 @@ #pragma once #include "morpheus/messages/memory/response_memory.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/objects/tensor_object.hpp" -#include #include #include #include #include -#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/messages/memory/tensor_memory.hpp b/morpheus/_lib/include/morpheus/messages/memory/tensor_memory.hpp index 569890bf75..7f6fe90abc 100644 --- a/morpheus/_lib/include/morpheus/messages/memory/tensor_memory.hpp +++ b/morpheus/_lib/include/morpheus/messages/memory/tensor_memory.hpp @@ -17,12 +17,12 @@ #pragma once -#include "morpheus/objects/tensor.hpp" - -#include -#include +#include "morpheus/objects/tensor_object.hpp" // for TensorIndex, TensorObject +#include // for size_t +#include #include +#include // for pair #include namespace morpheus { diff --git a/morpheus/_lib/include/morpheus/messages/meta.hpp b/morpheus/_lib/include/morpheus/messages/meta.hpp index 2cc920dcd2..2a2edf3ea3 100644 --- a/morpheus/_lib/include/morpheus/messages/meta.hpp +++ b/morpheus/_lib/include/morpheus/messages/meta.hpp @@ -17,11 +17,14 @@ #pragma once +#include "morpheus/objects/data_table.hpp" // for IDataTable #include "morpheus/objects/table_info.hpp" #include -#include +#include // for size_type +#include +#include // for size_t #include #include diff --git a/morpheus/_lib/include/morpheus/messages/multi.hpp b/morpheus/_lib/include/morpheus/messages/multi.hpp index 0302c1e6f7..a40d90ff4c 100644 --- a/morpheus/_lib/include/morpheus/messages/multi.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi.hpp @@ -19,19 +19,14 @@ #include "morpheus/messages/meta.hpp" #include "morpheus/objects/table_info.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/objects/tensor_object.hpp" -#include -#include -#include #include -#include -#include -#include +#include // for DCHECK_NOTNULL #include -#include +#include // IWYU pragma: keep +#include // for size_t #include #include #include // for pair diff --git a/morpheus/_lib/include/morpheus/messages/multi_inference.hpp b/morpheus/_lib/include/morpheus/messages/multi_inference.hpp index 71522a7ef6..13c3b9ce37 100644 --- a/morpheus/_lib/include/morpheus/messages/multi_inference.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi_inference.hpp @@ -20,7 +20,6 @@ #include "morpheus/messages/memory/inference_memory.hpp" #include "morpheus/messages/meta.hpp" #include "morpheus/messages/multi.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/objects/tensor_object.hpp" #include @@ -29,6 +28,8 @@ #include #include #include +#include // for pair +#include namespace morpheus { /****** Component public implementations********************/ diff --git a/morpheus/_lib/include/morpheus/messages/multi_inference_fil.hpp b/morpheus/_lib/include/morpheus/messages/multi_inference_fil.hpp index 7468d82a32..73bb8b5068 100644 --- a/morpheus/_lib/include/morpheus/messages/multi_inference_fil.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi_inference_fil.hpp @@ -17,13 +17,14 @@ #pragma once +#include "morpheus/messages/memory/inference_memory.hpp" // for InferenceMemory +#include "morpheus/messages/meta.hpp" // for MessageMeta #include "morpheus/messages/multi_inference.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/objects/tensor_object.hpp" -#include #include +#include // for size_t #include namespace morpheus { diff --git a/morpheus/_lib/include/morpheus/messages/multi_inference_nlp.hpp b/morpheus/_lib/include/morpheus/messages/multi_inference_nlp.hpp index 4d09cef3dd..debbd6b384 100644 --- a/morpheus/_lib/include/morpheus/messages/multi_inference_nlp.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi_inference_nlp.hpp @@ -17,16 +17,16 @@ #pragma once +#include "morpheus/messages/memory/inference_memory.hpp" // for InferenceMemory +#include "morpheus/messages/meta.hpp" // for MessageMeta #include "morpheus/messages/multi_inference.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/objects/tensor_object.hpp" #include -#include +#include // for object #include #include -#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/messages/multi_response.hpp b/morpheus/_lib/include/morpheus/messages/multi_response.hpp index 32fb59e9da..1730b09718 100644 --- a/morpheus/_lib/include/morpheus/messages/multi_response.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi_response.hpp @@ -20,10 +20,7 @@ #include "morpheus/messages/memory/response_memory.hpp" #include "morpheus/messages/meta.hpp" #include "morpheus/messages/multi.hpp" -#include "morpheus/objects/table_info.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/objects/tensor_object.hpp" -#include "morpheus/utilities/table_util.hpp" #include #include @@ -31,6 +28,8 @@ #include #include #include +#include // for pair +#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/messages/multi_response_probs.hpp b/morpheus/_lib/include/morpheus/messages/multi_response_probs.hpp index 7d0d744725..67e4164259 100644 --- a/morpheus/_lib/include/morpheus/messages/multi_response_probs.hpp +++ b/morpheus/_lib/include/morpheus/messages/multi_response_probs.hpp @@ -21,14 +21,13 @@ #include "morpheus/messages/meta.hpp" #include "morpheus/messages/multi.hpp" #include "morpheus/messages/multi_response.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/objects/tensor_object.hpp" #include #include +#include // for size_t #include -#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/objects/python_data_table.hpp b/morpheus/_lib/include/morpheus/objects/python_data_table.hpp index 0e5ec93f16..bb78d346fa 100644 --- a/morpheus/_lib/include/morpheus/objects/python_data_table.hpp +++ b/morpheus/_lib/include/morpheus/objects/python_data_table.hpp @@ -17,9 +17,11 @@ #pragma once +#include "morpheus/objects/data_table.hpp" // for IDataTable #include "morpheus/objects/table_info.hpp" -#include +#include // for size_type +#include // for object namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/objects/rmm_tensor.hpp b/morpheus/_lib/include/morpheus/objects/rmm_tensor.hpp index 81a1f469c8..bdf61f720c 100644 --- a/morpheus/_lib/include/morpheus/objects/rmm_tensor.hpp +++ b/morpheus/_lib/include/morpheus/objects/rmm_tensor.hpp @@ -19,14 +19,12 @@ #include "morpheus/objects/tensor_object.hpp" #include "morpheus/utilities/type_util.hpp" +#include "morpheus/utilities/type_util_detail.hpp" // for DataType -#include #include -#include -#include +#include // for size_t #include -#include #include // for pair #include diff --git a/morpheus/_lib/include/morpheus/objects/tensor.hpp b/morpheus/_lib/include/morpheus/objects/tensor.hpp index ee73ce3f3e..e66752c7b7 100644 --- a/morpheus/_lib/include/morpheus/objects/tensor.hpp +++ b/morpheus/_lib/include/morpheus/objects/tensor.hpp @@ -17,18 +17,12 @@ #pragma once -#include "morpheus/objects/rmm_tensor.hpp" #include "morpheus/objects/tensor_object.hpp" -#include "morpheus/utilities/matx_util.hpp" #include "morpheus/utilities/type_util.hpp" -#include -#include -#include -#include #include -#include +#include // for size_t #include #include #include diff --git a/morpheus/_lib/include/morpheus/utilities/table_util.hpp b/morpheus/_lib/include/morpheus/utilities/table_util.hpp index 656c53d270..9e31f2ca88 100644 --- a/morpheus/_lib/include/morpheus/utilities/table_util.hpp +++ b/morpheus/_lib/include/morpheus/utilities/table_util.hpp @@ -16,17 +16,9 @@ */ #include -#include -#include -#include +#include // IWYU pragma: keep -#include -#include -#include - -#include -#include -#include +#include #pragma once diff --git a/morpheus/_lib/src/messages/memory/inference_memory.cpp b/morpheus/_lib/src/messages/memory/inference_memory.cpp index b2e8ba4513..3306c09b53 100644 --- a/morpheus/_lib/src/messages/memory/inference_memory.cpp +++ b/morpheus/_lib/src/messages/memory/inference_memory.cpp @@ -17,10 +17,8 @@ #include "morpheus/messages/memory/inference_memory.hpp" -#include - #include -#include +#include // for move namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/messages/memory/inference_memory_fil.cpp b/morpheus/_lib/src/messages/memory/inference_memory_fil.cpp index 2d4fd70140..f6e0fc8162 100644 --- a/morpheus/_lib/src/messages/memory/inference_memory_fil.cpp +++ b/morpheus/_lib/src/messages/memory/inference_memory_fil.cpp @@ -18,17 +18,17 @@ #include "morpheus/messages/memory/inference_memory_fil.hpp" #include "morpheus/messages/memory/inference_memory.hpp" -#include "morpheus/objects/tensor.hpp" +#include "morpheus/messages/memory/tensor_memory.hpp" #include "morpheus/utilities/cupy_util.hpp" -#include #include #include +#include // this->tensors is a map #include +#include // for runtime_error #include #include -#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/messages/memory/inference_memory_nlp.cpp b/morpheus/_lib/src/messages/memory/inference_memory_nlp.cpp index 5b756a588f..ffa78d8d8f 100644 --- a/morpheus/_lib/src/messages/memory/inference_memory_nlp.cpp +++ b/morpheus/_lib/src/messages/memory/inference_memory_nlp.cpp @@ -18,13 +18,16 @@ #include "morpheus/messages/memory/inference_memory_nlp.hpp" #include "morpheus/messages/memory/inference_memory.hpp" -#include "morpheus/objects/tensor.hpp" +#include "morpheus/messages/memory/tensor_memory.hpp" #include "morpheus/utilities/cupy_util.hpp" -#include +#include // for size_type #include #include +#include // this->tensors is a map +#include // for runtime_error +#include // for move, pair namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/messages/memory/response_memory.cpp b/morpheus/_lib/src/messages/memory/response_memory.cpp index 8dd7b1bb87..9af4a4b882 100644 --- a/morpheus/_lib/src/messages/memory/response_memory.cpp +++ b/morpheus/_lib/src/messages/memory/response_memory.cpp @@ -17,14 +17,12 @@ #include "morpheus/messages/memory/response_memory.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/utilities/cupy_util.hpp" -#include #include #include -#include +#include // for move namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/messages/memory/response_memory_probs.cpp b/morpheus/_lib/src/messages/memory/response_memory_probs.cpp index d514b3928f..575182788e 100644 --- a/morpheus/_lib/src/messages/memory/response_memory_probs.cpp +++ b/morpheus/_lib/src/messages/memory/response_memory_probs.cpp @@ -18,15 +18,16 @@ #include "morpheus/messages/memory/response_memory_probs.hpp" #include "morpheus/messages/memory/response_memory.hpp" -#include "morpheus/objects/tensor.hpp" +#include "morpheus/messages/memory/tensor_memory.hpp" #include "morpheus/utilities/cupy_util.hpp" -#include #include #include #include +#include // this->tensors is a map #include +#include // for runtime_error #include namespace morpheus { diff --git a/morpheus/_lib/src/messages/memory/tensor_memory.cpp b/morpheus/_lib/src/messages/memory/tensor_memory.cpp index 0e5dbb8031..01c6377edc 100644 --- a/morpheus/_lib/src/messages/memory/tensor_memory.cpp +++ b/morpheus/_lib/src/messages/memory/tensor_memory.cpp @@ -17,12 +17,6 @@ #include "morpheus/messages/memory/tensor_memory.hpp" -#include "morpheus/objects/tensor.hpp" -#include "morpheus/utilities/cupy_util.hpp" - -#include -#include - #include #include diff --git a/morpheus/_lib/src/messages/meta.cpp b/morpheus/_lib/src/messages/meta.cpp index a7f3217483..1df4e162f3 100644 --- a/morpheus/_lib/src/messages/meta.cpp +++ b/morpheus/_lib/src/messages/meta.cpp @@ -28,7 +28,6 @@ #include #include -#include namespace morpheus { /******* Component-private Classes *********************/ diff --git a/morpheus/_lib/src/messages/multi.cpp b/morpheus/_lib/src/messages/multi.cpp index 52512e0509..5b6661e6cc 100644 --- a/morpheus/_lib/src/messages/multi.cpp +++ b/morpheus/_lib/src/messages/multi.cpp @@ -19,6 +19,7 @@ #include "morpheus/messages/meta.hpp" #include "morpheus/objects/table_info.hpp" +#include "morpheus/utilities/type_util.hpp" #include #include diff --git a/morpheus/_lib/src/messages/multi_inference.cpp b/morpheus/_lib/src/messages/multi_inference.cpp index 24c4d8a013..c5fb6379e8 100644 --- a/morpheus/_lib/src/messages/multi_inference.cpp +++ b/morpheus/_lib/src/messages/multi_inference.cpp @@ -18,15 +18,18 @@ #include "morpheus/messages/multi_inference.hpp" #include "morpheus/messages/memory/inference_memory.hpp" +#include "morpheus/messages/memory/tensor_memory.hpp" // for TensorMemory::tensor_map_t #include "morpheus/messages/meta.hpp" #include "morpheus/messages/multi.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/utilities/cupy_util.hpp" #include +#include #include +#include // for int32_t #include +#include // needed for logging #include #include diff --git a/morpheus/_lib/src/messages/multi_inference_fil.cpp b/morpheus/_lib/src/messages/multi_inference_fil.cpp index 676258a8b9..e22a1650a3 100644 --- a/morpheus/_lib/src/messages/multi_inference_fil.cpp +++ b/morpheus/_lib/src/messages/multi_inference_fil.cpp @@ -20,13 +20,11 @@ #include "morpheus/messages/memory/inference_memory.hpp" #include "morpheus/messages/meta.hpp" #include "morpheus/messages/multi_inference.hpp" -#include "morpheus/objects/tensor.hpp" #include #include #include -#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/messages/multi_inference_nlp.cpp b/morpheus/_lib/src/messages/multi_inference_nlp.cpp index e737ca4e0a..16945b6702 100644 --- a/morpheus/_lib/src/messages/multi_inference_nlp.cpp +++ b/morpheus/_lib/src/messages/multi_inference_nlp.cpp @@ -20,7 +20,6 @@ #include "morpheus/messages/memory/inference_memory.hpp" #include "morpheus/messages/meta.hpp" #include "morpheus/messages/multi_inference.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/utilities/cupy_util.hpp" #include @@ -28,7 +27,6 @@ #include #include -#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/messages/multi_response.cpp b/morpheus/_lib/src/messages/multi_response.cpp index 59e92900d4..7ffbf59469 100644 --- a/morpheus/_lib/src/messages/multi_response.cpp +++ b/morpheus/_lib/src/messages/multi_response.cpp @@ -18,9 +18,9 @@ #include "morpheus/messages/multi_response.hpp" #include "morpheus/messages/memory/response_memory.hpp" +#include "morpheus/messages/memory/tensor_memory.hpp" // for TensorMemory::tensor_map_t #include "morpheus/messages/meta.hpp" #include "morpheus/messages/multi.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/objects/tensor_object.hpp" #include "morpheus/utilities/cupy_util.hpp" @@ -30,6 +30,7 @@ #include #include +#include // needed for logging #include #include diff --git a/morpheus/_lib/src/messages/multi_response_probs.cpp b/morpheus/_lib/src/messages/multi_response_probs.cpp index 051380caa2..cdc1171f02 100644 --- a/morpheus/_lib/src/messages/multi_response_probs.cpp +++ b/morpheus/_lib/src/messages/multi_response_probs.cpp @@ -18,7 +18,6 @@ #include "morpheus/messages/multi_response_probs.hpp" #include "morpheus/messages/meta.hpp" -#include "morpheus/objects/tensor.hpp" #include "morpheus/utilities/cupy_util.hpp" #include @@ -26,7 +25,6 @@ #include #include -#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/objects/fiber_queue.cpp b/morpheus/_lib/src/objects/fiber_queue.cpp index 274ca9ae06..c5be4fcffe 100644 --- a/morpheus/_lib/src/objects/fiber_queue.cpp +++ b/morpheus/_lib/src/objects/fiber_queue.cpp @@ -19,8 +19,7 @@ #include #include // for gil_scoped_release -#include -#include // for PyErr_SetNone +#include #include #include // for ref, reference_wrapper diff --git a/morpheus/_lib/src/objects/python_data_table.cpp b/morpheus/_lib/src/objects/python_data_table.cpp index 7ab871e965..a3442bfac0 100644 --- a/morpheus/_lib/src/objects/python_data_table.cpp +++ b/morpheus/_lib/src/objects/python_data_table.cpp @@ -21,8 +21,9 @@ #include "morpheus/utilities/cudf_util.hpp" #include +#include // for object::cast #include -#include +#include #include diff --git a/morpheus/_lib/src/objects/rmm_tensor.cpp b/morpheus/_lib/src/objects/rmm_tensor.cpp index d27e8fa273..79bfcf28a8 100644 --- a/morpheus/_lib/src/objects/rmm_tensor.cpp +++ b/morpheus/_lib/src/objects/rmm_tensor.cpp @@ -17,18 +17,25 @@ #include "morpheus/objects/rmm_tensor.hpp" +#include "morpheus/objects/dev_mem_info.hpp" // for DevMemInfo #include "morpheus/objects/tensor_object.hpp" #include "morpheus/utilities/matx_util.hpp" #include "morpheus/utilities/tensor_util.hpp" // for get_element_stride #include "morpheus/utilities/type_util.hpp" -#include -#include +#include // for cudaMemcpy, cudaMemcpy2D, cudaMemcpyDeviceToDevice +#include // for DCHECK_LT, COMPACT_GOOGLE_LOG_FATAL, DCHECK, DCHECK_EQ, LogMessageFatal +#include // for cuda_stream_per_thread #include +#include // for SRF_CHECK_CUDA +#include // for copy, transform #include +#include // for multiplies, plus, minus +#include // for back_insert_iterator, back_inserter #include -#include +#include // for accumulate, transform_reduce +#include // needed for logging #include namespace morpheus { diff --git a/morpheus/_lib/src/objects/table_info.cpp b/morpheus/_lib/src/objects/table_info.cpp index 0f1ce95c3e..8f3b3d7c81 100644 --- a/morpheus/_lib/src/objects/table_info.cpp +++ b/morpheus/_lib/src/objects/table_info.cpp @@ -23,10 +23,9 @@ #include #include #include -#include -#include -#include -#include // IWYU pragma: keep +#include // for gil_scoped_acquire +#include // IWYU pragma: keep +#include // IWYU pragma: keep #include // for find, transform #include // needed for pybind11::make_tuple diff --git a/morpheus/_lib/src/objects/tensor.cpp b/morpheus/_lib/src/objects/tensor.cpp index 632977665d..0d0f56d651 100644 --- a/morpheus/_lib/src/objects/tensor.cpp +++ b/morpheus/_lib/src/objects/tensor.cpp @@ -21,11 +21,14 @@ #include "morpheus/objects/tensor_object.hpp" #include "morpheus/utilities/type_util.hpp" +#include // for cudaMemcpy, cudaMemcpyDeviceToHost #include +#include // for SRF_CHECK_CUDA #include #include #include +#include // for move #include namespace morpheus { diff --git a/morpheus/_lib/src/objects/wrapped_tensor.cpp b/morpheus/_lib/src/objects/wrapped_tensor.cpp index 7b314c4eb5..b2cf9bc59b 100644 --- a/morpheus/_lib/src/objects/wrapped_tensor.cpp +++ b/morpheus/_lib/src/objects/wrapped_tensor.cpp @@ -22,7 +22,9 @@ #include #include -#include +#include // needed for make_tuple +#include // for uintptr_t +#include // get_shape & get_stride return vectors namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/python_modules/common.cpp b/morpheus/_lib/src/python_modules/common.cpp index ca2ab266fe..b90a59b91e 100644 --- a/morpheus/_lib/src/python_modules/common.cpp +++ b/morpheus/_lib/src/python_modules/common.cpp @@ -16,6 +16,7 @@ */ #include "morpheus/objects/fiber_queue.hpp" +#include "morpheus/objects/tensor_object.hpp" // for TensorObject #include "morpheus/objects/wrapped_tensor.hpp" #include "morpheus/utilities/cudf_util.hpp" diff --git a/morpheus/_lib/src/python_modules/messages.cpp b/morpheus/_lib/src/python_modules/messages.cpp index 019e4204b0..0ecd9d3c46 100644 --- a/morpheus/_lib/src/python_modules/messages.cpp +++ b/morpheus/_lib/src/python_modules/messages.cpp @@ -38,6 +38,7 @@ #include #include +#include #include #include diff --git a/morpheus/_lib/src/stages/add_scores.cpp b/morpheus/_lib/src/stages/add_scores.cpp index 0b23890382..f6017ddf39 100644 --- a/morpheus/_lib/src/stages/add_scores.cpp +++ b/morpheus/_lib/src/stages/add_scores.cpp @@ -17,6 +17,7 @@ #include "morpheus/stages/add_scores.hpp" +#include "morpheus/objects/tensor.hpp" #include "morpheus/utilities/matx_util.hpp" #include diff --git a/morpheus/_lib/src/utilities/cudf_util.cpp b/morpheus/_lib/src/utilities/cudf_util.cpp index f1ed1768a6..8dada42067 100644 --- a/morpheus/_lib/src/utilities/cudf_util.cpp +++ b/morpheus/_lib/src/utilities/cudf_util.cpp @@ -19,10 +19,9 @@ #include "morpheus/objects/table_info.hpp" -#include +#include // IWYU pragma: keep #include -#include -#include +#include #include // Needed for logging #include // for move diff --git a/morpheus/_lib/src/utilities/table_util.cpp b/morpheus/_lib/src/utilities/table_util.cpp index 0cbf327088..777569c1c0 100644 --- a/morpheus/_lib/src/utilities/table_util.cpp +++ b/morpheus/_lib/src/utilities/table_util.cpp @@ -20,9 +20,11 @@ #include #include #include +#include #include -#include +#include // needed for logging +#include // for runtime_error namespace fs = std::filesystem; namespace py = pybind11; From 9477b6d9dc8b3649785e99204ec0780061af302b Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 15 Aug 2022 11:32:16 -0700 Subject: [PATCH 29/41] wip --- ci/iwyu/mappings.imp | 32 +++++++++++++++++++ .../_lib/cmake/libraries/cudf_helpers.cmake | 11 +++++++ .../_lib/include/morpheus/io/serializers.hpp | 1 - morpheus/_lib/src/io/serializers.cpp | 9 ++++-- morpheus/_lib/src/messages/multi.cpp | 13 +++++++- morpheus/_lib/src/python_modules/messages.cpp | 9 ++++-- morpheus/_lib/src/python_modules/stages.cpp | 6 ++++ .../_lib/src/stages/add_classification.cpp | 1 + morpheus/_lib/src/stages/add_scores.cpp | 9 ++++++ morpheus/_lib/src/stages/deserialize.cpp | 3 +- morpheus/_lib/src/stages/preprocess_fil.cpp | 2 ++ morpheus/_lib/src/stages/preprocess_nlp.cpp | 1 + morpheus/_lib/src/stages/triton_inference.cpp | 1 + morpheus/_lib/src/utilities/cupy_util.cpp | 13 +++++++- 14 files changed, 102 insertions(+), 9 deletions(-) diff --git a/ci/iwyu/mappings.imp b/ci/iwyu/mappings.imp index 75a77bcf13..bc9b6647bd 100644 --- a/ci/iwyu/mappings.imp +++ b/ci/iwyu/mappings.imp @@ -37,6 +37,37 @@ # Protobuf { "include": [ "", private, "", "public" ] }, +# rxcpp +# Hide includes that are exported by +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-util.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-predef.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-subscription.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-observer.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-scheduler.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-subscriber.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-notification.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-coordination.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-sources.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-subjects.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-operators.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-observable.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-connectable_observable.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "\"rxcpp/rx-grouped_observable.hpp\"", private, "", "public" ] }, + ## Symbol mappings # stdlib { "symbol": ["__gnu_cxx::__enable_if::__type", "private", "", "public"] }, @@ -84,6 +115,7 @@ { "symbol": ["pybind11::key_error", "private", "", "public"] }, { "symbol": ["pybind11::overload_cast", "private", "", "public"] }, { "symbol": ["pybind11::return_value_policy", "private", "", "public"] }, +{ "symbol": ["pybind11::return_value_policy::move", "private", "", "public"] }, { "symbol": ["pybind11::return_value_policy::reference_internal", "private", "", "public"] }, { "symbol": ["pybind11::stop_iteration", "private", "", "public"] }, diff --git a/morpheus/_lib/cmake/libraries/cudf_helpers.cmake b/morpheus/_lib/cmake/libraries/cudf_helpers.cmake index f95dd47e08..ffe4b965fc 100644 --- a/morpheus/_lib/cmake/libraries/cudf_helpers.cmake +++ b/morpheus/_lib/cmake/libraries/cudf_helpers.cmake @@ -33,6 +33,17 @@ morpheus_add_cython_libraries( # The `style_checks` target allows these to be generated without a full build of Morpheus. add_dependencies(style_checks ${cudf_helpers_target}) + +# Disable clang-tidy and IWYU for cython generated code +set_target_properties( + ${cudf_helpers_target} + PROPERTIES + CXX_CLANG_TIDY "" + C_INCLUDE_WHAT_YOU_USE "" + CXX_INCLUDE_WHAT_YOU_USE "" + EXPORT_COMPILE_COMMANDS OFF +) + if (MORPHEUS_PYTHON_INPLACE_BUILD) inplace_build_copy(${cudf_helpers_target} ${MORPHEUS_LIB_ROOT}) endif() diff --git a/morpheus/_lib/include/morpheus/io/serializers.hpp b/morpheus/_lib/include/morpheus/io/serializers.hpp index 2a8f7ee265..79f698d911 100644 --- a/morpheus/_lib/include/morpheus/io/serializers.hpp +++ b/morpheus/_lib/include/morpheus/io/serializers.hpp @@ -21,7 +21,6 @@ #include #include -#include namespace morpheus { diff --git a/morpheus/_lib/src/io/serializers.cpp b/morpheus/_lib/src/io/serializers.cpp index 5bc4527555..2b1cef1e1d 100644 --- a/morpheus/_lib/src/io/serializers.cpp +++ b/morpheus/_lib/src/io/serializers.cpp @@ -19,17 +19,22 @@ #include #include +#include // for column_name_info, sink_info, table_metadata #include #include +#include #include #include #include #include -#include +#include // for array +#include // for size_t #include #include -#include +#include // IWYU pragma: keep +#include +// IWYU pragma: no_include namespace morpheus { diff --git a/morpheus/_lib/src/messages/multi.cpp b/morpheus/_lib/src/messages/multi.cpp index 5b6661e6cc..424dd4ce44 100644 --- a/morpheus/_lib/src/messages/multi.cpp +++ b/morpheus/_lib/src/messages/multi.cpp @@ -20,16 +20,27 @@ #include "morpheus/messages/meta.hpp" #include "morpheus/objects/table_info.hpp" #include "morpheus/utilities/type_util.hpp" +#include "morpheus/utilities/type_util_detail.hpp" // for TypeId, DataType +#include // for cudaMemcpy, cudaMemcpy2D, cudaMemcpyDeviceToDevice +#include // for column_view #include #include #include #include - +#include +#include +#include // for get_current_device_resource +#include // for SRF_CHECK_CUDA + +#include // for transform +#include // needed for pybind11::make_tuple +#include // for uint8_t #include #include #include #include +// IWYU pragma: no_include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/python_modules/messages.cpp b/morpheus/_lib/src/python_modules/messages.cpp index 0ecd9d3c46..84c963f571 100644 --- a/morpheus/_lib/src/python_modules/messages.cpp +++ b/morpheus/_lib/src/python_modules/messages.cpp @@ -20,6 +20,7 @@ #include "morpheus/messages/memory/inference_memory_nlp.hpp" #include "morpheus/messages/memory/response_memory.hpp" #include "morpheus/messages/memory/response_memory_probs.hpp" +#include "morpheus/messages/memory/tensor_memory.hpp" #include "morpheus/messages/meta.hpp" #include "morpheus/messages/multi.hpp" #include "morpheus/messages/multi_inference.hpp" @@ -27,19 +28,21 @@ #include "morpheus/messages/multi_inference_nlp.hpp" #include "morpheus/messages/multi_response.hpp" #include "morpheus/messages/multi_response_probs.hpp" -#include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/data_table.hpp" #include "morpheus/utilities/cudf_util.hpp" -#include #include // IWYU pragma: keep #include #include -#include // IWYU pragma: keep +#include // IWYU pragma: keep +#include // for pysrf::import +#include // for Status #include #include #include #include +#include #include namespace morpheus { diff --git a/morpheus/_lib/src/python_modules/stages.cpp b/morpheus/_lib/src/python_modules/stages.cpp index 87e527e2fe..724a505d4d 100644 --- a/morpheus/_lib/src/python_modules/stages.cpp +++ b/morpheus/_lib/src/python_modules/stages.cpp @@ -28,8 +28,14 @@ #include "morpheus/stages/write_to_file.hpp" #include "morpheus/utilities/cudf_util.hpp" +#include // for multiple_inheritance +#include // for arg, init, class_, module_, str_attr_accessor, PYBIND11_MODULE, pybind11 +#include // for dict, sequence +#include // for pysrf::import #include +#include + namespace morpheus { namespace py = pybind11; diff --git a/morpheus/_lib/src/stages/add_classification.cpp b/morpheus/_lib/src/stages/add_classification.cpp index f67bb6b71c..a358414916 100644 --- a/morpheus/_lib/src/stages/add_classification.cpp +++ b/morpheus/_lib/src/stages/add_classification.cpp @@ -17,6 +17,7 @@ #include "morpheus/stages/add_classification.hpp" +#include "morpheus/objects/tensor.hpp" #include "morpheus/utilities/matx_util.hpp" #include diff --git a/morpheus/_lib/src/stages/add_scores.cpp b/morpheus/_lib/src/stages/add_scores.cpp index f6017ddf39..67c5307493 100644 --- a/morpheus/_lib/src/stages/add_scores.cpp +++ b/morpheus/_lib/src/stages/add_scores.cpp @@ -20,10 +20,19 @@ #include "morpheus/objects/tensor.hpp" #include "morpheus/utilities/matx_util.hpp" +#include +#include // for Status +#include // for SinkProperties<>::sink_type_t +#include // for SourceProperties<>::source_type_t +#include // for Object + #include +#include // for size_t #include +#include #include #include +#include namespace morpheus { // Component public implementations diff --git a/morpheus/_lib/src/stages/deserialize.cpp b/morpheus/_lib/src/stages/deserialize.cpp index e98c8e8477..c5b5cd3b91 100644 --- a/morpheus/_lib/src/stages/deserialize.cpp +++ b/morpheus/_lib/src/stages/deserialize.cpp @@ -15,9 +15,10 @@ * limitations under the License. */ -#include "morpheus/stages/deserialization.hpp" +#include "morpheus/stages/deserialize.hpp" #include +#include #include #include diff --git a/morpheus/_lib/src/stages/preprocess_fil.cpp b/morpheus/_lib/src/stages/preprocess_fil.cpp index fc33c9d559..e963913807 100644 --- a/morpheus/_lib/src/stages/preprocess_fil.cpp +++ b/morpheus/_lib/src/stages/preprocess_fil.cpp @@ -18,12 +18,14 @@ #include "morpheus/stages/preprocess_fil.hpp" #include "morpheus/messages/memory/inference_memory_fil.hpp" +#include "morpheus/objects/tensor.hpp" #include "morpheus/utilities/matx_util.hpp" #include "morpheus/utilities/type_util.hpp" #include "morpheus/utilities/type_util_detail.hpp" #include #include +#include #include #include #include diff --git a/morpheus/_lib/src/stages/preprocess_nlp.cpp b/morpheus/_lib/src/stages/preprocess_nlp.cpp index 9b4f130f61..4f24707bd1 100644 --- a/morpheus/_lib/src/stages/preprocess_nlp.cpp +++ b/morpheus/_lib/src/stages/preprocess_nlp.cpp @@ -18,6 +18,7 @@ #include "morpheus/stages/preprocess_nlp.hpp" #include "morpheus/messages/multi_inference.hpp" +#include "morpheus/objects/tensor.hpp" #include "morpheus/utilities/type_util.hpp" #include diff --git a/morpheus/_lib/src/stages/triton_inference.cpp b/morpheus/_lib/src/stages/triton_inference.cpp index 0e6c4eb7f0..2ec5675a52 100644 --- a/morpheus/_lib/src/stages/triton_inference.cpp +++ b/morpheus/_lib/src/stages/triton_inference.cpp @@ -18,6 +18,7 @@ #include "morpheus/stages/triton_inference.hpp" #include "morpheus/messages/multi_response_probs.hpp" +#include "morpheus/objects/tensor.hpp" #include "morpheus/objects/triton_in_out.hpp" #include "morpheus/utilities/matx_util.hpp" #include "morpheus/utilities/stage_util.hpp" diff --git a/morpheus/_lib/src/utilities/cupy_util.cpp b/morpheus/_lib/src/utilities/cupy_util.cpp index e449de825a..4b33b42bf5 100644 --- a/morpheus/_lib/src/utilities/cupy_util.cpp +++ b/morpheus/_lib/src/utilities/cupy_util.cpp @@ -18,13 +18,24 @@ #include "morpheus/utilities/cupy_util.hpp" #include "morpheus/objects/tensor.hpp" +#include "morpheus/utilities/type_util.hpp" // for DType +#include // for COMPACT_GOOGLE_LOG_FATAL, DCHECK, LogMessageFatal #include #include // IWYU pragma: keep #include // IWYU pragma: keep #include #include -#include // IWYU pragma: keep +#include // IWYU pragma: keep +#include // for cuda_stream_per_thread +#include // for device_buffer + +#include // for array +#include // for size_t +#include // for uintptr_t +#include // for make_shared +#include // for string +#include // for vector namespace morpheus { pybind11::object CupyUtil::cp_module = pybind11::none(); From 57e042beeb2e63b57d4a7807cd2f9d153bcd1175 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 15 Aug 2022 13:41:09 -0700 Subject: [PATCH 30/41] wip --- ci/iwyu/mappings.imp | 7 +++ .../include/morpheus/objects/data_table.hpp | 50 +++++++++---------- .../morpheus/stages/add_classification.hpp | 8 +++ .../{deserialization.hpp => deserialize.hpp} | 7 +++ .../morpheus/stages/filter_detection.hpp | 8 +++ .../include/morpheus/stages/write_to_file.hpp | 9 +++- morpheus/_lib/src/python_modules/stages.cpp | 2 +- .../_lib/src/stages/add_classification.cpp | 18 ++++++- morpheus/_lib/src/stages/filter_detection.cpp | 19 +++++-- morpheus/_lib/src/stages/write_to_file.cpp | 12 +++-- 10 files changed, 103 insertions(+), 37 deletions(-) rename morpheus/_lib/include/morpheus/stages/{deserialization.hpp => deserialize.hpp} (86%) diff --git a/ci/iwyu/mappings.imp b/ci/iwyu/mappings.imp index bc9b6647bd..5ee1de6469 100644 --- a/ci/iwyu/mappings.imp +++ b/ci/iwyu/mappings.imp @@ -39,18 +39,24 @@ # rxcpp # Hide includes that are exported by +{ "include": [ "\"rx-includes.hpp\"", private, "", "public" ] }, { "include": [ "", private, "", "public" ] }, { "include": [ "\"rxcpp/rx-util.hpp\"", private, "", "public" ] }, { "include": [ "", private, "", "public" ] }, { "include": [ "\"rxcpp/rx-predef.hpp\"", private, "", "public" ] }, { "include": [ "", private, "", "public" ] }, { "include": [ "\"rxcpp/rx-subscription.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, { "include": [ "", private, "", "public" ] }, { "include": [ "\"rxcpp/rx-observer.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, { "include": [ "", private, "", "public" ] }, { "include": [ "\"rxcpp/rx-scheduler.hpp\"", private, "", "public" ] }, { "include": [ "", private, "", "public" ] }, { "include": [ "\"rxcpp/rx-subscriber.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, { "include": [ "", private, "", "public" ] }, { "include": [ "\"rxcpp/rx-notification.hpp\"", private, "", "public" ] }, { "include": [ "", private, "", "public" ] }, @@ -67,6 +73,7 @@ { "include": [ "\"rxcpp/rx-connectable_observable.hpp\"", private, "", "public" ] }, { "include": [ "", private, "", "public" ] }, { "include": [ "\"rxcpp/rx-grouped_observable.hpp\"", private, "", "public" ] }, +{ "include": [ "", private, "", "public" ] }, ## Symbol mappings # stdlib diff --git a/morpheus/_lib/include/morpheus/objects/data_table.hpp b/morpheus/_lib/include/morpheus/objects/data_table.hpp index 989e42fcd1..28e796f411 100644 --- a/morpheus/_lib/include/morpheus/objects/data_table.hpp +++ b/morpheus/_lib/include/morpheus/objects/data_table.hpp @@ -18,35 +18,35 @@ #pragma once #include - #include namespace morpheus { - class TableInfo; +class TableInfo; + +/****** Component public implementations *******************/ +/****** IDataTable******************************************/ + +/** + * @brief Owning object which owns a unique_ptr, table_metadata, and index information + * Why this doesnt exist in cudf is beyond me + */ +struct IDataTable : public std::enable_shared_from_this +{ + IDataTable() = default; - /****** Component public implementations *******************/ - /****** IDataTable******************************************/ + /** + * TODO(Documentation) + */ + virtual cudf::size_type count() const = 0; + + /** + * TODO(Documentation) + */ + virtual TableInfo get_info() const = 0; /** - * @brief Owning object which owns a unique_ptr, table_metadata, and index information - * Why this doesnt exist in cudf is beyond me + * TODO(Documentation) */ - struct IDataTable : public std::enable_shared_from_this { - IDataTable() = default; - - /** - * TODO(Documentation) - */ - virtual cudf::size_type count() const = 0; - - /** - * TODO(Documentation) - */ - virtual TableInfo get_info() const = 0; - - /** - * TODO(Documentation) - */ - virtual const pybind11::object &get_py_object() const = 0; - }; -} + virtual const pybind11::object &get_py_object() const = 0; +}; +} // namespace morpheus diff --git a/morpheus/_lib/include/morpheus/stages/add_classification.hpp b/morpheus/_lib/include/morpheus/stages/add_classification.hpp index 850ff726c4..36bd8538ea 100644 --- a/morpheus/_lib/include/morpheus/stages/add_classification.hpp +++ b/morpheus/_lib/include/morpheus/stages/add_classification.hpp @@ -20,10 +20,18 @@ #include "morpheus/messages/multi_response_probs.hpp" #include +#include +#include // for Status +#include // for SinkProperties<>::sink_type_t +#include // for SourceProperties<>::source_type_t #include +#include // for Object +#include // for size_t +#include #include #include +#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/stages/deserialization.hpp b/morpheus/_lib/include/morpheus/stages/deserialize.hpp similarity index 86% rename from morpheus/_lib/include/morpheus/stages/deserialization.hpp rename to morpheus/_lib/include/morpheus/stages/deserialize.hpp index 6b4cf821da..3d801d56a7 100644 --- a/morpheus/_lib/include/morpheus/stages/deserialization.hpp +++ b/morpheus/_lib/include/morpheus/stages/deserialize.hpp @@ -21,10 +21,17 @@ #include "morpheus/messages/multi.hpp" #include +#include +#include // for Status +#include // for SinkProperties<>::sink_type_t +#include // for SourceProperties<>::source_type_t #include +#include // for Object +#include // for size_t #include #include +#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/stages/filter_detection.hpp b/morpheus/_lib/include/morpheus/stages/filter_detection.hpp index cc7c3b4dfc..e5529be3cf 100644 --- a/morpheus/_lib/include/morpheus/stages/filter_detection.hpp +++ b/morpheus/_lib/include/morpheus/stages/filter_detection.hpp @@ -20,10 +20,18 @@ #include "morpheus/messages/multi_response_probs.hpp" #include +#include +#include // for Status +#include // for SinkProperties<>::sink_type_t +#include // for SourceProperties<>::source_type_t #include +#include // for Object +#include // for size_t +#include #include #include +#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/stages/write_to_file.hpp b/morpheus/_lib/include/morpheus/stages/write_to_file.hpp index f966ea5242..cb56faa117 100644 --- a/morpheus/_lib/include/morpheus/stages/write_to_file.hpp +++ b/morpheus/_lib/include/morpheus/stages/write_to_file.hpp @@ -17,17 +17,22 @@ #pragma once -#include "morpheus/io/serializers.hpp" #include "morpheus/messages/meta.hpp" #include "morpheus/objects/file_types.hpp" -#include "morpheus/utilities/string_util.hpp" #include +#include +#include // for Status +#include // for SinkProperties<>::sink_type_t +#include // for SourceProperties<>::source_type_t #include +#include // for Object #include +#include // for function #include #include +#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/python_modules/stages.cpp b/morpheus/_lib/src/python_modules/stages.cpp index 724a505d4d..57df6042a1 100644 --- a/morpheus/_lib/src/python_modules/stages.cpp +++ b/morpheus/_lib/src/python_modules/stages.cpp @@ -17,7 +17,7 @@ #include "morpheus/stages/add_classification.hpp" #include "morpheus/stages/add_scores.hpp" -#include "morpheus/stages/deserialization.hpp" +#include "morpheus/stages/deserialize.hpp" #include "morpheus/stages/file_source.hpp" #include "morpheus/stages/filter_detection.hpp" #include "morpheus/stages/kafka_source.hpp" diff --git a/morpheus/_lib/src/stages/add_classification.cpp b/morpheus/_lib/src/stages/add_classification.cpp index a358414916..97b1f0f871 100644 --- a/morpheus/_lib/src/stages/add_classification.cpp +++ b/morpheus/_lib/src/stages/add_classification.cpp @@ -17,13 +17,29 @@ #include "morpheus/stages/add_classification.hpp" +#include "morpheus/objects/dev_mem_info.hpp" // for DevMemInfo #include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" // for TensorIndex, TensorObject #include "morpheus/utilities/matx_util.hpp" +#include "morpheus/utilities/type_util.hpp" // for DType +#include "morpheus/utilities/type_util_detail.hpp" // for DataType +#include // for cudaMemcpy, cudaMemcpyDeviceToDevice +#include +#include // for cuda_stream_per_thread +#include // for device_buffer +#include // for SRF_CHECK_CUDA + +#include // for min_element, transform #include #include +#include // for divides, bind, placeholders #include -#include +#include // needed for logging +#include // for declval +#include // for move +// IWYU thinks we need __alloc_traits<>::value_type for vector assignments +// IWYU pragma: no_include namespace morpheus { // Component public implementations diff --git a/morpheus/_lib/src/stages/filter_detection.cpp b/morpheus/_lib/src/stages/filter_detection.cpp index 7197c7afaa..ec8c847e00 100644 --- a/morpheus/_lib/src/stages/filter_detection.cpp +++ b/morpheus/_lib/src/stages/filter_detection.cpp @@ -15,18 +15,27 @@ * limitations under the License. */ -#include "morpheus/stages/filter_detection.hpp" +#include "morpheus/stages/filter_detection.hpp" // IWYU pragma: accosiated +#include "morpheus/objects/dev_mem_info.hpp" // for DevMemInfo +#include "morpheus/objects/tensor_object.hpp" // for TensorIndex, TensorObject #include "morpheus/utilities/matx_util.hpp" -#include "morpheus/utilities/tensor_util.hpp" // for TensorUtils::get_element_stride +#include "morpheus/utilities/tensor_util.hpp" // for TensorUtils::get_element_stride +#include "morpheus/utilities/type_util_detail.hpp" // for DataType -#include // for CHECK +#include // for cudaMemcpy, cudaMemcpyDeviceToDevice, cudaMemcpyDeviceToHost +#include // for CHECK +#include // for cuda_stream_per_thread +#include // for device_buffer +#include // for SRF_CHECK_CUDA #include +#include // for uint8_t #include #include -#include // needed for glog -#include // for pair +#include // needed for glog +#include // for declval (indirectly via templates) +#include // for pair namespace morpheus { diff --git a/morpheus/_lib/src/stages/write_to_file.cpp b/morpheus/_lib/src/stages/write_to_file.cpp index cda3ed00f3..b177bdb17b 100644 --- a/morpheus/_lib/src/stages/write_to_file.cpp +++ b/morpheus/_lib/src/stages/write_to_file.cpp @@ -15,16 +15,22 @@ * limitations under the License. */ -#include "morpheus/stages/write_to_file.hpp" +#include "morpheus/stages/write_to_file.hpp" // IWYU pragma: accosiated -#include "morpheus/utilities/matx_util.hpp" +#include "morpheus/io/serializers.hpp" +#include "morpheus/utilities/string_util.hpp" + +#include #include #include -#include +#include // for invalid_argument, runtime_error #include +#include // for declval (indirectly via templates) +#include // for forward, move, addressof namespace morpheus { + // Component public implementations // ************ WriteToFileStage **************************** // WriteToFileStage::WriteToFileStage(const std::string &filename, From 9951bdd99d71c8044f9937736fb33a3d188c1366 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 15 Aug 2022 13:58:59 -0700 Subject: [PATCH 31/41] wip --- .../include/morpheus/stages/add_scores.hpp | 8 +++++++ .../morpheus/stages/preprocess_fil.hpp | 6 +++++ .../morpheus/stages/preprocess_nlp.hpp | 7 ++++++ morpheus/_lib/src/stages/add_scores.cpp | 19 +++++++++++----- morpheus/_lib/src/stages/deserialize.cpp | 3 ++- morpheus/_lib/src/stages/preprocess_fil.cpp | 22 +++++++++++++------ morpheus/_lib/src/stages/preprocess_nlp.cpp | 11 ++++++++-- 7 files changed, 61 insertions(+), 15 deletions(-) diff --git a/morpheus/_lib/include/morpheus/stages/add_scores.hpp b/morpheus/_lib/include/morpheus/stages/add_scores.hpp index 02ba811bab..c7affd8e84 100644 --- a/morpheus/_lib/include/morpheus/stages/add_scores.hpp +++ b/morpheus/_lib/include/morpheus/stages/add_scores.hpp @@ -20,10 +20,18 @@ #include "morpheus/messages/multi_response_probs.hpp" #include +#include // for apply, make_subscriber, observable_member, is_on_error<>::not_void, is_on_next_of<>::not_void, trace_activity +#include // for Status +#include // for SinkProperties<>::sink_type_t +#include // for SourceProperties<>::source_type_t #include +#include // for Object +#include // for size_t +#include #include #include +#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/stages/preprocess_fil.hpp b/morpheus/_lib/include/morpheus/stages/preprocess_fil.hpp index 6b19073ba1..03aa91396b 100644 --- a/morpheus/_lib/include/morpheus/stages/preprocess_fil.hpp +++ b/morpheus/_lib/include/morpheus/stages/preprocess_fil.hpp @@ -21,10 +21,16 @@ #include "morpheus/messages/multi_inference.hpp" #include +#include // for apply, make_subscriber, observable_member, is_on_error<>::not_void, is_on_next_of<>::not_void, from +#include // for Status +#include // for SinkProperties<>::sink_type_t +#include // for SourceProperties<>::source_type_t #include +#include // for Object #include #include +#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/stages/preprocess_nlp.hpp b/morpheus/_lib/include/morpheus/stages/preprocess_nlp.hpp index 977dadb1b9..afbe650378 100644 --- a/morpheus/_lib/include/morpheus/stages/preprocess_nlp.hpp +++ b/morpheus/_lib/include/morpheus/stages/preprocess_nlp.hpp @@ -21,10 +21,17 @@ #include "morpheus/messages/multi_inference.hpp" #include +#include // for apply, make_subscriber, observable_member, is_on_error<>::not_void, is_on_next_of<>::not_void, from +#include // for Status +#include // for SinkProperties<>::sink_type_t +#include // for SourceProperties<>::source_type_t #include +#include // for Object +#include // for uint32_t #include #include +#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/stages/add_scores.cpp b/morpheus/_lib/src/stages/add_scores.cpp index 67c5307493..b3e81c79af 100644 --- a/morpheus/_lib/src/stages/add_scores.cpp +++ b/morpheus/_lib/src/stages/add_scores.cpp @@ -18,21 +18,30 @@ #include "morpheus/stages/add_scores.hpp" #include "morpheus/objects/tensor.hpp" -#include "morpheus/utilities/matx_util.hpp" +#include "morpheus/objects/tensor_object.hpp" // for TensorIndex, TensorObject -#include +#include // for cudaMemcpy, cudaMemcpyDeviceToDevice +#include +#include // for cuda_stream_per_thread +#include // for device_buffer #include // for Status +#include // for SRF_CHECK_CUDA #include // for SinkProperties<>::sink_type_t #include // for SourceProperties<>::source_type_t #include // for Object -#include -#include // for size_t +#include // for min_element, transform +#include // for size_t #include +#include // for divides, placeholders #include #include -#include +#include // for logging +#include // for declval +#include // for move #include +// IWYU thinks we need __alloc_traits<>::value_type for vector assignments +// IWYU pragma: no_include namespace morpheus { // Component public implementations diff --git a/morpheus/_lib/src/stages/deserialize.cpp b/morpheus/_lib/src/stages/deserialize.cpp index c5b5cd3b91..3ca4ad0163 100644 --- a/morpheus/_lib/src/stages/deserialize.cpp +++ b/morpheus/_lib/src/stages/deserialize.cpp @@ -21,10 +21,11 @@ #include #include +#include // for min #include #include #include -#include +#include // for declval #include namespace morpheus { diff --git a/morpheus/_lib/src/stages/preprocess_fil.cpp b/morpheus/_lib/src/stages/preprocess_fil.cpp index e963913807..20ca1875e2 100644 --- a/morpheus/_lib/src/stages/preprocess_fil.cpp +++ b/morpheus/_lib/src/stages/preprocess_fil.cpp @@ -18,29 +18,37 @@ #include "morpheus/stages/preprocess_fil.hpp" #include "morpheus/messages/memory/inference_memory_fil.hpp" +#include "morpheus/messages/meta.hpp" // for MessageMeta +#include "morpheus/objects/dev_mem_info.hpp" // for DevMemInfo +#include "morpheus/objects/table_info.hpp" // for TableInfo #include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" // for TensorIndex #include "morpheus/utilities/matx_util.hpp" #include "morpheus/utilities/type_util.hpp" #include "morpheus/utilities/type_util_detail.hpp" -#include -#include -#include +#include // for cudaMemcpy, cudaMemcpyDeviceToDevice +#include // for column, column::contents +#include // for column_view +#include // for table_view #include #include -#include -#include +#include // for object_api::operator(), operator""_a #include +#include // for str_attr_accessor, arg #include #include -#include +#include // for cuda_stream_per_thread +#include // for device_buffer +#include // for SRF_CHECK_CUDA #include +#include #include #include #include #include -#include +#include // for declval #include namespace morpheus { diff --git a/morpheus/_lib/src/stages/preprocess_nlp.cpp b/morpheus/_lib/src/stages/preprocess_nlp.cpp index 4f24707bd1..eeb1820db2 100644 --- a/morpheus/_lib/src/stages/preprocess_nlp.cpp +++ b/morpheus/_lib/src/stages/preprocess_nlp.cpp @@ -17,21 +17,28 @@ #include "morpheus/stages/preprocess_nlp.hpp" +#include "morpheus/messages/memory/inference_memory.hpp" // for InferenceMemory +#include "morpheus/messages/memory/tensor_memory.hpp" // for TensorMemory::tensor_map_t #include "morpheus/messages/multi_inference.hpp" +#include "morpheus/objects/table_info.hpp" // for TableInfo #include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" // for TensorIndex, TensorObject #include "morpheus/utilities/type_util.hpp" +#include // for column, column::contents +#include // for strings_column_view #include #include -#include #include #include +#include // for device_buffer #include #include #include +#include #include -#include +#include // for declval #include namespace morpheus { From 4ea55b6e4a10ba21eada08e33becc3b55b14811d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 15 Aug 2022 14:07:22 -0700 Subject: [PATCH 32/41] WIP --- .../include/morpheus/stages/kafka_source.hpp | 9 +++++++-- morpheus/_lib/src/stages/kafka_source.cpp | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/morpheus/_lib/include/morpheus/stages/kafka_source.hpp b/morpheus/_lib/include/morpheus/stages/kafka_source.hpp index c3370cf485..fdf86611f2 100644 --- a/morpheus/_lib/include/morpheus/stages/kafka_source.hpp +++ b/morpheus/_lib/include/morpheus/stages/kafka_source.hpp @@ -22,10 +22,15 @@ #include #include #include -#include -#include +#include // for apply, make_subscriber, observable_member, is_on_error<>::not_void, is_on_next_of<>::not_void, trace_activity +#include // for Status +#include // for SourceProperties<>::source_type_t #include +#include // for Object +#include // for size_t +#include // for int32_t, uint32_t +#include #include #include #include diff --git a/morpheus/_lib/src/stages/kafka_source.cpp b/morpheus/_lib/src/stages/kafka_source.cpp index e8eeb4da1e..fc870be6f3 100644 --- a/morpheus/_lib/src/stages/kafka_source.cpp +++ b/morpheus/_lib/src/stages/kafka_source.cpp @@ -21,30 +21,40 @@ #include "morpheus/utilities/stage_util.hpp" #include "morpheus/utilities/string_util.hpp" +#include // for sleep_for, yield #include -#include +#include // for column #include +#include // for string_scalar #include +#include // for strings_column_view +#include // for table #include -#include #include #include -#include #include #include #include +#include // for SharedFuture +#include // for find, min, transform #include #include #include #include #include +#include // for initializer_list +#include // for back_insert_iterator, back_inserter #include #include #include #include #include #include +// IWYU thinks we need atomic for vector.emplace_back of a unique_ptr +// and __alloc_traits<>::value_type for vector assignments +// IWYU pragma: no_include +// IWYU pragma: no_include #define CHECK_KAFKA(command, expected, msg) \ { \ From bee06de1b0d9a11fe9ce625071bbe44386d05d39 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 15 Aug 2022 14:34:35 -0700 Subject: [PATCH 33/41] wip --- ci/iwyu/mappings.imp | 1 + .../include/morpheus/stages/file_source.hpp | 6 ++++++ .../_lib/include/morpheus/stages/serialize.hpp | 9 ++++++++- morpheus/_lib/src/stages/file_source.cpp | 18 ++++++++++++------ morpheus/_lib/src/stages/serialize.cpp | 9 +++++++-- 5 files changed, 34 insertions(+), 9 deletions(-) diff --git a/ci/iwyu/mappings.imp b/ci/iwyu/mappings.imp index 5ee1de6469..3e9d5f8d80 100644 --- a/ci/iwyu/mappings.imp +++ b/ci/iwyu/mappings.imp @@ -124,6 +124,7 @@ { "symbol": ["pybind11::return_value_policy", "private", "", "public"] }, { "symbol": ["pybind11::return_value_policy::move", "private", "", "public"] }, { "symbol": ["pybind11::return_value_policy::reference_internal", "private", "", "public"] }, +{ "symbol": ["pybind11::size_t", "private", "", "public"] }, { "symbol": ["pybind11::stop_iteration", "private", "", "public"] }, # spdlog diff --git a/morpheus/_lib/include/morpheus/stages/file_source.hpp b/morpheus/_lib/include/morpheus/stages/file_source.hpp index 4eb27ffcfe..7dc4c54f15 100644 --- a/morpheus/_lib/include/morpheus/stages/file_source.hpp +++ b/morpheus/_lib/include/morpheus/stages/file_source.hpp @@ -19,11 +19,17 @@ #include "morpheus/messages/meta.hpp" +#include // for table_with_metadata #include +#include // for apply, make_subscriber, observable_member, is_on_error<>::not_void, is_on_next_of<>::not_void, trace_activity +#include // for Status +#include // for SourceProperties<>::source_type_t #include +#include // for Object #include #include +#include // for vector namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/include/morpheus/stages/serialize.hpp b/morpheus/_lib/include/morpheus/stages/serialize.hpp index 038ced9ce1..58dca40efa 100644 --- a/morpheus/_lib/include/morpheus/stages/serialize.hpp +++ b/morpheus/_lib/include/morpheus/stages/serialize.hpp @@ -17,15 +17,22 @@ #pragma once +#include "morpheus/messages/meta.hpp" // for MessageMeta #include "morpheus/messages/multi.hpp" +#include "morpheus/objects/table_info.hpp" // for TableInfo #include +#include // for apply, make_subscriber, observable_member, is_on_error<>::not_void, is_on_next_of<>::not_void, from +#include // for Status +#include // for SinkProperties<>::sink_type_t +#include // for SourceProperties<>::source_type_t #include +#include // for Object -#include #include #include #include +#include // for vector namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/stages/file_source.cpp b/morpheus/_lib/src/stages/file_source.cpp index 825f10c6c5..9db0dd895f 100644 --- a/morpheus/_lib/src/stages/file_source.cpp +++ b/morpheus/_lib/src/stages/file_source.cpp @@ -17,25 +17,31 @@ #include "morpheus/stages/file_source.hpp" +#include // for column #include #include +#include // for string_scalar #include +#include // for strings_column_view +#include // for table #include #include -#include -#include +#include // for object_api::operator() #include -#include +#include // for str_attr_accessor +#include // for pybind11::int_ #include -#include +#include // for find +#include // for size_t #include -#include #include -#include #include #include +#include // for runtime_error #include +// IWYU thinks we need __alloc_traits<>::value_type for vector assignments +// IWYU pragma: no_include namespace morpheus { // Component public implementations diff --git a/morpheus/_lib/src/stages/serialize.cpp b/morpheus/_lib/src/stages/serialize.cpp index 55319862b5..3dc533c77e 100644 --- a/morpheus/_lib/src/stages/serialize.cpp +++ b/morpheus/_lib/src/stages/serialize.cpp @@ -17,14 +17,19 @@ #include "morpheus/stages/serialize.hpp" +#include // for gil_scoped_acquire + #include #include #include +#include // for declval +#include // for move +// IWYU thinks basic_stringbuf & map are needed for the regex constructor +// IWYU pragma: no_include +// IWYU pragma: no_include namespace morpheus { -using namespace std::literals; - constexpr std::regex_constants::syntax_option_type RegexOptions = std::regex_constants::ECMAScript | std::regex_constants::icase; From 78fb4390d23131dcb34a8ea42f8096e820b6ad45 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 15 Aug 2022 14:47:19 -0700 Subject: [PATCH 34/41] WIP --- ci/iwyu/mappings.imp | 3 +++ .../morpheus/stages/triton_inference.hpp | 7 +++++++ morpheus/_lib/src/stages/triton_inference.cpp | 18 +++++++++++++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/ci/iwyu/mappings.imp b/ci/iwyu/mappings.imp index 3e9d5f8d80..902aa07385 100644 --- a/ci/iwyu/mappings.imp +++ b/ci/iwyu/mappings.imp @@ -75,6 +75,9 @@ { "include": [ "\"rxcpp/rx-grouped_observable.hpp\"", private, "", "public" ] }, { "include": [ "", private, "", "public" ] }, +#Triton Client +{ "include": ["\"common.h\"", "private", "", "public"] }, + ## Symbol mappings # stdlib { "symbol": ["__gnu_cxx::__enable_if::__type", "private", "", "public"] }, diff --git a/morpheus/_lib/include/morpheus/stages/triton_inference.hpp b/morpheus/_lib/include/morpheus/stages/triton_inference.hpp index 14d99d338a..d847984d67 100644 --- a/morpheus/_lib/include/morpheus/stages/triton_inference.hpp +++ b/morpheus/_lib/include/morpheus/stages/triton_inference.hpp @@ -23,10 +23,17 @@ #include #include +#include // for apply, make_subscriber, observable_member, is_on_error<>::not_void, is_on_next_of<>::not_void, from +#include // for Status +#include // for SinkProperties<>::sink_type_t +#include // for SourceProperties<>::source_type_t #include +#include // for Object +#include #include #include +#include namespace morpheus { /****** Component public implementations *******************/ diff --git a/morpheus/_lib/src/stages/triton_inference.cpp b/morpheus/_lib/src/stages/triton_inference.cpp index 2ec5675a52..da58a5e063 100644 --- a/morpheus/_lib/src/stages/triton_inference.cpp +++ b/morpheus/_lib/src/stages/triton_inference.cpp @@ -17,25 +17,41 @@ #include "morpheus/stages/triton_inference.hpp" +#include "morpheus/messages/memory/inference_memory.hpp" // for InferenceMemory +#include "morpheus/messages/memory/response_memory.hpp" // for ResponseMemory +#include "morpheus/messages/memory/tensor_memory.hpp" // for TensorMemory::tensor_map_t #include "morpheus/messages/multi_response_probs.hpp" +#include "morpheus/objects/dev_mem_info.hpp" // for DevMemInfo #include "morpheus/objects/tensor.hpp" +#include "morpheus/objects/tensor_object.hpp" // for TensorIndex, TensorObject #include "morpheus/objects/triton_in_out.hpp" #include "morpheus/utilities/matx_util.hpp" #include "morpheus/utilities/stage_util.hpp" +#include "morpheus/utilities/string_util.hpp" // for MORPHEUS_CONCAT_STR #include "morpheus/utilities/type_util.hpp" +#include "morpheus/utilities/type_util_detail.hpp" // for DataType +#include // for cudaMemcpy, cudaMemcpy2D, cudaMemcpyDeviceToHost, cudaMemcpyHostToDevice #include #include #include #include +#include // for cuda_stream_per_thread +#include // for device_buffer +#include // for SRF_CHECK_CUDA +#include // for min #include #include #include +#include // for multiplies #include -#include +#include // for accumulate #include +#include // for runtime_error, out_of_range +#include // for declval #include +// IWYU pragma: no_include #define CHECK_TRITON(method) ::InferenceClientStage__check_triton_errors(method, #method, __FILE__, __LINE__); From 943cf6dc7949235a4b0ef95c3eab938d0902b5f2 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 15 Aug 2022 15:46:47 -0700 Subject: [PATCH 35/41] formatting --- morpheus/_lib/include/morpheus/objects/tensor_object.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/morpheus/_lib/include/morpheus/objects/tensor_object.hpp b/morpheus/_lib/include/morpheus/objects/tensor_object.hpp index a7f9056e1e..7de293b371 100644 --- a/morpheus/_lib/include/morpheus/objects/tensor_object.hpp +++ b/morpheus/_lib/include/morpheus/objects/tensor_object.hpp @@ -32,8 +32,8 @@ #include // for size_t, byte #include #include -#include // for shared_ptr -#include // IWYU pragma: keep +#include // for shared_ptr +#include // IWYU pragma: keep #include #include // for runtime_error #include From 5940e541e988909dc9c9c38930352c70c9fe1b8c Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 15 Aug 2022 16:58:48 -0700 Subject: [PATCH 36/41] Move the actual cuda soruces to their own cmake target and disable IWYU for that target --- .../_lib/cmake/libraries/cuda_utils.cmake | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/morpheus/_lib/cmake/libraries/cuda_utils.cmake b/morpheus/_lib/cmake/libraries/cuda_utils.cmake index 0708506a09..8ee728c8c6 100644 --- a/morpheus/_lib/cmake/libraries/cuda_utils.cmake +++ b/morpheus/_lib/cmake/libraries/cuda_utils.cmake @@ -16,15 +16,45 @@ list(APPEND CMAKE_MESSAGE_CONTEXT "cuda_utils") find_package(pybind11 REQUIRED) +# Place the two cuda sources in their own target and disable IWYU for that target. +add_library(cuda_utils_objs +OBJECT + ${MORPHEUS_LIB_ROOT}/src/utilities/matx_util.cu + ${MORPHEUS_LIB_ROOT}/src/utilities/type_util.cu +) + + +set_target_properties( + cuda_utils_objs + PROPERTIES + C_INCLUDE_WHAT_YOU_USE "" + CXX_INCLUDE_WHAT_YOU_USE "" + EXPORT_COMPILE_COMMANDS OFF +) + +target_include_directories(cuda_utils_objs + PUBLIC + "${MORPHEUS_LIB_ROOT}/include" + cudf::cudf + matx::matx + srf::pysrf +) + +target_link_libraries(cuda_utils_objs + PUBLIC + cudf::cudf + matx::matx + srf::pysrf +) + add_library(cuda_utils SHARED + $ ${MORPHEUS_LIB_ROOT}/src/objects/dev_mem_info.cpp ${MORPHEUS_LIB_ROOT}/src/objects/table_info.cpp ${MORPHEUS_LIB_ROOT}/src/objects/tensor_object.cpp - ${MORPHEUS_LIB_ROOT}/src/utilities/matx_util.cu ${MORPHEUS_LIB_ROOT}/src/utilities/tensor_util.cpp ${MORPHEUS_LIB_ROOT}/src/utilities/type_util_detail.cpp - ${MORPHEUS_LIB_ROOT}/src/utilities/type_util.cu ) target_include_directories(cuda_utils From 3340c1118044e35158b169991c241d5a443d32d0 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 16 Aug 2022 07:43:38 -0700 Subject: [PATCH 37/41] Formatting --- morpheus/_lib/tests/test_tensor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morpheus/_lib/tests/test_tensor.cpp b/morpheus/_lib/tests/test_tensor.cpp index 195ae7cdaf..66a08c9229 100644 --- a/morpheus/_lib/tests/test_tensor.cpp +++ b/morpheus/_lib/tests/test_tensor.cpp @@ -17,8 +17,8 @@ #include "./test_morpheus.hpp" // IWYU pragma: associated -#include "morpheus/utilities/tensor_util.hpp" // for TensorUtils, TensorUtils::shape_type #include "morpheus/objects/tensor_object.hpp" // for TensorIndex +#include "morpheus/utilities/tensor_util.hpp" // for TensorUtils, TensorUtils::shape_type #include // for AssertionResult, SuiteApiResolver, TestInfo, EXPECT_TRUE, Message, TEST_F, Test, TestFactoryImpl, TestPartResult From 8a12998e2e380fcb5ad204c49c0fcc78804d08f9 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 16 Aug 2022 08:28:09 -0700 Subject: [PATCH 38/41] Set cstddef as the correct header for size_t --- ci/iwyu/mappings.imp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/iwyu/mappings.imp b/ci/iwyu/mappings.imp index 902aa07385..2fa46c3a69 100644 --- a/ci/iwyu/mappings.imp +++ b/ci/iwyu/mappings.imp @@ -84,6 +84,8 @@ { "symbol": ["std::__success_type>>::type" , "private", "", "public"] }, { "symbol": ["__cxxabiv1::__forced_unwind", "private", "", "public"] }, { "symbol": ["std::filesystem", "private", "", "public"] }, +{ "symbol": ["size_t", , "private", "", "public"] }, +{ "symbol": ["std::size_t", , "private", "", "public"] }, # boost { "symbol": ["__forced_unwind", "private", "", "public"] }, From 1a1eca41f3277db4930683d2bb8f90fce5bf5ba9 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 16 Aug 2022 08:28:24 -0700 Subject: [PATCH 39/41] Change dirs using pushd/popd --- ci/scripts/jenkins/checks.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh index e208fcc550..c699f59ed4 100755 --- a/ci/scripts/jenkins/checks.sh +++ b/ci/scripts/jenkins/checks.sh @@ -17,11 +17,12 @@ set -e source ${WORKSPACE}/ci/scripts/jenkins/common.sh +export IWYU_DIR="${WORKSPACE_TMP}/iwyu" fetch_base_branch gpuci_logger "Creating conda env" -rm -rf ${MORPHEUS_ROOT}/.cache/ ${MORPHEUS_ROOT}/build/ ${WORKSPACE_TMP}/iwyu +rm -rf ${MORPHEUS_ROOT}/.cache/ ${MORPHEUS_ROOT}/build/ ${IWYU_DIR} conda config --add pkgs_dirs /opt/conda/pkgs conda config --env --add channels conda-forge conda config --env --set channel_alias ${CONDA_CHANNEL_ALIAS:-"https://conda.anaconda.org"} @@ -34,10 +35,8 @@ mamba env update -q -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER show_conda_info gpuci_logger "Installing IWYU" -export IWYU_DIR="${WORKSPACE_TMP}/iwyu" -cd ${WORKSPACE_TMP} git clone https://github.com/include-what-you-use/include-what-you-use.git ${IWYU_DIR} -cd ${IWYU_DIR} +pushd ${IWYU_DIR} git checkout clang_12 cmake -G Ninja \ -DCMAKE_PREFIX_PATH=$(llvm-config --cmakedir) \ @@ -48,7 +47,7 @@ cmake -G Ninja \ cmake --build . --parallel ${PARALLEL_LEVEL} --target install -cd ${MORPHEUS_ROOT} +popd gpuci_logger "Runing Python style checks" ${MORPHEUS_ROOT}/ci/scripts/python_checks.sh From 3375f4d2e136fe942e7e5bb97d88aa1165a3dc3d Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 16 Aug 2022 08:51:41 -0700 Subject: [PATCH 40/41] iwyu --- morpheus/_lib/src/stages/filter_detection.cpp | 2 ++ morpheus/_lib/tests/test_tensor.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/morpheus/_lib/src/stages/filter_detection.cpp b/morpheus/_lib/src/stages/filter_detection.cpp index ec8c847e00..1040440642 100644 --- a/morpheus/_lib/src/stages/filter_detection.cpp +++ b/morpheus/_lib/src/stages/filter_detection.cpp @@ -36,6 +36,8 @@ #include // needed for glog #include // for declval (indirectly via templates) #include // for pair +// IWYU thinks we need ext/new_allocator.h for size_t for some reason +// IWYU pragma: no_include namespace morpheus { diff --git a/morpheus/_lib/tests/test_tensor.cpp b/morpheus/_lib/tests/test_tensor.cpp index 66a08c9229..ff58575ddf 100644 --- a/morpheus/_lib/tests/test_tensor.cpp +++ b/morpheus/_lib/tests/test_tensor.cpp @@ -26,6 +26,8 @@ #include // for allocator, operator==, basic_string, string #include // for vector // IWYU pragma: no_include "morpheus/utilities/string_util.hpp" +// IWYU thinks we need ext/new_allocator.h for size_t for some reason +// IWYU pragma: no_include using namespace morpheus; From e7fe61759446cd59babaca01f2c06009db5b8144 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 16 Aug 2022 09:16:00 -0700 Subject: [PATCH 41/41] update year --- morpheus/_lib/include/morpheus/objects/data_table.hpp | 2 +- morpheus/_lib/include/morpheus/objects/fiber_queue.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/morpheus/_lib/include/morpheus/objects/data_table.hpp b/morpheus/_lib/include/morpheus/objects/data_table.hpp index 28e796f411..c0f7f7356c 100644 --- a/morpheus/_lib/include/morpheus/objects/data_table.hpp +++ b/morpheus/_lib/include/morpheus/objects/data_table.hpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/morpheus/_lib/include/morpheus/objects/fiber_queue.hpp b/morpheus/_lib/include/morpheus/objects/fiber_queue.hpp index 3ec8959b94..a13a5c6d52 100644 --- a/morpheus/_lib/include/morpheus/objects/fiber_queue.hpp +++ b/morpheus/_lib/include/morpheus/objects/fiber_queue.hpp @@ -1,5 +1,5 @@ /** - * SPDX-FileCopyrightText: Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License");