From de9ce756b9e7e0536ddd5b86f2a6d5d5b664d7d2 Mon Sep 17 00:00:00 2001 From: Martijn Govers Date: Mon, 9 Sep 2024 12:16:40 +0200 Subject: [PATCH 1/3] more efficient columnar deserialization Signed-off-by: Martijn Govers --- .../power_grid_model/auxiliary/dataset.hpp | 12 +++-- .../auxiliary/serialization/common.hpp | 5 ++ .../auxiliary/serialization/deserializer.hpp | 53 ++++++++++++++----- 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp index de1dd420e..7ac46541e 100644 --- a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp +++ b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp @@ -233,15 +233,19 @@ template class Dataset { } constexpr bool is_row_based(Idx const i) const { return is_row_based(buffers_[i]); } constexpr bool is_row_based(Buffer const& buffer) const { return buffer.data != nullptr; } - constexpr bool is_columnar(std::string_view component) const { + constexpr bool is_columnar(std::string_view component, bool with_attribute_buffers = false) const { Idx const idx = find_component(component, false); if (idx == invalid_index) { return false; } - return is_columnar(idx); + return is_columnar(idx, with_attribute_buffers); + } + constexpr bool is_columnar(Idx const i, bool with_attribute_buffers = false) const { + return is_columnar(buffers_[i], with_attribute_buffers); + } + constexpr bool is_columnar(Buffer const& buffer, bool with_attribute_buffers = false) const { + return !is_row_based(buffer) && !(with_attribute_buffers && buffer.attributes.empty()); } - constexpr bool is_columnar(Idx const i) const { return !is_row_based(i); } - constexpr bool is_columnar(Buffer const& buffer) const { return !is_row_based(buffer); } Idx find_component(std::string_view component, bool required = false) const { auto const found = std::ranges::find_if(dataset_info_.component_info, [component](ComponentInfo const& x) { diff --git a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp index 2a504b55f..4d1cf36be 100644 --- a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp +++ b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp @@ -46,6 +46,11 @@ reordered_attribute_buffers(BufferType& buffer, std::span{}; }); + + if (std::ranges::all_of(result, [](auto const& attribute_buffer) { return attribute_buffer.data == nullptr; })) { + result = {}; + } + return result; } diff --git a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp index b63870ea9..dad213811 100644 --- a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp +++ b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp @@ -703,7 +703,7 @@ class Deserializer { void parse_component(Idx component_idx) { if (dataset_handler_.is_row_based(component_idx)) { parse_component(row_based, component_idx); - } else { + } else if (dataset_handler_.is_columnar(component_idx, true)) { parse_component(columnar, component_idx); } } @@ -713,7 +713,7 @@ class Deserializer { auto const& buffer = dataset_handler_.get_buffer(component_idx); assert(dataset_handler_.is_row_based(buffer) == detail::is_row_based_v); - assert(dataset_handler_.is_columnar(buffer) == detail::is_columnar_v); + assert(dataset_handler_.is_columnar(buffer, true) == detail::is_columnar_v); assert(is_row_based(buffer) == detail::is_row_based_v); assert(is_columnar(buffer) == detail::is_columnar_v); @@ -744,9 +744,9 @@ class Deserializer { } return {}; }(); - auto const reordered_attribute_buffers = detail::is_columnar_v - ? detail::reordered_attribute_buffers(buffer, attributes) - : std::vector>{}; + auto reordered_attribute_buffers = detail::is_columnar_v + ? detail::reordered_attribute_buffers(buffer, attributes) + : std::vector>{}; BufferView const buffer_view{ .buffer = &buffer, .idx = 0, .reordered_attribute_buffers = reordered_attribute_buffers}; @@ -779,23 +779,48 @@ class Deserializer { if (msg_data.size == 0) { return; } + // set offset and skip array header offset_ = msg_data.offset; parse_map_array(); + for (element_number_ = 0; element_number_ != msg_data.size; ++element_number_) { BufferView const element_buffer = advance(buffer_view, element_number_); // check the element is map or array - auto const element_visitor = parse_map_array(); - if (element_visitor.is_map) { - parse_map_element(row_or_column_tag, element_buffer, element_visitor.size, component); - } else { - parse_array_element(row_or_column_tag, element_buffer, element_visitor.size, component, attributes); - } + parse_element(row_or_column_tag, element_buffer, component, attributes); } element_number_ = -1; offset_ = 0; } + void parse_element(row_based_t tag, BufferView const& buffer_view, MetaComponent const& component, + std::span attributes) { + assert(is_row_based(buffer_view)); + + auto const element_visitor = parse_map_array(); + if (element_visitor.is_map) { + parse_map_element(tag, buffer_view, element_visitor.size, component); + } else { + parse_array_element(tag, buffer_view, element_visitor.size, component, attributes); + } + } + + void parse_element(columnar_t tag, BufferView const& buffer_view, MetaComponent const& component, + std::span attributes) { + assert(is_columnar(buffer_view)); + + auto const element_visitor = parse_map_array(); + if (element_visitor.is_map) { + parse_map_array(); + parse_map_element(tag, buffer_view, element_visitor.size, component); + } else if (!buffer_view.reordered_attribute_buffers.empty()) { + parse_map_array(); + parse_array_element(tag, buffer_view, element_visitor.size, component, attributes); + } else { + parse_skip(); + } + } + void parse_map_element(row_based_t tag, BufferView const& buffer_view, Idx map_size, MetaComponent const& component) { while (map_size-- != 0) { @@ -813,6 +838,8 @@ class Deserializer { void parse_map_element(columnar_t /*tag*/, BufferView const& buffer_view, Idx map_size, MetaComponent const& /*component*/) { + assert(!buffer_view.buffer->attributes.empty()); + while (map_size-- != 0) { attribute_key_ = parse_string(); if (auto it = std::ranges::find_if(buffer_view.buffer->attributes, @@ -936,7 +963,9 @@ class Deserializer { assert(buffer_view.buffer != nullptr); return is_columnar(*buffer_view.buffer); } - static constexpr bool is_columnar(WritableDataset::Buffer const& buffer) { return buffer.data == nullptr; } + static constexpr bool is_columnar(WritableDataset::Buffer const& buffer) { + return buffer.data == nullptr && !buffer.attributes.empty(); + } [[noreturn]] void handle_error(std::exception const& e) { std::stringstream ss; From 04058692fa1f0137442bedc20bde2bc025485f71 Mon Sep 17 00:00:00 2001 From: Martijn Govers Date: Mon, 9 Sep 2024 12:35:49 +0200 Subject: [PATCH 2/3] clang-tidy Signed-off-by: Martijn Govers --- .../power_grid_model/auxiliary/serialization/deserializer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp index dad213811..e040e52b3 100644 --- a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp +++ b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp @@ -744,7 +744,7 @@ class Deserializer { } return {}; }(); - auto reordered_attribute_buffers = detail::is_columnar_v + auto const reordered_attribute_buffers = detail::is_columnar_v ? detail::reordered_attribute_buffers(buffer, attributes) : std::vector>{}; From eb56b4b1fa08cce961342462965ca2fcb982358a Mon Sep 17 00:00:00 2001 From: Martijn Govers Date: Mon, 9 Sep 2024 13:27:38 +0200 Subject: [PATCH 3/3] process comments Signed-off-by: Martijn Govers --- .../power_grid_model/auxiliary/serialization/common.hpp | 5 +++++ .../auxiliary/serialization/deserializer.hpp | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp index 4d1cf36be..ae848d2a7 100644 --- a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp +++ b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/common.hpp @@ -23,6 +23,11 @@ concept row_based_or_columnar_c = std::derived_from || std::deri template constexpr bool is_row_based_v = std::derived_from; template constexpr bool is_columnar_v = std::derived_from; +// obtain attribute buffers of a columnar dataset buffer, ordered by the provided meta attributes. +// +// If none of the provided meta attributes are present in the provided attribute buffers, the result is empty. +// Otherwise, returns a list of attribute buffers with the same order as the input attribute order. +// The attribute buffers are copies of the associated attribute buffers, when provided, and otherwise empty. template requires requires(BufferType const& b) { { b.attributes } -> std::convertible_to>>; diff --git a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp index e040e52b3..861d69cb2 100644 --- a/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp +++ b/power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/serialization/deserializer.hpp @@ -745,8 +745,8 @@ class Deserializer { return {}; }(); auto const reordered_attribute_buffers = detail::is_columnar_v - ? detail::reordered_attribute_buffers(buffer, attributes) - : std::vector>{}; + ? detail::reordered_attribute_buffers(buffer, attributes) + : std::vector>{}; BufferView const buffer_view{ .buffer = &buffer, .idx = 0, .reordered_attribute_buffers = reordered_attribute_buffers};