From 8f35f3ffbc4d1bc326952dee40fab27524393bcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Thu, 3 Sep 2020 20:39:05 +0200 Subject: [PATCH] #123 implement memory footprint mode Implement a function and a mode for measuring total memory footprint of an object. - use partial specialization for dispatch - add implementation and tests for std::unique_ptr, std::string, std::vector --- .gitignore | 5 + src/checkpoint/checkpoint_api.h | 10 ++ src/checkpoint/checkpoint_api.impl.h | 5 + src/checkpoint/container/string_serialize.h | 6 + .../container/unique_ptr_serialize.h | 19 +++ src/checkpoint/container/vector_serialize.h | 6 + src/checkpoint/dispatch/dispatch.h | 11 ++ src/checkpoint/dispatch/dispatch.impl.h | 8 + .../dispatch/dispatch_serializer_nonbyte.h | 74 +++++++++ src/checkpoint/serializers/base_serializer.h | 2 + src/checkpoint/serializers/footprinter.cc | 58 +++++++ src/checkpoint/serializers/footprinter.h | 84 +++++++++++ .../serializers/serializers_headers.h | 1 + src/checkpoint/traits/serializable_traits.h | 17 +++ tests/unit/test_footprinter.cc | 142 ++++++++++++++++++ 15 files changed, 448 insertions(+) create mode 100644 src/checkpoint/serializers/footprinter.cc create mode 100644 src/checkpoint/serializers/footprinter.h create mode 100644 tests/unit/test_footprinter.cc diff --git a/.gitignore b/.gitignore index 567609b1..977974c7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ build/ +CMakeFiles/ + +*.a +*.cmake +*.tcl diff --git a/src/checkpoint/checkpoint_api.h b/src/checkpoint/checkpoint_api.h index 1f279f35..52b36fca 100644 --- a/src/checkpoint/checkpoint_api.h +++ b/src/checkpoint/checkpoint_api.h @@ -157,6 +157,16 @@ std::unique_ptr deserialize(SerializedReturnType&& in); template std::size_t getSize(T& target); +/** + * \brief Get memory footprint of \c target + * + * \param[in] target reference to \c T to measure footprint + * + * \return memory footprint of the \c target + */ +template +std::size_t getMemoryFootprint(T& target); + /** * \brief Serialize \c T to file with filename \c file * diff --git a/src/checkpoint/checkpoint_api.impl.h b/src/checkpoint/checkpoint_api.impl.h index 5e3c16a4..455ee735 100644 --- a/src/checkpoint/checkpoint_api.impl.h +++ b/src/checkpoint/checkpoint_api.impl.h @@ -91,6 +91,11 @@ std::size_t getSize(T& target) { return dispatch::Standard::size(target); } +template +std::size_t getMemoryFootprint(T& target) { + return dispatch::Standard::footprint(target); +} + template void serializeToFile(T& target, std::string const& file) { auto len = getSize(target); diff --git a/src/checkpoint/container/string_serialize.h b/src/checkpoint/container/string_serialize.h index e333a0ce..93611c9a 100644 --- a/src/checkpoint/container/string_serialize.h +++ b/src/checkpoint/container/string_serialize.h @@ -65,6 +65,12 @@ void serialize(Serializer& s, std::string& str) { dispatch::serializeArray(s, str.c_str(), str.size()); } +template +void getMemoryFootprint(Serializer& s, std::string& str) { + s.countBytes(str); + dispatch::serializeArray(s, str.c_str(), str.capacity()); +} + } /* end namespace checkpoint */ #endif /*INCLUDED_CHECKPOINT_CONTAINER_STRING_SERIALIZE_H*/ diff --git a/src/checkpoint/container/unique_ptr_serialize.h b/src/checkpoint/container/unique_ptr_serialize.h index 3d1953e8..b6c0a67f 100644 --- a/src/checkpoint/container/unique_ptr_serialize.h +++ b/src/checkpoint/container/unique_ptr_serialize.h @@ -66,6 +66,25 @@ void serialize(Serializer& s, std::unique_ptr& ptr) { } } +template +void getMemoryFootprint(Serializer& s, std::unique_ptr& ptr) { + s.countBytes(ptr); + + if (ptr != nullptr) { + s | *ptr; + } +} + +// FIXME - not related to unique_ptr +template +void getMemoryFootprint(Serializer& s, T* ptr) { + s.countBytes(ptr); + + if (ptr != nullptr) { + s | *ptr; + } +} + } /* end namespace checkpoint */ #endif /*INCLUDED_CHECKPOINT_CONTAINER_UNIQUE_PTR_SERIALIZE_H*/ diff --git a/src/checkpoint/container/vector_serialize.h b/src/checkpoint/container/vector_serialize.h index b3c9f81b..25f47e6b 100644 --- a/src/checkpoint/container/vector_serialize.h +++ b/src/checkpoint/container/vector_serialize.h @@ -65,6 +65,12 @@ void serialize(Serializer& s, std::vector& vec) { dispatch::serializeArray(s, &vec[0], vec.size()); } +template +void getMemoryFootprint(Serializer& s, std::vector& vec) { + s.countBytes(vec); + dispatch::serializeArray(s, &vec[0], vec.size()); +} + template void serialize(Serializer& s, std::vector& vec) { serializeVectorMeta(s, vec); diff --git a/src/checkpoint/dispatch/dispatch.h b/src/checkpoint/dispatch/dispatch.h index c5f1e886..297e0198 100644 --- a/src/checkpoint/dispatch/dispatch.h +++ b/src/checkpoint/dispatch/dispatch.h @@ -135,6 +135,17 @@ struct Standard { template static SerialSizeType size(T& target, Args&&... args); + /** + * \brief Recursively get the memory footprint of \c T + * + * \param[in] target the target to measure + * \param[in] args arguments to the footprinter's constructor + * + * \return memory footprint of \c T + */ + template + static SerialSizeType footprint(T& target, Args&&... args); + /** * \brief Pack \c target that requires \c size number of bytes. * diff --git a/src/checkpoint/dispatch/dispatch.impl.h b/src/checkpoint/dispatch/dispatch.impl.h index 61a645b5..8d40cb54 100644 --- a/src/checkpoint/dispatch/dispatch.impl.h +++ b/src/checkpoint/dispatch/dispatch.impl.h @@ -91,6 +91,14 @@ SerialSizeType Standard::size(T& target, Args&&... args) { return sizer.getSize(); } +template +SerialSizeType Standard::footprint(T& target, Args&&... args) { + auto footprinter = Traverse::with( + target, std::forward(args)... + ); + return footprinter.getMemoryFootprint(); +} + template PackerT Standard::pack(T& target, SerialSizeType const& size, Args&&... args) { return Traverse::with(target, size, std::forward(args)...); diff --git a/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h b/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h index a7f668df..b14630a4 100644 --- a/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h +++ b/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h @@ -219,6 +219,80 @@ struct SerializerDispatchNonByte { } }; +#if !HAS_DETECTION_COMPONENT +template +struct hasGetMemoryFootprint { + template < + typename C, + typename = decltype(std::declval().getMemoryFootprint(std::declval())) + > + static std::true_type test(int); + + template + static std::false_type test(...); + + static constexpr bool value = decltype(test(0))::value; +}; +#endif + +template +struct SerializerDispatchNonByte > { + using Dispatcher = MemoryFootprintDispatch; + + void operator()(Footprinter& s, T* val, SerialSizeType num); + + #if HAS_DETECTION_COMPONENT + template + using hasInGetMemoryFootprint = + std::enable_if_t< + SerializableTraits::has_get_memory_footprint_intrusive, T + >; + + template + using hasNoninGetMemoryFootprint = + std::enable_if_t< + SerializableTraits::has_get_memory_footprint_nonintrusive, T + >; + #else + template + using hasInGetMemoryFootprint = + std::enable_if_t::value, T>; + + template + using hasNoninGetMemoryFootprint = + std::enable_if_t::value, T>; + #endif + + template + void apply( + Footprinter& s, T* val, SerialSizeType num, + hasInGetMemoryFootprint* = nullptr + ) { + for (SerialSizeType i = 0; i < num; i++) { + Dispatcher::serializeIntrusive(s, val[i]); + } + } + + template + void apply( + Footprinter& s, T* val, SerialSizeType num, + hasNoninGetMemoryFootprint* = nullptr + ) { + for (SerialSizeType i = 0; i < num; i++) { + Dispatcher::serializeNonIntrusive(s, val[i]); + } + } +}; + +template +void SerializerDispatchNonByte< + Footprinter, + T, + MemoryFootprintDispatch +>::operator()(Footprinter& s, T* val, SerialSizeType num) { + apply(s, val, num); +} + }} /* end namespace checkpoint::dispatch */ #endif /*INCLUDED_CHECKPOINT_DISPATCH_DISPATCH_SERIALIZER_NONBYTE_H*/ diff --git a/src/checkpoint/serializers/base_serializer.h b/src/checkpoint/serializers/base_serializer.h index 2ddaf831..3c33f1b9 100644 --- a/src/checkpoint/serializers/base_serializer.h +++ b/src/checkpoint/serializers/base_serializer.h @@ -57,6 +57,7 @@ enum struct eSerializationMode : int8_t { Unpacking = 1, Packing = 2, Sizing = 3, + Footprinting = 4, Invalid = -1 }; @@ -79,6 +80,7 @@ struct Serializer { bool isSizing() const { return cur_mode_ == ModeType::Sizing; } bool isPacking() const { return cur_mode_ == ModeType::Packing; } bool isUnpacking() const { return cur_mode_ == ModeType::Unpacking; } + bool isFootprinting() const { return cur_mode_ == ModeType::Footprinting; } template void contiguousTyped(SerializerT& serdes, T* ptr, SerialSizeType num_elms) { diff --git a/src/checkpoint/serializers/footprinter.cc b/src/checkpoint/serializers/footprinter.cc new file mode 100644 index 00000000..b478ef65 --- /dev/null +++ b/src/checkpoint/serializers/footprinter.cc @@ -0,0 +1,58 @@ +/* +//@HEADER +// ***************************************************************************** +// +// footprinter.cc +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#include "checkpoint/serializers/footprinter.h" + +namespace checkpoint { + +SerialSizeType Footprinter::getMemoryFootprint() const { + return num_bytes_; +} + +void Footprinter::contiguousBytes(void*, SerialSizeType size, SerialSizeType num_elms) { + //printf("contiguousBytes: size=%zu, num_elms=%zu\n", size, num_elms); + num_bytes_ += size * num_elms; +} + +} /* end namespace checkpoint */ diff --git a/src/checkpoint/serializers/footprinter.h b/src/checkpoint/serializers/footprinter.h new file mode 100644 index 00000000..0d623aee --- /dev/null +++ b/src/checkpoint/serializers/footprinter.h @@ -0,0 +1,84 @@ +/* +//@HEADER +// ***************************************************************************** +// +// footprinter.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_SERIALIZERS_FOOTPRINTER_H +#define INCLUDED_CHECKPOINT_SERIALIZERS_FOOTPRINTER_H + +#include "checkpoint/common.h" +#include "checkpoint/serializers/base_serializer.h" + +//#include + +namespace checkpoint { + +template +struct MemoryFootprintDispatch { + static void serializeIntrusive(SerializerT& s, T& t) { + //printf("MemoryFootprintDispatch::serializeIntrusive - type is %s\n", typeid(T).name()); + t.getMemoryFootprint(s); + } + static void serializeNonIntrusive(SerializerT& s, T& t) { + //printf("MemoryFootprintDispatch::serializeNonIntrusive - type is %s\n", typeid(T).name()); + getMemoryFootprint(s, t); + } +}; + +struct Footprinter : Serializer { + template + using DispatcherType = MemoryFootprintDispatch; + + Footprinter() : Serializer(ModeType::Footprinting) { } + + SerialSizeType getMemoryFootprint() const; + void contiguousBytes(void*, SerialSizeType size, SerialSizeType num_elms); + template + void countBytes(const T& t) { num_bytes_ += sizeof t; } + +private: + SerialSizeType num_bytes_ = 0; +}; + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_SERIALIZERS_FOOTPRINTER_H*/ diff --git a/src/checkpoint/serializers/serializers_headers.h b/src/checkpoint/serializers/serializers_headers.h index 92b09b7b..b08569f6 100644 --- a/src/checkpoint/serializers/serializers_headers.h +++ b/src/checkpoint/serializers/serializers_headers.h @@ -47,6 +47,7 @@ #include "checkpoint/common.h" #include "checkpoint/serializers/base_serializer.h" +#include "checkpoint/serializers/footprinter.h" #include "checkpoint/serializers/sizer.h" #include "checkpoint/serializers/packer.h" #include "checkpoint/serializers/unpacker.h" diff --git a/src/checkpoint/traits/serializable_traits.h b/src/checkpoint/traits/serializable_traits.h index 30840c93..9cfcf9c2 100644 --- a/src/checkpoint/traits/serializable_traits.h +++ b/src/checkpoint/traits/serializable_traits.h @@ -132,6 +132,23 @@ struct SerializableTraits { static constexpr auto const has_serialize_function = has_serialize_instrusive or has_serialize_noninstrusive; + template + using footprint_intrusive_t = decltype( + std::declval().getMemoryFootprint(std::declval()) + ); + + static constexpr auto const has_get_memory_footprint_intrusive = + detection::is_detected::value; + + template + using footprint_nonintrustive_t = decltype(getMemoryFootprint( + std::declval(), + std::declval() + )); + + static constexpr auto const has_get_memory_footprint_nonintrusive = + detection::is_detected::value; + // This defines what it means to have parent serializability static constexpr auto const has_parent_serialize = has_serializeParent::value; diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc new file mode 100644 index 00000000..32003a27 --- /dev/null +++ b/tests/unit/test_footprinter.cc @@ -0,0 +1,142 @@ +/* +//@HEADER +// ***************************************************************************** +// +// test_footprinter.cc +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2019 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#include + +#include "test_harness.h" + +#include + +namespace checkpoint { namespace tests { namespace unit { + +struct TestFootprinter : TestHarness { }; + +struct Test1 { + template + void serialize(Serializer& s) { + s | d; + } + + template + void getMemoryFootprint(Serializer& s) { + s | d; + } + + double d; +}; + +struct Test2 { + float f; +}; + +template +void getMemoryFootprint(Serializer& s, Test2 t) { + s | t.f; +} + +TEST_F(TestFootprinter, test_basic_types) { + int i; + EXPECT_EQ(checkpoint::getMemoryFootprint(i), sizeof(i)); + + double d; + EXPECT_EQ(checkpoint::getMemoryFootprint(d), sizeof(d)); + + { + int* ptr = nullptr; + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr)); + } + + { + int* ptr = new int(); + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr) + sizeof(*ptr)); + } +} + +TEST_F(TestFootprinter, test_unique_ptr) { + { + std::unique_ptr ptr; + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr)); + } + + { + auto ptr = std::make_unique(); + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr) + sizeof(*ptr)); + } + + { + auto ptr = std::make_unique(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(ptr), + sizeof(ptr) + sizeof(*ptr) + ); + } +} + +TEST_F(TestFootprinter, test_string) { + std::string s = "123456789"; + EXPECT_EQ( + checkpoint::getMemoryFootprint(s), + sizeof(s) + s.capacity() * sizeof(s[0])); +} + +TEST_F(TestFootprinter, test_vector) { + { + std::vector v = {1, 2, 3, 4, 5}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(v), + sizeof(v) + v.capacity() * sizeof(int)); + } + + { + std::vector v = { new Test1(), nullptr }; + EXPECT_EQ( + checkpoint::getMemoryFootprint(v), + sizeof(v) + v.capacity() * sizeof(Test1*) + sizeof(*v[0]) + ); + + delete v[0]; + } +} + +}}} // end namespace checkpoint::tests::unit