From 69c8d40cbb34f03a53b12197e4e787864f8d5b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Wed, 2 Sep 2020 15:38:13 +0200 Subject: [PATCH 01/16] #123 add README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index e69de29b..90a84867 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,3 @@ +# *checkpoint* => serialization and checkpointing library + +![Docker Image CI](https://github.com/DARMA-tasking/checkpoint/workflows/Docker%20Image%20CI/badge.svg) From f29cf0c14740dfe17d518cd71c97d9bc3c10e492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Thu, 3 Sep 2020 20:39:05 +0200 Subject: [PATCH 02/16] #123 implement memory footprint mode Implement a function and a mode for measuring total memory footprint of an object. - use existing 'serialize' infrastructure - add implementation and tests for std::unique_ptr, std::string, std::vector --- .gitignore | 6 + src/checkpoint/checkpoint_api.h | 10 ++ src/checkpoint/checkpoint_api.impl.h | 5 + .../container/unique_ptr_serialize.h | 10 ++ src/checkpoint/dispatch/dispatch.h | 11 ++ src/checkpoint/dispatch/dispatch.impl.h | 8 + .../dispatch/dispatch_serializer_nonbyte.h | 16 ++ src/checkpoint/serializers/base_serializer.h | 2 + src/checkpoint/serializers/footprinter.cc | 58 +++++++ src/checkpoint/serializers/footprinter.h | 67 ++++++++ .../serializers/serializers_headers.h | 1 + src/checkpoint/traits/serializable_traits.h | 17 ++ tests/unit/test_footprinter.cc | 147 ++++++++++++++++++ 13 files changed, 358 insertions(+) create mode 100644 src/checkpoint/serializers/footprinter.cc create mode 100644 src/checkpoint/serializers/footprinter.h create mode 100644 tests/unit/test_footprinter.cc diff --git a/.gitignore b/.gitignore index 7798ee77..a7d20e73 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ build/ +CMakeFiles/ + .dir-locals.el + +*.a +*.cmake +*.tcl diff --git a/src/checkpoint/checkpoint_api.h b/src/checkpoint/checkpoint_api.h index 1f279f35..52b36fca 100644 --- a/src/checkpoint/checkpoint_api.h +++ b/src/checkpoint/checkpoint_api.h @@ -157,6 +157,16 @@ std::unique_ptr deserialize(SerializedReturnType&& in); template std::size_t getSize(T& target); +/** + * \brief Get memory footprint of \c target + * + * \param[in] target reference to \c T to measure footprint + * + * \return memory footprint of the \c target + */ +template +std::size_t getMemoryFootprint(T& target); + /** * \brief Serialize \c T to file with filename \c file * diff --git a/src/checkpoint/checkpoint_api.impl.h b/src/checkpoint/checkpoint_api.impl.h index 5e3c16a4..455ee735 100644 --- a/src/checkpoint/checkpoint_api.impl.h +++ b/src/checkpoint/checkpoint_api.impl.h @@ -91,6 +91,11 @@ std::size_t getSize(T& target) { return dispatch::Standard::size(target); } +template +std::size_t getMemoryFootprint(T& target) { + return dispatch::Standard::footprint(target); +} + template void serializeToFile(T& target, std::string const& file) { auto len = getSize(target); diff --git a/src/checkpoint/container/unique_ptr_serialize.h b/src/checkpoint/container/unique_ptr_serialize.h index 3d1953e8..ad591a44 100644 --- a/src/checkpoint/container/unique_ptr_serialize.h +++ b/src/checkpoint/container/unique_ptr_serialize.h @@ -66,6 +66,16 @@ void serialize(Serializer& s, std::unique_ptr& ptr) { } } +// FIXME - not related to unique_ptr +template +void serialize(Serializer& s, T* ptr) { + s.countBytes(ptr); + + if (ptr != nullptr) { + s | *ptr; + } +} + } /* end namespace checkpoint */ #endif /*INCLUDED_CHECKPOINT_CONTAINER_UNIQUE_PTR_SERIALIZE_H*/ diff --git a/src/checkpoint/dispatch/dispatch.h b/src/checkpoint/dispatch/dispatch.h index c5f1e886..297e0198 100644 --- a/src/checkpoint/dispatch/dispatch.h +++ b/src/checkpoint/dispatch/dispatch.h @@ -135,6 +135,17 @@ struct Standard { template static SerialSizeType size(T& target, Args&&... args); + /** + * \brief Recursively get the memory footprint of \c T + * + * \param[in] target the target to measure + * \param[in] args arguments to the footprinter's constructor + * + * \return memory footprint of \c T + */ + template + static SerialSizeType footprint(T& target, Args&&... args); + /** * \brief Pack \c target that requires \c size number of bytes. * diff --git a/src/checkpoint/dispatch/dispatch.impl.h b/src/checkpoint/dispatch/dispatch.impl.h index 61a645b5..8d40cb54 100644 --- a/src/checkpoint/dispatch/dispatch.impl.h +++ b/src/checkpoint/dispatch/dispatch.impl.h @@ -91,6 +91,14 @@ SerialSizeType Standard::size(T& target, Args&&... args) { return sizer.getSize(); } +template +SerialSizeType Standard::footprint(T& target, Args&&... args) { + auto footprinter = Traverse::with( + target, std::forward(args)... + ); + return footprinter.getMemoryFootprint(); +} + template PackerT Standard::pack(T& target, SerialSizeType const& size, Args&&... args) { return Traverse::with(target, size, std::forward(args)...); diff --git a/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h b/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h index a7f668df..446202a2 100644 --- a/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h +++ b/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h @@ -219,6 +219,22 @@ struct SerializerDispatchNonByte { } }; +#if !HAS_DETECTION_COMPONENT +template +struct hasGetMemoryFootprint { + template < + typename C, + typename = decltype(std::declval().getMemoryFootprint(std::declval())) + > + static std::true_type test(int); + + template + static std::false_type test(...); + + static constexpr bool value = decltype(test(0))::value; +}; +#endif + }} /* end namespace checkpoint::dispatch */ #endif /*INCLUDED_CHECKPOINT_DISPATCH_DISPATCH_SERIALIZER_NONBYTE_H*/ diff --git a/src/checkpoint/serializers/base_serializer.h b/src/checkpoint/serializers/base_serializer.h index 2ddaf831..3c33f1b9 100644 --- a/src/checkpoint/serializers/base_serializer.h +++ b/src/checkpoint/serializers/base_serializer.h @@ -57,6 +57,7 @@ enum struct eSerializationMode : int8_t { Unpacking = 1, Packing = 2, Sizing = 3, + Footprinting = 4, Invalid = -1 }; @@ -79,6 +80,7 @@ struct Serializer { bool isSizing() const { return cur_mode_ == ModeType::Sizing; } bool isPacking() const { return cur_mode_ == ModeType::Packing; } bool isUnpacking() const { return cur_mode_ == ModeType::Unpacking; } + bool isFootprinting() const { return cur_mode_ == ModeType::Footprinting; } template void contiguousTyped(SerializerT& serdes, T* ptr, SerialSizeType num_elms) { diff --git a/src/checkpoint/serializers/footprinter.cc b/src/checkpoint/serializers/footprinter.cc new file mode 100644 index 00000000..b478ef65 --- /dev/null +++ b/src/checkpoint/serializers/footprinter.cc @@ -0,0 +1,58 @@ +/* +//@HEADER +// ***************************************************************************** +// +// footprinter.cc +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#include "checkpoint/serializers/footprinter.h" + +namespace checkpoint { + +SerialSizeType Footprinter::getMemoryFootprint() const { + return num_bytes_; +} + +void Footprinter::contiguousBytes(void*, SerialSizeType size, SerialSizeType num_elms) { + //printf("contiguousBytes: size=%zu, num_elms=%zu\n", size, num_elms); + num_bytes_ += size * num_elms; +} + +} /* end namespace checkpoint */ diff --git a/src/checkpoint/serializers/footprinter.h b/src/checkpoint/serializers/footprinter.h new file mode 100644 index 00000000..d7d94024 --- /dev/null +++ b/src/checkpoint/serializers/footprinter.h @@ -0,0 +1,67 @@ +/* +//@HEADER +// ***************************************************************************** +// +// footprinter.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_SERIALIZERS_FOOTPRINTER_H +#define INCLUDED_CHECKPOINT_SERIALIZERS_FOOTPRINTER_H + +#include "checkpoint/common.h" +#include "checkpoint/serializers/base_serializer.h" + +namespace checkpoint { + +struct Footprinter : Serializer { + Footprinter() : Serializer(ModeType::Footprinting) { } + + SerialSizeType getMemoryFootprint() const; + void contiguousBytes(void*, SerialSizeType size, SerialSizeType num_elms); + template + void countBytes(const T& t) { num_bytes_ += sizeof t; } + +private: + SerialSizeType num_bytes_ = 0; +}; + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_SERIALIZERS_FOOTPRINTER_H*/ diff --git a/src/checkpoint/serializers/serializers_headers.h b/src/checkpoint/serializers/serializers_headers.h index 92b09b7b..b08569f6 100644 --- a/src/checkpoint/serializers/serializers_headers.h +++ b/src/checkpoint/serializers/serializers_headers.h @@ -47,6 +47,7 @@ #include "checkpoint/common.h" #include "checkpoint/serializers/base_serializer.h" +#include "checkpoint/serializers/footprinter.h" #include "checkpoint/serializers/sizer.h" #include "checkpoint/serializers/packer.h" #include "checkpoint/serializers/unpacker.h" diff --git a/src/checkpoint/traits/serializable_traits.h b/src/checkpoint/traits/serializable_traits.h index 30840c93..9cfcf9c2 100644 --- a/src/checkpoint/traits/serializable_traits.h +++ b/src/checkpoint/traits/serializable_traits.h @@ -132,6 +132,23 @@ struct SerializableTraits { static constexpr auto const has_serialize_function = has_serialize_instrusive or has_serialize_noninstrusive; + template + using footprint_intrusive_t = decltype( + std::declval().getMemoryFootprint(std::declval()) + ); + + static constexpr auto const has_get_memory_footprint_intrusive = + detection::is_detected::value; + + template + using footprint_nonintrustive_t = decltype(getMemoryFootprint( + std::declval(), + std::declval() + )); + + static constexpr auto const has_get_memory_footprint_nonintrusive = + detection::is_detected::value; + // This defines what it means to have parent serializability static constexpr auto const has_parent_serialize = has_serializeParent::value; diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc new file mode 100644 index 00000000..55a0dcca --- /dev/null +++ b/tests/unit/test_footprinter.cc @@ -0,0 +1,147 @@ +/* +//@HEADER +// ***************************************************************************** +// +// test_footprinter.cc +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2019 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#include + +#include "test_harness.h" + +#include + +namespace checkpoint { namespace tests { namespace unit { + +struct TestFootprinter : TestHarness { }; + +struct Test1 { + template + void serialize(Serializer& s) { + s | d; + } + + template + void getMemoryFootprint(Serializer& s) { + s | d; + } + + double d; +}; + +struct Test2 { + float f; +}; + +template +void getMemoryFootprint(Serializer& s, Test2 t) { + s | t.f; +} + +template +void serialize(Serializer& s, Test2 t) { + s | t.f; +} + +TEST_F(TestFootprinter, test_basic_types) { + int i; + EXPECT_EQ(checkpoint::getMemoryFootprint(i), sizeof(i)); + + double d; + EXPECT_EQ(checkpoint::getMemoryFootprint(d), sizeof(d)); + + { + int* ptr = nullptr; + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr)); + } + + { + int* ptr = new int(); + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr) + sizeof(*ptr)); + } +} + +TEST_F(TestFootprinter, test_unique_ptr) { + { + std::unique_ptr ptr; + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr)); + } + + { + auto ptr = std::make_unique(); + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr) + sizeof(*ptr)); + } + + { + auto ptr = std::make_unique(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(ptr), + sizeof(ptr) + sizeof(*ptr) + ); + } +} + +TEST_F(TestFootprinter, test_string) { + std::string s = "123456789"; + EXPECT_EQ( + checkpoint::getMemoryFootprint(s), + sizeof(s) + s.capacity() * sizeof(s[0])); +} + +TEST_F(TestFootprinter, test_vector) { + { + std::vector v = {1, 2, 3, 4, 5}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(v), + sizeof(v) + v.capacity() * sizeof(int)); + } + + { + std::vector v = { new Test1(), nullptr }; + EXPECT_EQ( + checkpoint::getMemoryFootprint(v), + sizeof(v) + v.capacity() * sizeof(Test1*) + sizeof(*v[0]) + ); + + delete v[0]; + } +} + +}}} // end namespace checkpoint::tests::unit From 72af815c337a09e86d6efc5d5f04a1e24f36870d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Wed, 9 Sep 2020 17:29:33 +0200 Subject: [PATCH 03/16] #123 implement memory footprinting for standard containers - adjust serialization code for Footprinting mode - add unit tests for measuring memory footprint of standard containers - fix compiler warning in traversal test --- src/checkpoint/checkpoint.h | 1 + src/checkpoint/container/array_serialize.h | 1 + .../container/container_serialize.h | 6 +- .../raw_ptr_serialize.h} | 30 ++- src/checkpoint/container/string_serialize.h | 9 +- src/checkpoint/container/tuple_serialize.h | 1 + .../container/unique_ptr_serialize.h | 16 +- src/checkpoint/container/vector_serialize.h | 9 +- .../dispatch/dispatch_serializer_nonbyte.h | 16 -- src/checkpoint/serializers/base_serializer.h | 3 + src/checkpoint/serializers/footprinter.h | 13 +- src/checkpoint/traits/serializable_traits.h | 17 -- tests/unit/test_footprinter.cc | 208 +++++++++++++++--- tests/unit/test_traversal.cc | 2 +- 14 files changed, 246 insertions(+), 86 deletions(-) rename src/checkpoint/{serializers/footprinter.cc => container/raw_ptr_serialize.h} (76%) diff --git a/src/checkpoint/checkpoint.h b/src/checkpoint/checkpoint.h index a08f5593..67e6440c 100644 --- a/src/checkpoint/checkpoint.h +++ b/src/checkpoint/checkpoint.h @@ -53,6 +53,7 @@ #include "checkpoint/container/enum_serialize.h" #include "checkpoint/container/list_serialize.h" #include "checkpoint/container/map_serialize.h" +#include "checkpoint/container/raw_ptr_serialize.h" #include "checkpoint/container/string_serialize.h" #include "checkpoint/container/tuple_serialize.h" #include "checkpoint/container/vector_serialize.h" diff --git a/src/checkpoint/container/array_serialize.h b/src/checkpoint/container/array_serialize.h index b4de49a0..6f8ff3d2 100644 --- a/src/checkpoint/container/array_serialize.h +++ b/src/checkpoint/container/array_serialize.h @@ -54,6 +54,7 @@ namespace checkpoint { template void serialize(Serializer& s, std::array& array) { + s.countBytes(array); dispatch::serializeArray(s, &array[0], array.size()); } diff --git a/src/checkpoint/container/container_serialize.h b/src/checkpoint/container/container_serialize.h index 8d81694b..db765f32 100644 --- a/src/checkpoint/container/container_serialize.h +++ b/src/checkpoint/container/container_serialize.h @@ -54,7 +54,11 @@ template inline typename ContainerT::size_type serializeContainerSize(Serializer& s, ContainerT& cont) { typename ContainerT::size_type cont_size = cont.size(); - s | cont_size; + if (s.isFootprinting()) { + s.countBytes(cont); + } else { + s | cont_size; + } return cont_size; } diff --git a/src/checkpoint/serializers/footprinter.cc b/src/checkpoint/container/raw_ptr_serialize.h similarity index 76% rename from src/checkpoint/serializers/footprinter.cc rename to src/checkpoint/container/raw_ptr_serialize.h index b478ef65..7e4418d5 100644 --- a/src/checkpoint/serializers/footprinter.cc +++ b/src/checkpoint/container/raw_ptr_serialize.h @@ -2,7 +2,7 @@ //@HEADER // ***************************************************************************** // -// footprinter.cc +// raw_ptr_serialize.h // DARMA Toolkit v. 1.0.0 // DARMA/checkpoint => Serialization Library // @@ -42,17 +42,31 @@ //@HEADER */ -#include "checkpoint/serializers/footprinter.h" +#if !defined INCLUDED_CHECKPOINT_CONTAINER_RAW_PTR_SERIALIZE_H +#define INCLUDED_CHECKPOINT_CONTAINER_RAW_PTR_SERIALIZE_H + +#include "checkpoint/common.h" namespace checkpoint { -SerialSizeType Footprinter::getMemoryFootprint() const { - return num_bytes_; -} +/** + * \brief Serialize raw pointer \c ptr + * + * For footprinting mode, count the pointer size and follow it (note that it + * doesn't work correctly for C-style arrays!). + * + * \param[in] serializer serializer to use + * \param[in] ptr pointer to serialize + */ +template +void serialize(Serializer& s, T* ptr) { + s.countBytes(ptr); -void Footprinter::contiguousBytes(void*, SerialSizeType size, SerialSizeType num_elms) { - //printf("contiguousBytes: size=%zu, num_elms=%zu\n", size, num_elms); - num_bytes_ += size * num_elms; + if (ptr != nullptr) { + s | *ptr; + } } } /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_CONTAINER_RAW_PTR_SERIALIZE_H*/ diff --git a/src/checkpoint/container/string_serialize.h b/src/checkpoint/container/string_serialize.h index e333a0ce..64c47e26 100644 --- a/src/checkpoint/container/string_serialize.h +++ b/src/checkpoint/container/string_serialize.h @@ -61,8 +61,13 @@ void serializeStringMeta(Serializer& s, std::string& str) { template void serialize(Serializer& s, std::string& str) { - serializeStringMeta(s, str); - dispatch::serializeArray(s, str.c_str(), str.size()); + if (s.isFootprinting()) { + s.countBytes(str); + dispatch::serializeArray(s, str.c_str(), str.capacity()); + } else { + serializeStringMeta(s, str); + dispatch::serializeArray(s, str.c_str(), str.size()); + } } } /* end namespace checkpoint */ diff --git a/src/checkpoint/container/tuple_serialize.h b/src/checkpoint/container/tuple_serialize.h index 1bb55721..f34c260e 100644 --- a/src/checkpoint/container/tuple_serialize.h +++ b/src/checkpoint/container/tuple_serialize.h @@ -67,6 +67,7 @@ void serialize(Serializer& s, std::tuple& tuple) { template void serialize(Serializer& s, std::pair& pair) { + s.countBytes(pair); s | pair.first; s | pair.second; } diff --git a/src/checkpoint/container/unique_ptr_serialize.h b/src/checkpoint/container/unique_ptr_serialize.h index ad591a44..12c1d897 100644 --- a/src/checkpoint/container/unique_ptr_serialize.h +++ b/src/checkpoint/container/unique_ptr_serialize.h @@ -54,7 +54,11 @@ namespace checkpoint { template void serialize(Serializer& s, std::unique_ptr& ptr) { bool is_null = ptr == nullptr; - s | is_null; + if (s.isFootprinting()) { + s.countBytes(ptr); + } else { + s | is_null; + } if (not is_null) { T* t = ptr.get(); @@ -66,16 +70,6 @@ void serialize(Serializer& s, std::unique_ptr& ptr) { } } -// FIXME - not related to unique_ptr -template -void serialize(Serializer& s, T* ptr) { - s.countBytes(ptr); - - if (ptr != nullptr) { - s | *ptr; - } -} - } /* end namespace checkpoint */ #endif /*INCLUDED_CHECKPOINT_CONTAINER_UNIQUE_PTR_SERIALIZE_H*/ diff --git a/src/checkpoint/container/vector_serialize.h b/src/checkpoint/container/vector_serialize.h index b3c9f81b..f9d87c4a 100644 --- a/src/checkpoint/container/vector_serialize.h +++ b/src/checkpoint/container/vector_serialize.h @@ -61,8 +61,13 @@ void serializeVectorMeta(Serializer& s, std::vector& vec) { template void serialize(Serializer& s, std::vector& vec) { - serializeVectorMeta(s, vec); - dispatch::serializeArray(s, &vec[0], vec.size()); + if (s.isFootprinting()) { + s.countBytes(vec); + dispatch::serializeArray(s, &vec[0], vec.capacity()); + } else { + serializeVectorMeta(s, vec); + dispatch::serializeArray(s, &vec[0], vec.size()); + } } template diff --git a/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h b/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h index 446202a2..a7f668df 100644 --- a/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h +++ b/src/checkpoint/dispatch/dispatch_serializer_nonbyte.h @@ -219,22 +219,6 @@ struct SerializerDispatchNonByte { } }; -#if !HAS_DETECTION_COMPONENT -template -struct hasGetMemoryFootprint { - template < - typename C, - typename = decltype(std::declval().getMemoryFootprint(std::declval())) - > - static std::true_type test(int); - - template - static std::false_type test(...); - - static constexpr bool value = decltype(test(0))::value; -}; -#endif - }} /* end namespace checkpoint::dispatch */ #endif /*INCLUDED_CHECKPOINT_DISPATCH_DISPATCH_SERIALIZER_NONBYTE_H*/ diff --git a/src/checkpoint/serializers/base_serializer.h b/src/checkpoint/serializers/base_serializer.h index 3c33f1b9..d47f6610 100644 --- a/src/checkpoint/serializers/base_serializer.h +++ b/src/checkpoint/serializers/base_serializer.h @@ -82,6 +82,9 @@ struct Serializer { bool isUnpacking() const { return cur_mode_ == ModeType::Unpacking; } bool isFootprinting() const { return cur_mode_ == ModeType::Footprinting; } + template + void countBytes(const T& t) {} + template void contiguousTyped(SerializerT& serdes, T* ptr, SerialSizeType num_elms) { serdes.contiguousBytes(static_cast(ptr), sizeof(T), num_elms); diff --git a/src/checkpoint/serializers/footprinter.h b/src/checkpoint/serializers/footprinter.h index d7d94024..1c301310 100644 --- a/src/checkpoint/serializers/footprinter.h +++ b/src/checkpoint/serializers/footprinter.h @@ -53,10 +53,17 @@ namespace checkpoint { struct Footprinter : Serializer { Footprinter() : Serializer(ModeType::Footprinting) { } - SerialSizeType getMemoryFootprint() const; - void contiguousBytes(void*, SerialSizeType size, SerialSizeType num_elms); + SerialSizeType getMemoryFootprint() const { + return num_bytes_; + } + void contiguousBytes(void*, SerialSizeType size, SerialSizeType num_elms) { + num_bytes_ += size * num_elms; + } + template - void countBytes(const T& t) { num_bytes_ += sizeof t; } + void countBytes(const T& t) { // this needs a better name + num_bytes_ += sizeof(t); + } private: SerialSizeType num_bytes_ = 0; diff --git a/src/checkpoint/traits/serializable_traits.h b/src/checkpoint/traits/serializable_traits.h index 9cfcf9c2..30840c93 100644 --- a/src/checkpoint/traits/serializable_traits.h +++ b/src/checkpoint/traits/serializable_traits.h @@ -132,23 +132,6 @@ struct SerializableTraits { static constexpr auto const has_serialize_function = has_serialize_instrusive or has_serialize_noninstrusive; - template - using footprint_intrusive_t = decltype( - std::declval().getMemoryFootprint(std::declval()) - ); - - static constexpr auto const has_get_memory_footprint_intrusive = - detection::is_detected::value; - - template - using footprint_nonintrustive_t = decltype(getMemoryFootprint( - std::declval(), - std::declval() - )); - - static constexpr auto const has_get_memory_footprint_nonintrusive = - detection::is_detected::value; - // This defines what it means to have parent serializability static constexpr auto const has_parent_serialize = has_serializeParent::value; diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc index 55a0dcca..b3d5a075 100644 --- a/tests/unit/test_footprinter.cc +++ b/tests/unit/test_footprinter.cc @@ -53,40 +53,37 @@ namespace checkpoint { namespace tests { namespace unit { struct TestFootprinter : TestHarness { }; struct Test1 { - template - void serialize(Serializer& s) { - s | d; - } + double d; template - void getMemoryFootprint(Serializer& s) { + void serialize(Serializer& s) { s | d; } - - double d; }; struct Test2 { float f; }; -template -void getMemoryFootprint(Serializer& s, Test2 t) { - s | t.f; -} - template void serialize(Serializer& s, Test2 t) { s | t.f; } -TEST_F(TestFootprinter, test_basic_types) { +TEST_F(TestFootprinter, test_fundamental_types) { int i; EXPECT_EQ(checkpoint::getMemoryFootprint(i), sizeof(i)); double d; EXPECT_EQ(checkpoint::getMemoryFootprint(d), sizeof(d)); + auto str = "12345"; + EXPECT_EQ( + checkpoint::getMemoryFootprint(str), + sizeof(str) + sizeof(*str) + // actually: sizeof(str) + 6 * sizeof(str[0]) + ); + { int* ptr = nullptr; EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr)); @@ -94,30 +91,156 @@ TEST_F(TestFootprinter, test_basic_types) { { int* ptr = new int(); - EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr) + sizeof(*ptr)); + EXPECT_EQ( + checkpoint::getMemoryFootprint(ptr), + sizeof(ptr) + sizeof(*ptr) + ); + delete ptr; + } + + { + int *arr = new int[5]; + EXPECT_EQ( + checkpoint::getMemoryFootprint(arr), + sizeof(arr) + sizeof(*arr) + // actually: sizeof(arr) + 5 * sizeof(arr[0]) + ); + delete [] arr; + } + + { + int arr[] = {1, 2, 3, 4, 5}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(arr), + sizeof(&arr) + sizeof(arr[0]) + // actually: sizeof(&arr) + sizeof(arr) + ); } } -TEST_F(TestFootprinter, test_unique_ptr) { +TEST_F(TestFootprinter, test_array) { + std::array a = {1, 2, 3}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(a), + sizeof(a) + a.size() * sizeof(a[0]) + ); +} + +TEST_F(TestFootprinter, test_deque) { + std::deque d = {1, 2, 3}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(d), + sizeof(d) + d.size() * sizeof(d.front()) + ); +} + +TEST_F(TestFootprinter, test_enum) { { - std::unique_ptr ptr; - EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr)); + enum color + { + red, + green, + blue + }; + color col = red; + + EXPECT_EQ(checkpoint::getMemoryFootprint(col), sizeof(col)); } { - auto ptr = std::make_unique(); - EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr) + sizeof(*ptr)); + enum struct color : char + { + red = 'r', + green = 'g', + blue = 'b' + }; + color col = color::green; + + EXPECT_EQ(checkpoint::getMemoryFootprint(col), sizeof(char)); + } +} + +TEST_F(TestFootprinter, test_list) { + std::list l = {1, 2, 3}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(l), + sizeof(l) + l.size() * sizeof(l.front()) + ); +} + +TEST_F(TestFootprinter, test_map) { + { + std::map m = {{1, 'a'}, {2, 'b'}, {3, 'c'}}; + auto p = *m.begin(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(m), + sizeof(m) + m.size() * (sizeof(p) + sizeof(p.first) + sizeof(p.second)) + ); } { - auto ptr = std::make_unique(); + std::unordered_map m = {{1, 'a'}, {2, 'b'}, {3, 'c'}}; + auto p = *m.begin(); EXPECT_EQ( - checkpoint::getMemoryFootprint(ptr), - sizeof(ptr) + sizeof(*ptr) + checkpoint::getMemoryFootprint(m), + sizeof(m) + m.size() * (sizeof(p) + sizeof(p.first) + sizeof(p.second)) + ); + } + + { + std::multimap m = {{1, 'a'}, {1, 'b'}, {1, 'c'}}; + auto p = *m.begin(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(m), + sizeof(m) + m.size() * (sizeof(p) + sizeof(p.first) + sizeof(p.second)) + ); + } + + { + std::unordered_multimap m = {{1, 'a'}, {1, 'b'}, {1, 'c'}}; + auto p = *m.begin(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(m), + sizeof(m) + m.size() * (sizeof(p) + sizeof(p.first) + sizeof(p.second)) + ); + } +} + +TEST_F(TestFootprinter, test_set) { + { + std::set s = {1, 2, 3}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(s), + sizeof(s) + s.size() * sizeof(*s.begin()) + ); + } + + { + std::unordered_set s = {1, 2, 3}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(s), + sizeof(s) + s.size() * sizeof(*s.begin()) + ); + } + + { + std::multiset s = {1, 1, 1}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(s), + sizeof(s) + s.size() * sizeof(*s.begin()) + ); + } + + { + std::unordered_multiset s = {1, 1, 1}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(s), + sizeof(s) + s.size() * sizeof(*s.begin()) ); } } +// does not account for small string optimisation TEST_F(TestFootprinter, test_string) { std::string s = "123456789"; EXPECT_EQ( @@ -125,21 +248,56 @@ TEST_F(TestFootprinter, test_string) { sizeof(s) + s.capacity() * sizeof(s[0])); } +// naive approach, just sum memory usage of all elements +TEST_F(TestFootprinter, test_tuple) { + int i = 1; + double d = 3.8; + std::string s = "123456789"; + auto t = std::make_tuple(i, d, s); + EXPECT_EQ( + checkpoint::getMemoryFootprint(t), + sizeof(i) + sizeof(d) + sizeof(s) + s.capacity() * sizeof(s[0]) + ); +} + +TEST_F(TestFootprinter, test_unique_ptr) { + { + std::unique_ptr ptr; + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr)); + } + + { + auto ptr = std::make_unique(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(ptr), + sizeof(ptr) + sizeof(*ptr) + ); + } + + { + auto ptr = std::make_unique(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(ptr), + sizeof(ptr) + sizeof(*ptr) + ); + } +} + TEST_F(TestFootprinter, test_vector) { { std::vector v = {1, 2, 3, 4, 5}; EXPECT_EQ( checkpoint::getMemoryFootprint(v), - sizeof(v) + v.capacity() * sizeof(int)); + sizeof(v) + v.capacity() * sizeof(v[0]) + ); } { std::vector v = { new Test1(), nullptr }; EXPECT_EQ( checkpoint::getMemoryFootprint(v), - sizeof(v) + v.capacity() * sizeof(Test1*) + sizeof(*v[0]) + sizeof(v) + v.capacity() * sizeof(v[0]) + sizeof(*v[0]) ); - delete v[0]; } } diff --git a/tests/unit/test_traversal.cc b/tests/unit/test_traversal.cc index a457c439..316a9e19 100644 --- a/tests/unit/test_traversal.cc +++ b/tests/unit/test_traversal.cc @@ -159,7 +159,7 @@ TEST_F(TestTraversal, test_traversal) { auto traverse = checkpoint::dispatch::Traverse::with(t); - EXPECT_EQ(traverse.bytes_, 10*4 + 10*8 + 4 + 4 + 100*4 + 8*3); + EXPECT_EQ(traverse.bytes_, std::size_t{10*4 + 10*8 + 4 + 4 + 100*4 + 8*3}); } TEST_F(TestTraversal, test_traversal_dispatcher) { From 4e9709b2efc489979b1fc16f74c3ee023d2b2d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Thu, 10 Sep 2020 01:01:33 +0200 Subject: [PATCH 04/16] #123 make CI tests output more verbose --- Dockerfile-ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-ubuntu b/Dockerfile-ubuntu index 8ef3a45d..d4029df7 100644 --- a/Dockerfile-ubuntu +++ b/Dockerfile-ubuntu @@ -73,7 +73,7 @@ RUN \ cmake -GNinja $kokkos_cmake_str -Dgtest_DIR=$GTEST_BUILD/install -DGTEST_ROOT=$GTEST_BUILD/install -DCHECKPOINT_BUILD_TESTS:BOOL=1 -DCMAKE_INSTALL_PREFIX=$CHECKPOINT_BUILD/install -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_C_COMPILER=$CC -Ddetector_DIR=$DETECTOR_BUILD/install $CHECKPOINT && \ ninja && \ ninja install && \ - ninja test + ctest --output-on-failure COPY $DETECTOR_BUILD/ $DETECTOR_BUILD COPY $CHECKPOINT_BUILD/ $CHECKPOINT_BUILD From 7ed71f7935bc83f3c607d4e6f1950a072aa39065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Thu, 10 Sep 2020 15:17:06 +0200 Subject: [PATCH 05/16] #123 implement memory footprinting for std::function --- src/checkpoint/checkpoint.h | 1 + src/checkpoint/container/function_serialize.h | 67 +++++++++++++++++++ tests/unit/test_footprinter.cc | 14 ++++ 3 files changed, 82 insertions(+) create mode 100644 src/checkpoint/container/function_serialize.h diff --git a/src/checkpoint/checkpoint.h b/src/checkpoint/checkpoint.h index 67e6440c..945bc390 100644 --- a/src/checkpoint/checkpoint.h +++ b/src/checkpoint/checkpoint.h @@ -51,6 +51,7 @@ #include "checkpoint/container/array_serialize.h" #include "checkpoint/container/enum_serialize.h" +#include "checkpoint/container/function_serialize.h" #include "checkpoint/container/list_serialize.h" #include "checkpoint/container/map_serialize.h" #include "checkpoint/container/raw_ptr_serialize.h" diff --git a/src/checkpoint/container/function_serialize.h b/src/checkpoint/container/function_serialize.h new file mode 100644 index 00000000..3848bfcd --- /dev/null +++ b/src/checkpoint/container/function_serialize.h @@ -0,0 +1,67 @@ +/* +//@HEADER +// ***************************************************************************** +// +// function_serialize.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_CONTAINER_FUNCTION_SERIALIZE_H +#define INCLUDED_CHECKPOINT_CONTAINER_FUNCTION_SERIALIZE_H + +#include "checkpoint/common.h" + +namespace checkpoint { + +/** + * \brief Serialize \c func + * + * + * \param[in] serializer serializer to use + * \param[in] func function to serialize + */ +template +void serialize(Serializer& s, std::function& fn) { + s.countBytes(fn); + //fn.target<...>(); +} + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_CONTAINER_FUNCTION_SERIALIZE_H*/ diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc index b3d5a075..a48a593b 100644 --- a/tests/unit/test_footprinter.cc +++ b/tests/unit/test_footprinter.cc @@ -160,6 +160,20 @@ TEST_F(TestFootprinter, test_enum) { } } +void fn() { + std::cout << 1; +} + +TEST_F(TestFootprinter, test_function) { + std::function f = fn; + auto ptr = f.target(); + EXPECT_NE(ptr, nullptr); + EXPECT_EQ( + checkpoint::getMemoryFootprint(f), + sizeof(f) + ); +} + TEST_F(TestFootprinter, test_list) { std::list l = {1, 2, 3}; EXPECT_EQ( From c44808ecfe2e2e3b130d6893a10f1c749f006216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Thu, 10 Sep 2020 21:54:57 +0200 Subject: [PATCH 06/16] #123 implement memory footprinting for std::queue and std::priority_queue --- src/checkpoint/checkpoint.h | 1 + src/checkpoint/container/queue_serialize.h | 79 ++++++++++++++++++++++ tests/unit/test_footprinter.cc | 26 +++++++ 3 files changed, 106 insertions(+) create mode 100644 src/checkpoint/container/queue_serialize.h diff --git a/src/checkpoint/checkpoint.h b/src/checkpoint/checkpoint.h index 945bc390..8bc4cde7 100644 --- a/src/checkpoint/checkpoint.h +++ b/src/checkpoint/checkpoint.h @@ -54,6 +54,7 @@ #include "checkpoint/container/function_serialize.h" #include "checkpoint/container/list_serialize.h" #include "checkpoint/container/map_serialize.h" +#include "checkpoint/container/queue_serialize.h" #include "checkpoint/container/raw_ptr_serialize.h" #include "checkpoint/container/string_serialize.h" #include "checkpoint/container/tuple_serialize.h" diff --git a/src/checkpoint/container/queue_serialize.h b/src/checkpoint/container/queue_serialize.h new file mode 100644 index 00000000..09b51d47 --- /dev/null +++ b/src/checkpoint/container/queue_serialize.h @@ -0,0 +1,79 @@ +/* +//@HEADER +// ***************************************************************************** +// +// queue_serialize.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_CONTAINER_QUEUE_SERIALIZE_H +#define INCLUDED_CHECKPOINT_CONTAINER_QUEUE_SERIALIZE_H + +#include + +#include "checkpoint/common.h" +#include "checkpoint/container/container_serialize.h" + +namespace checkpoint { + +template +void serialize(Serializer& s, std::queue q) { + s.countBytes(q); + + while (not q.empty()) + { + s | q.front(); + q.pop(); + } +} + +template +void serialize(Serializer& s, std::priority_queue q) { + s.countBytes(q); + + while (not q.empty()) + { + s | q.top(); + q.pop(); + } +} + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_CONTAINER_QUEUE_SERIALIZE_H*/ diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc index a48a593b..65d67966 100644 --- a/tests/unit/test_footprinter.cc +++ b/tests/unit/test_footprinter.cc @@ -220,6 +220,32 @@ TEST_F(TestFootprinter, test_map) { } } +TEST_F(TestFootprinter, test_queue) { + { + std::queue q; + q.push(1); + q.push(2); + q.push(3); + EXPECT_EQ( + checkpoint::getMemoryFootprint(q), + sizeof(q) + q.size() * sizeof(q.front()) + ); + EXPECT_EQ(q.size(), std::size_t{3}); + } + + { + std::priority_queue q; + q.push(1); + q.push(2); + q.push(3); + EXPECT_EQ( + checkpoint::getMemoryFootprint(q), + sizeof(q) + q.size() * sizeof(q.top()) + ); + EXPECT_EQ(q.size(), std::size_t{3}); + } +} + TEST_F(TestFootprinter, test_set) { { std::set s = {1, 2, 3}; From 03a4516f0eab532b25c8a9435fbda8465b7a2c73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Thu, 10 Sep 2020 22:33:10 +0200 Subject: [PATCH 07/16] #123 implement memory footprinting for std::chrono::duration --- src/checkpoint/checkpoint.h | 1 + src/checkpoint/container/chrono_serialize.h | 62 +++++++++++++++++++++ tests/unit/test_footprinter.cc | 8 +++ 3 files changed, 71 insertions(+) create mode 100644 src/checkpoint/container/chrono_serialize.h diff --git a/src/checkpoint/checkpoint.h b/src/checkpoint/checkpoint.h index 8bc4cde7..6708fd06 100644 --- a/src/checkpoint/checkpoint.h +++ b/src/checkpoint/checkpoint.h @@ -50,6 +50,7 @@ #include "checkpoint/traits/serializable_traits.h" #include "checkpoint/container/array_serialize.h" +#include "checkpoint/container/chrono_serialize.h" #include "checkpoint/container/enum_serialize.h" #include "checkpoint/container/function_serialize.h" #include "checkpoint/container/list_serialize.h" diff --git a/src/checkpoint/container/chrono_serialize.h b/src/checkpoint/container/chrono_serialize.h new file mode 100644 index 00000000..b1577ac5 --- /dev/null +++ b/src/checkpoint/container/chrono_serialize.h @@ -0,0 +1,62 @@ +/* +//@HEADER +// ***************************************************************************** +// +// chrono_serialize.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_CONTAINER_CHRONO_SERIALIZE_H +#define INCLUDED_CHECKPOINT_CONTAINER_CHRONO_SERIALIZE_H + +#include + +#include "checkpoint/common.h" + + +namespace checkpoint { + +template +void serialize(Serializer& s, std::chrono::duration d) { + s.countBytes(d); +} + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_CONTAINER_CHRONO_SERIALIZE_H*/ diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc index 65d67966..4fc90faa 100644 --- a/tests/unit/test_footprinter.cc +++ b/tests/unit/test_footprinter.cc @@ -126,6 +126,14 @@ TEST_F(TestFootprinter, test_array) { ); } +TEST_F(TestFootprinter, test_chrono_duration) { + std::chrono::seconds sec(1); + EXPECT_EQ( + checkpoint::getMemoryFootprint(sec), + sizeof(sec) + ); +} + TEST_F(TestFootprinter, test_deque) { std::deque d = {1, 2, 3}; EXPECT_EQ( From 3a066b3d544406b1914ec618a11de70ca7dc8f9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Fri, 11 Sep 2020 18:19:51 +0200 Subject: [PATCH 08/16] #123 implement memory footprinting for std::shared_ptr --- src/checkpoint/checkpoint.h | 1 + .../container/shared_ptr_serialize.h | 63 +++++++++++++++++++ tests/unit/test_footprinter.cc | 15 +++++ 3 files changed, 79 insertions(+) create mode 100644 src/checkpoint/container/shared_ptr_serialize.h diff --git a/src/checkpoint/checkpoint.h b/src/checkpoint/checkpoint.h index 6708fd06..d0826861 100644 --- a/src/checkpoint/checkpoint.h +++ b/src/checkpoint/checkpoint.h @@ -57,6 +57,7 @@ #include "checkpoint/container/map_serialize.h" #include "checkpoint/container/queue_serialize.h" #include "checkpoint/container/raw_ptr_serialize.h" +#include "checkpoint/container/shared_ptr_serialize.h" #include "checkpoint/container/string_serialize.h" #include "checkpoint/container/tuple_serialize.h" #include "checkpoint/container/vector_serialize.h" diff --git a/src/checkpoint/container/shared_ptr_serialize.h b/src/checkpoint/container/shared_ptr_serialize.h new file mode 100644 index 00000000..92e0b2de --- /dev/null +++ b/src/checkpoint/container/shared_ptr_serialize.h @@ -0,0 +1,63 @@ +/* +//@HEADER +// ***************************************************************************** +// +// shared_ptr_serialize.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_CONTAINER_SHARED_PTR_SERIALIZE_H +#define INCLUDED_CHECKPOINT_CONTAINER_SHARED_PTR_SERIALIZE_H + +#include "checkpoint/common.h" + +namespace checkpoint { + +template +void serialize(Serializer& s, std::shared_ptr& ptr) { + s.countBytes(ptr); + + if (ptr != nullptr) { + s | *ptr; + } +} + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_CONTAINER_SHARED_PTR_SERIALIZE_H*/ diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc index 4fc90faa..2bd60586 100644 --- a/tests/unit/test_footprinter.cc +++ b/tests/unit/test_footprinter.cc @@ -228,6 +228,21 @@ TEST_F(TestFootprinter, test_map) { } } +TEST_F(TestFootprinter, test_shared_ptr) { + { + std::shared_ptr ptr; + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr)); + } + + { + auto ptr = std::make_shared(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(ptr), + sizeof(ptr) + sizeof(*ptr) + ); + } +} + TEST_F(TestFootprinter, test_queue) { { std::queue q; From 77e766aa1197e9efe5a3aa7e5d48560a542ab2a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Fri, 11 Sep 2020 23:06:10 +0200 Subject: [PATCH 09/16] #123 implement memory footprinting for FILE --- src/checkpoint/container/raw_ptr_serialize.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/checkpoint/container/raw_ptr_serialize.h b/src/checkpoint/container/raw_ptr_serialize.h index 7e4418d5..85feed40 100644 --- a/src/checkpoint/container/raw_ptr_serialize.h +++ b/src/checkpoint/container/raw_ptr_serialize.h @@ -67,6 +67,15 @@ void serialize(Serializer& s, T* ptr) { } } +template +void serialize(Serializer& s, FILE* ptr) { + s.countBytes(ptr); + + if (ptr != nullptr) { + s.countBytes(*ptr); + } +} + } /* end namespace checkpoint */ #endif /*INCLUDED_CHECKPOINT_CONTAINER_RAW_PTR_SERIALIZE_H*/ From 290c11fde71ea3ef8bd6ccfd45df905997f5d8aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Fri, 11 Sep 2020 23:14:00 +0200 Subject: [PATCH 10/16] #123 compare calculated footprint with sizeof --- src/checkpoint/checkpoint_api.impl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/checkpoint/checkpoint_api.impl.h b/src/checkpoint/checkpoint_api.impl.h index 455ee735..22ad3063 100644 --- a/src/checkpoint/checkpoint_api.impl.h +++ b/src/checkpoint/checkpoint_api.impl.h @@ -93,6 +93,7 @@ std::size_t getSize(T& target) { template std::size_t getMemoryFootprint(T& target) { + // FIXME: use std::max to make sure that it's not smaller than sizeof(T) return dispatch::Standard::footprint(target); } From fac01c9e24fccf51e6fd2384321cb8665bc11680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Sat, 12 Sep 2020 01:32:40 +0200 Subject: [PATCH 11/16] #123 avoid double-counting footprint for std::array --- src/checkpoint/container/array_serialize.h | 1 - tests/unit/test_footprinter.cc | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/checkpoint/container/array_serialize.h b/src/checkpoint/container/array_serialize.h index 6f8ff3d2..b4de49a0 100644 --- a/src/checkpoint/container/array_serialize.h +++ b/src/checkpoint/container/array_serialize.h @@ -54,7 +54,6 @@ namespace checkpoint { template void serialize(Serializer& s, std::array& array) { - s.countBytes(array); dispatch::serializeArray(s, &array[0], array.size()); } diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc index 2bd60586..6f82ef33 100644 --- a/tests/unit/test_footprinter.cc +++ b/tests/unit/test_footprinter.cc @@ -122,7 +122,7 @@ TEST_F(TestFootprinter, test_array) { std::array a = {1, 2, 3}; EXPECT_EQ( checkpoint::getMemoryFootprint(a), - sizeof(a) + a.size() * sizeof(a[0]) + a.size() * sizeof(a[0]) ); } From 6462c2bc3ca4acd63105af802508723a92a8527d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Sat, 12 Sep 2020 01:54:49 +0200 Subject: [PATCH 12/16] #123 prohibit use in other modes whe only Footprinting mode is supported --- src/checkpoint/container/chrono_serialize.h | 8 ++- src/checkpoint/container/function_serialize.h | 25 +++++++-- src/checkpoint/container/queue_serialize.h | 35 +++++++++++-- src/checkpoint/container/raw_ptr_serialize.h | 51 ++++++++++++++++--- .../container/shared_ptr_serialize.h | 19 +++++-- src/checkpoint/container/string_serialize.h | 11 +++- 6 files changed, 126 insertions(+), 23 deletions(-) diff --git a/src/checkpoint/container/chrono_serialize.h b/src/checkpoint/container/chrono_serialize.h index b1577ac5..7cbe9df3 100644 --- a/src/checkpoint/container/chrono_serialize.h +++ b/src/checkpoint/container/chrono_serialize.h @@ -45,15 +45,19 @@ #if !defined INCLUDED_CHECKPOINT_CONTAINER_CHRONO_SERIALIZE_H #define INCLUDED_CHECKPOINT_CONTAINER_CHRONO_SERIALIZE_H -#include - #include "checkpoint/common.h" +#include namespace checkpoint { template void serialize(Serializer& s, std::chrono::duration d) { + checkpointAssert( + s.isFootprinting(), + "Only memory footprinting is supported for this type" + ); + s.countBytes(d); } diff --git a/src/checkpoint/container/function_serialize.h b/src/checkpoint/container/function_serialize.h index 3848bfcd..1d55b19e 100644 --- a/src/checkpoint/container/function_serialize.h +++ b/src/checkpoint/container/function_serialize.h @@ -47,19 +47,36 @@ #include "checkpoint/common.h" +#include + namespace checkpoint { /** - * \brief Serialize \c func + * \brief Serialize function \c func * + * Only footprinting mode is supported at the moment. * * \param[in] serializer serializer to use * \param[in] func function to serialize */ -template -void serialize(Serializer& s, std::function& fn) { +template +void serialize(SerializerT& s, std::function& fn) { + serializeFunction(s, fn); +} + +template < + typename SerializerT, + typename Res, + typename... ArgTypes, + typename = std::enable_if_t< + std::is_same< + SerializerT, + checkpoint::Footprinter + >::value + > +> +void serializeFunction(SerializerT& s, std::function& fn) { s.countBytes(fn); - //fn.target<...>(); } } /* end namespace checkpoint */ diff --git a/src/checkpoint/container/queue_serialize.h b/src/checkpoint/container/queue_serialize.h index 09b51d47..c47b2988 100644 --- a/src/checkpoint/container/queue_serialize.h +++ b/src/checkpoint/container/queue_serialize.h @@ -45,17 +45,29 @@ #if !defined INCLUDED_CHECKPOINT_CONTAINER_QUEUE_SERIALIZE_H #define INCLUDED_CHECKPOINT_CONTAINER_QUEUE_SERIALIZE_H -#include - #include "checkpoint/common.h" -#include "checkpoint/container/container_serialize.h" + +#include namespace checkpoint { template void serialize(Serializer& s, std::queue q) { - s.countBytes(q); + serializeQueue(s, q); +} +template < + typename SerializerT, + typename T, + typename = std::enable_if_t< + std::is_same< + SerializerT, + checkpoint::Footprinter + >::value + > +> +void serializeQueue(SerializerT& s, std::queue q) { + s.countBytes(q); while (not q.empty()) { s | q.front(); @@ -65,8 +77,21 @@ void serialize(Serializer& s, std::queue q) { template void serialize(Serializer& s, std::priority_queue q) { - s.countBytes(q); + serializeQueue(s, q); +} +template < + typename SerializerT, + typename T, + typename = std::enable_if_t< + std::is_same< + SerializerT, + checkpoint::Footprinter + >::value + > +> +void serializeQueue(SerializerT& s, std::priority_queue q) { + s.countBytes(q); while (not q.empty()) { s | q.top(); diff --git a/src/checkpoint/container/raw_ptr_serialize.h b/src/checkpoint/container/raw_ptr_serialize.h index 85feed40..c530eb2c 100644 --- a/src/checkpoint/container/raw_ptr_serialize.h +++ b/src/checkpoint/container/raw_ptr_serialize.h @@ -52,25 +52,60 @@ namespace checkpoint { /** * \brief Serialize raw pointer \c ptr * - * For footprinting mode, count the pointer size and follow it (note that it - * doesn't work correctly for C-style arrays!). + * Only footprinting mode is supported at the moment. Counts the pointer size + * and follows it (note that it doesn't work correctly for C-style arrays!). * * \param[in] serializer serializer to use * \param[in] ptr pointer to serialize */ -template -void serialize(Serializer& s, T* ptr) { - s.countBytes(ptr); +template +void serialize(SerializerT& s, T* ptr) { + serializeRawPtr(s, ptr); +} +template < + typename SerializerT, + typename T, + typename = std::enable_if_t< + std::is_same< + SerializerT, + checkpoint::Footprinter + >::value + > +> +void serializeRawPtr(SerializerT& s, T* ptr) { + s.countBytes(ptr); if (ptr != nullptr) { s | *ptr; } } -template -void serialize(Serializer& s, FILE* ptr) { - s.countBytes(ptr); +/** + * \brief Serialize FILE pointer \c ptr + * + * Only footprinting mode is supported at the moment. Counts the pointer and + * pointee (underlying struct) size without following it. + * + * \param[in] serializer serializer to use + * \param[in] ptr pointer to serialize + */ +template +void serialize(SerializerT& s, FILE* ptr) { + serializeFilePtr(s, ptr); +} + +template < + typename SerializerT, + typename = std::enable_if_t< + std::is_same< + SerializerT, + checkpoint::Footprinter + >::value + > +> +void serializeFilePtr(SerializerT& s, FILE* ptr) { + s.countBytes(ptr); if (ptr != nullptr) { s.countBytes(*ptr); } diff --git a/src/checkpoint/container/shared_ptr_serialize.h b/src/checkpoint/container/shared_ptr_serialize.h index 92e0b2de..e9327455 100644 --- a/src/checkpoint/container/shared_ptr_serialize.h +++ b/src/checkpoint/container/shared_ptr_serialize.h @@ -49,10 +49,23 @@ namespace checkpoint { -template -void serialize(Serializer& s, std::shared_ptr& ptr) { - s.countBytes(ptr); +template +void serialize(SerializerT& s, std::shared_ptr& ptr) { + serializeSharedPtr(s, ptr); +} +template < + typename SerializerT, + typename T, + typename = std::enable_if_t< + std::is_same< + SerializerT, + checkpoint::Footprinter + >::value + > +> +void serializeSharedPtr(SerializerT& s, std::shared_ptr& ptr) { + s.countBytes(ptr); if (ptr != nullptr) { s | *ptr; } diff --git a/src/checkpoint/container/string_serialize.h b/src/checkpoint/container/string_serialize.h index 64c47e26..07c4beaf 100644 --- a/src/checkpoint/container/string_serialize.h +++ b/src/checkpoint/container/string_serialize.h @@ -46,7 +46,6 @@ #define INCLUDED_CHECKPOINT_CONTAINER_STRING_SERIALIZE_H #include "checkpoint/common.h" -#include "checkpoint/serializers/serializers_headers.h" #include @@ -59,6 +58,16 @@ void serializeStringMeta(Serializer& s, std::string& str) { str.resize(str_size); } +/** + * \brief Serialize string \c str + * + * Resizes a string to it's actual size and serializes it. + * Note: footprinting mode does not detect small string optimization, so + * a limited overcount is possible. + * + * \param[in] serializer serializer to use + * \param[in] str string to serialize + */ template void serialize(Serializer& s, std::string& str) { if (s.isFootprinting()) { From 4ac59102a9f66fbd987ce0408a23cbc9ebba8997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Tue, 15 Sep 2020 13:16:31 +0200 Subject: [PATCH 13/16] #123 complete serialization support for std::chrono::duration --- src/checkpoint/container/chrono_serialize.h | 11 ++++------- tests/unit/test_footprinter.cc | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/checkpoint/container/chrono_serialize.h b/src/checkpoint/container/chrono_serialize.h index 7cbe9df3..a71dd3f8 100644 --- a/src/checkpoint/container/chrono_serialize.h +++ b/src/checkpoint/container/chrono_serialize.h @@ -52,13 +52,10 @@ namespace checkpoint { template -void serialize(Serializer& s, std::chrono::duration d) { - checkpointAssert( - s.isFootprinting(), - "Only memory footprinting is supported for this type" - ); - - s.countBytes(d); +void serialize(Serializer& s, std::chrono::duration& d) { + Rep r = d.count(); + s | r; + d = std::chrono::duration(r); } } /* end namespace checkpoint */ diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc index 6f82ef33..4f0823fe 100644 --- a/tests/unit/test_footprinter.cc +++ b/tests/unit/test_footprinter.cc @@ -132,6 +132,22 @@ TEST_F(TestFootprinter, test_chrono_duration) { checkpoint::getMemoryFootprint(sec), sizeof(sec) ); + + { + std::chrono::milliseconds milisec{300}; + auto ms_ret = checkpoint::serialize(milisec); + auto deserialized = + checkpoint::deserialize(ms_ret->getBuffer()); + EXPECT_EQ(milisec, *deserialized); + } + + { + std::chrono::hours hrs{24}; + auto hrs_ret = checkpoint::serialize(hrs); + auto deserialized = + checkpoint::deserialize(hrs_ret->getBuffer()); + EXPECT_EQ(hrs, *deserialized); + } } TEST_F(TestFootprinter, test_deque) { From 5c16a235dc173a1502be7f8f98ce6ab24f116586 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Wed, 16 Sep 2020 00:51:56 +0200 Subject: [PATCH 14/16] #123 fix code review issues - do not size underlying 'struct FILE' when footprinting FILE* pointer - simplify queue serialization to avoid copying - fix grammar typo --- src/checkpoint/container/queue_serialize.h | 33 +++----------------- src/checkpoint/container/raw_ptr_serialize.h | 8 ++--- src/checkpoint/container/string_serialize.h | 2 +- 3 files changed, 8 insertions(+), 35 deletions(-) diff --git a/src/checkpoint/container/queue_serialize.h b/src/checkpoint/container/queue_serialize.h index c47b2988..a81b0426 100644 --- a/src/checkpoint/container/queue_serialize.h +++ b/src/checkpoint/container/queue_serialize.h @@ -52,37 +52,18 @@ namespace checkpoint { template -void serialize(Serializer& s, std::queue q) { +void serialize(Serializer& s, const std::queue& q) { serializeQueue(s, q); } -template < - typename SerializerT, - typename T, - typename = std::enable_if_t< - std::is_same< - SerializerT, - checkpoint::Footprinter - >::value - > -> -void serializeQueue(SerializerT& s, std::queue q) { - s.countBytes(q); - while (not q.empty()) - { - s | q.front(); - q.pop(); - } -} - template -void serialize(Serializer& s, std::priority_queue q) { +void serialize(Serializer& s, const std::priority_queue& q) { serializeQueue(s, q); } template < typename SerializerT, - typename T, + typename Q, typename = std::enable_if_t< std::is_same< SerializerT, @@ -90,13 +71,9 @@ template < >::value > > -void serializeQueue(SerializerT& s, std::priority_queue q) { +void serializeQueue(SerializerT& s, const Q& q) { s.countBytes(q); - while (not q.empty()) - { - s | q.top(); - q.pop(); - } + s.contiguousBytes(nullptr, sizeof(typename Q::value_type), q.size()); } } /* end namespace checkpoint */ diff --git a/src/checkpoint/container/raw_ptr_serialize.h b/src/checkpoint/container/raw_ptr_serialize.h index c530eb2c..a1dc3849 100644 --- a/src/checkpoint/container/raw_ptr_serialize.h +++ b/src/checkpoint/container/raw_ptr_serialize.h @@ -83,13 +83,12 @@ void serializeRawPtr(SerializerT& s, T* ptr) { /** * \brief Serialize FILE pointer \c ptr * - * Only footprinting mode is supported at the moment. Counts the pointer and - * pointee (underlying struct) size without following it. + * Only footprinting mode is supported at the moment. Counts the pointer size + * without following it. * * \param[in] serializer serializer to use * \param[in] ptr pointer to serialize */ - template void serialize(SerializerT& s, FILE* ptr) { serializeFilePtr(s, ptr); @@ -106,9 +105,6 @@ template < > void serializeFilePtr(SerializerT& s, FILE* ptr) { s.countBytes(ptr); - if (ptr != nullptr) { - s.countBytes(*ptr); - } } } /* end namespace checkpoint */ diff --git a/src/checkpoint/container/string_serialize.h b/src/checkpoint/container/string_serialize.h index 07c4beaf..801d0827 100644 --- a/src/checkpoint/container/string_serialize.h +++ b/src/checkpoint/container/string_serialize.h @@ -61,7 +61,7 @@ void serializeStringMeta(Serializer& s, std::string& str) { /** * \brief Serialize string \c str * - * Resizes a string to it's actual size and serializes it. + * Resizes a string to its actual size and serializes it. * Note: footprinting mode does not detect small string optimization, so * a limited overcount is possible. * From 2fa68ed1a035f239f2a2e65d15e0d50ef979fe7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Wed, 16 Sep 2020 17:13:58 +0200 Subject: [PATCH 15/16] #123 make sure that memory footprint is not smaller than 'sizeof' --- src/checkpoint/checkpoint_api.impl.h | 6 ++++-- tests/unit/test_footprinter.cc | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/checkpoint/checkpoint_api.impl.h b/src/checkpoint/checkpoint_api.impl.h index 22ad3063..ee439521 100644 --- a/src/checkpoint/checkpoint_api.impl.h +++ b/src/checkpoint/checkpoint_api.impl.h @@ -93,8 +93,10 @@ std::size_t getSize(T& target) { template std::size_t getMemoryFootprint(T& target) { - // FIXME: use std::max to make sure that it's not smaller than sizeof(T) - return dispatch::Standard::footprint(target); + return std::max( + dispatch::Standard::footprint(target), + sizeof(target) + ); } template diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc index 4f0823fe..4e8de4be 100644 --- a/tests/unit/test_footprinter.cc +++ b/tests/unit/test_footprinter.cc @@ -112,7 +112,7 @@ TEST_F(TestFootprinter, test_fundamental_types) { int arr[] = {1, 2, 3, 4, 5}; EXPECT_EQ( checkpoint::getMemoryFootprint(arr), - sizeof(&arr) + sizeof(arr[0]) + sizeof(arr) // actually: sizeof(&arr) + sizeof(arr) ); } From 59665e578a52a3b6bce1883466edd3683df4e9e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cezary=20Skrzy=C5=84ski?= Date: Wed, 16 Sep 2020 21:48:23 +0200 Subject: [PATCH 16/16] #123 implement memory footprinting for std::vector --- src/checkpoint/container/vector_serialize.h | 5 +++++ tests/unit/test_footprinter.cc | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/src/checkpoint/container/vector_serialize.h b/src/checkpoint/container/vector_serialize.h index f9d87c4a..68ff803f 100644 --- a/src/checkpoint/container/vector_serialize.h +++ b/src/checkpoint/container/vector_serialize.h @@ -72,6 +72,11 @@ void serialize(Serializer& s, std::vector& vec) { template void serialize(Serializer& s, std::vector& vec) { + if (s.isFootprinting()) { + s.countBytes(vec); + return; + } + serializeVectorMeta(s, vec); if (!s.isUnpacking()) { diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc index 4e8de4be..4e5a7ced 100644 --- a/tests/unit/test_footprinter.cc +++ b/tests/unit/test_footprinter.cc @@ -379,6 +379,14 @@ TEST_F(TestFootprinter, test_vector) { ); delete v[0]; } + + { + std::vector v = { false, true, false, true, true }; + EXPECT_EQ( + checkpoint::getMemoryFootprint(v), + sizeof(v) + ); + } } }}} // end namespace checkpoint::tests::unit