diff --git a/.gitignore b/.gitignore index 7798ee77..a7d20e73 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ build/ +CMakeFiles/ + .dir-locals.el + +*.a +*.cmake +*.tcl diff --git a/Dockerfile-ubuntu b/Dockerfile-ubuntu index 8ef3a45d..d4029df7 100644 --- a/Dockerfile-ubuntu +++ b/Dockerfile-ubuntu @@ -73,7 +73,7 @@ RUN \ cmake -GNinja $kokkos_cmake_str -Dgtest_DIR=$GTEST_BUILD/install -DGTEST_ROOT=$GTEST_BUILD/install -DCHECKPOINT_BUILD_TESTS:BOOL=1 -DCMAKE_INSTALL_PREFIX=$CHECKPOINT_BUILD/install -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_C_COMPILER=$CC -Ddetector_DIR=$DETECTOR_BUILD/install $CHECKPOINT && \ ninja && \ ninja install && \ - ninja test + ctest --output-on-failure COPY $DETECTOR_BUILD/ $DETECTOR_BUILD COPY $CHECKPOINT_BUILD/ $CHECKPOINT_BUILD diff --git a/README.md b/README.md index e69de29b..90a84867 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,3 @@ +# *checkpoint* => serialization and checkpointing library + +![Docker Image CI](https://github.com/DARMA-tasking/checkpoint/workflows/Docker%20Image%20CI/badge.svg) diff --git a/src/checkpoint/checkpoint.h b/src/checkpoint/checkpoint.h index a08f5593..d0826861 100644 --- a/src/checkpoint/checkpoint.h +++ b/src/checkpoint/checkpoint.h @@ -50,9 +50,14 @@ #include "checkpoint/traits/serializable_traits.h" #include "checkpoint/container/array_serialize.h" +#include "checkpoint/container/chrono_serialize.h" #include "checkpoint/container/enum_serialize.h" +#include "checkpoint/container/function_serialize.h" #include "checkpoint/container/list_serialize.h" #include "checkpoint/container/map_serialize.h" +#include "checkpoint/container/queue_serialize.h" +#include "checkpoint/container/raw_ptr_serialize.h" +#include "checkpoint/container/shared_ptr_serialize.h" #include "checkpoint/container/string_serialize.h" #include "checkpoint/container/tuple_serialize.h" #include "checkpoint/container/vector_serialize.h" diff --git a/src/checkpoint/checkpoint_api.h b/src/checkpoint/checkpoint_api.h index 1f279f35..52b36fca 100644 --- a/src/checkpoint/checkpoint_api.h +++ b/src/checkpoint/checkpoint_api.h @@ -157,6 +157,16 @@ std::unique_ptr deserialize(SerializedReturnType&& in); template std::size_t getSize(T& target); +/** + * \brief Get memory footprint of \c target + * + * \param[in] target reference to \c T to measure footprint + * + * \return memory footprint of the \c target + */ +template +std::size_t getMemoryFootprint(T& target); + /** * \brief Serialize \c T to file with filename \c file * diff --git a/src/checkpoint/checkpoint_api.impl.h b/src/checkpoint/checkpoint_api.impl.h index 5e3c16a4..ee439521 100644 --- a/src/checkpoint/checkpoint_api.impl.h +++ b/src/checkpoint/checkpoint_api.impl.h @@ -91,6 +91,14 @@ std::size_t getSize(T& target) { return dispatch::Standard::size(target); } +template +std::size_t getMemoryFootprint(T& target) { + return std::max( + dispatch::Standard::footprint(target), + sizeof(target) + ); +} + template void serializeToFile(T& target, std::string const& file) { auto len = getSize(target); diff --git a/src/checkpoint/container/chrono_serialize.h b/src/checkpoint/container/chrono_serialize.h new file mode 100644 index 00000000..a71dd3f8 --- /dev/null +++ b/src/checkpoint/container/chrono_serialize.h @@ -0,0 +1,63 @@ +/* +//@HEADER +// ***************************************************************************** +// +// chrono_serialize.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_CONTAINER_CHRONO_SERIALIZE_H +#define INCLUDED_CHECKPOINT_CONTAINER_CHRONO_SERIALIZE_H + +#include "checkpoint/common.h" + +#include + +namespace checkpoint { + +template +void serialize(Serializer& s, std::chrono::duration& d) { + Rep r = d.count(); + s | r; + d = std::chrono::duration(r); +} + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_CONTAINER_CHRONO_SERIALIZE_H*/ diff --git a/src/checkpoint/container/container_serialize.h b/src/checkpoint/container/container_serialize.h index 8d81694b..db765f32 100644 --- a/src/checkpoint/container/container_serialize.h +++ b/src/checkpoint/container/container_serialize.h @@ -54,7 +54,11 @@ template inline typename ContainerT::size_type serializeContainerSize(Serializer& s, ContainerT& cont) { typename ContainerT::size_type cont_size = cont.size(); - s | cont_size; + if (s.isFootprinting()) { + s.countBytes(cont); + } else { + s | cont_size; + } return cont_size; } diff --git a/src/checkpoint/container/function_serialize.h b/src/checkpoint/container/function_serialize.h new file mode 100644 index 00000000..1d55b19e --- /dev/null +++ b/src/checkpoint/container/function_serialize.h @@ -0,0 +1,84 @@ +/* +//@HEADER +// ***************************************************************************** +// +// function_serialize.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_CONTAINER_FUNCTION_SERIALIZE_H +#define INCLUDED_CHECKPOINT_CONTAINER_FUNCTION_SERIALIZE_H + +#include "checkpoint/common.h" + +#include + +namespace checkpoint { + +/** + * \brief Serialize function \c func + * + * Only footprinting mode is supported at the moment. + * + * \param[in] serializer serializer to use + * \param[in] func function to serialize + */ +template +void serialize(SerializerT& s, std::function& fn) { + serializeFunction(s, fn); +} + +template < + typename SerializerT, + typename Res, + typename... ArgTypes, + typename = std::enable_if_t< + std::is_same< + SerializerT, + checkpoint::Footprinter + >::value + > +> +void serializeFunction(SerializerT& s, std::function& fn) { + s.countBytes(fn); +} + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_CONTAINER_FUNCTION_SERIALIZE_H*/ diff --git a/src/checkpoint/container/queue_serialize.h b/src/checkpoint/container/queue_serialize.h new file mode 100644 index 00000000..a81b0426 --- /dev/null +++ b/src/checkpoint/container/queue_serialize.h @@ -0,0 +1,81 @@ +/* +//@HEADER +// ***************************************************************************** +// +// queue_serialize.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_CONTAINER_QUEUE_SERIALIZE_H +#define INCLUDED_CHECKPOINT_CONTAINER_QUEUE_SERIALIZE_H + +#include "checkpoint/common.h" + +#include + +namespace checkpoint { + +template +void serialize(Serializer& s, const std::queue& q) { + serializeQueue(s, q); +} + +template +void serialize(Serializer& s, const std::priority_queue& q) { + serializeQueue(s, q); +} + +template < + typename SerializerT, + typename Q, + typename = std::enable_if_t< + std::is_same< + SerializerT, + checkpoint::Footprinter + >::value + > +> +void serializeQueue(SerializerT& s, const Q& q) { + s.countBytes(q); + s.contiguousBytes(nullptr, sizeof(typename Q::value_type), q.size()); +} + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_CONTAINER_QUEUE_SERIALIZE_H*/ diff --git a/src/checkpoint/container/raw_ptr_serialize.h b/src/checkpoint/container/raw_ptr_serialize.h new file mode 100644 index 00000000..a1dc3849 --- /dev/null +++ b/src/checkpoint/container/raw_ptr_serialize.h @@ -0,0 +1,112 @@ +/* +//@HEADER +// ***************************************************************************** +// +// raw_ptr_serialize.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_CONTAINER_RAW_PTR_SERIALIZE_H +#define INCLUDED_CHECKPOINT_CONTAINER_RAW_PTR_SERIALIZE_H + +#include "checkpoint/common.h" + +namespace checkpoint { + +/** + * \brief Serialize raw pointer \c ptr + * + * Only footprinting mode is supported at the moment. Counts the pointer size + * and follows it (note that it doesn't work correctly for C-style arrays!). + * + * \param[in] serializer serializer to use + * \param[in] ptr pointer to serialize + */ +template +void serialize(SerializerT& s, T* ptr) { + serializeRawPtr(s, ptr); +} + +template < + typename SerializerT, + typename T, + typename = std::enable_if_t< + std::is_same< + SerializerT, + checkpoint::Footprinter + >::value + > +> +void serializeRawPtr(SerializerT& s, T* ptr) { + s.countBytes(ptr); + if (ptr != nullptr) { + s | *ptr; + } +} + +/** + * \brief Serialize FILE pointer \c ptr + * + * Only footprinting mode is supported at the moment. Counts the pointer size + * without following it. + * + * \param[in] serializer serializer to use + * \param[in] ptr pointer to serialize + */ +template +void serialize(SerializerT& s, FILE* ptr) { + serializeFilePtr(s, ptr); +} + +template < + typename SerializerT, + typename = std::enable_if_t< + std::is_same< + SerializerT, + checkpoint::Footprinter + >::value + > +> +void serializeFilePtr(SerializerT& s, FILE* ptr) { + s.countBytes(ptr); +} + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_CONTAINER_RAW_PTR_SERIALIZE_H*/ diff --git a/src/checkpoint/container/shared_ptr_serialize.h b/src/checkpoint/container/shared_ptr_serialize.h new file mode 100644 index 00000000..e9327455 --- /dev/null +++ b/src/checkpoint/container/shared_ptr_serialize.h @@ -0,0 +1,76 @@ +/* +//@HEADER +// ***************************************************************************** +// +// shared_ptr_serialize.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_CONTAINER_SHARED_PTR_SERIALIZE_H +#define INCLUDED_CHECKPOINT_CONTAINER_SHARED_PTR_SERIALIZE_H + +#include "checkpoint/common.h" + +namespace checkpoint { + +template +void serialize(SerializerT& s, std::shared_ptr& ptr) { + serializeSharedPtr(s, ptr); +} + +template < + typename SerializerT, + typename T, + typename = std::enable_if_t< + std::is_same< + SerializerT, + checkpoint::Footprinter + >::value + > +> +void serializeSharedPtr(SerializerT& s, std::shared_ptr& ptr) { + s.countBytes(ptr); + if (ptr != nullptr) { + s | *ptr; + } +} + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_CONTAINER_SHARED_PTR_SERIALIZE_H*/ diff --git a/src/checkpoint/container/string_serialize.h b/src/checkpoint/container/string_serialize.h index e333a0ce..801d0827 100644 --- a/src/checkpoint/container/string_serialize.h +++ b/src/checkpoint/container/string_serialize.h @@ -46,7 +46,6 @@ #define INCLUDED_CHECKPOINT_CONTAINER_STRING_SERIALIZE_H #include "checkpoint/common.h" -#include "checkpoint/serializers/serializers_headers.h" #include @@ -59,10 +58,25 @@ void serializeStringMeta(Serializer& s, std::string& str) { str.resize(str_size); } +/** + * \brief Serialize string \c str + * + * Resizes a string to its actual size and serializes it. + * Note: footprinting mode does not detect small string optimization, so + * a limited overcount is possible. + * + * \param[in] serializer serializer to use + * \param[in] str string to serialize + */ template void serialize(Serializer& s, std::string& str) { - serializeStringMeta(s, str); - dispatch::serializeArray(s, str.c_str(), str.size()); + if (s.isFootprinting()) { + s.countBytes(str); + dispatch::serializeArray(s, str.c_str(), str.capacity()); + } else { + serializeStringMeta(s, str); + dispatch::serializeArray(s, str.c_str(), str.size()); + } } } /* end namespace checkpoint */ diff --git a/src/checkpoint/container/tuple_serialize.h b/src/checkpoint/container/tuple_serialize.h index 1bb55721..f34c260e 100644 --- a/src/checkpoint/container/tuple_serialize.h +++ b/src/checkpoint/container/tuple_serialize.h @@ -67,6 +67,7 @@ void serialize(Serializer& s, std::tuple& tuple) { template void serialize(Serializer& s, std::pair& pair) { + s.countBytes(pair); s | pair.first; s | pair.second; } diff --git a/src/checkpoint/container/unique_ptr_serialize.h b/src/checkpoint/container/unique_ptr_serialize.h index 3d1953e8..12c1d897 100644 --- a/src/checkpoint/container/unique_ptr_serialize.h +++ b/src/checkpoint/container/unique_ptr_serialize.h @@ -54,7 +54,11 @@ namespace checkpoint { template void serialize(Serializer& s, std::unique_ptr& ptr) { bool is_null = ptr == nullptr; - s | is_null; + if (s.isFootprinting()) { + s.countBytes(ptr); + } else { + s | is_null; + } if (not is_null) { T* t = ptr.get(); diff --git a/src/checkpoint/container/vector_serialize.h b/src/checkpoint/container/vector_serialize.h index b3c9f81b..68ff803f 100644 --- a/src/checkpoint/container/vector_serialize.h +++ b/src/checkpoint/container/vector_serialize.h @@ -61,12 +61,22 @@ void serializeVectorMeta(Serializer& s, std::vector& vec) { template void serialize(Serializer& s, std::vector& vec) { - serializeVectorMeta(s, vec); - dispatch::serializeArray(s, &vec[0], vec.size()); + if (s.isFootprinting()) { + s.countBytes(vec); + dispatch::serializeArray(s, &vec[0], vec.capacity()); + } else { + serializeVectorMeta(s, vec); + dispatch::serializeArray(s, &vec[0], vec.size()); + } } template void serialize(Serializer& s, std::vector& vec) { + if (s.isFootprinting()) { + s.countBytes(vec); + return; + } + serializeVectorMeta(s, vec); if (!s.isUnpacking()) { diff --git a/src/checkpoint/dispatch/dispatch.h b/src/checkpoint/dispatch/dispatch.h index c5f1e886..297e0198 100644 --- a/src/checkpoint/dispatch/dispatch.h +++ b/src/checkpoint/dispatch/dispatch.h @@ -135,6 +135,17 @@ struct Standard { template static SerialSizeType size(T& target, Args&&... args); + /** + * \brief Recursively get the memory footprint of \c T + * + * \param[in] target the target to measure + * \param[in] args arguments to the footprinter's constructor + * + * \return memory footprint of \c T + */ + template + static SerialSizeType footprint(T& target, Args&&... args); + /** * \brief Pack \c target that requires \c size number of bytes. * diff --git a/src/checkpoint/dispatch/dispatch.impl.h b/src/checkpoint/dispatch/dispatch.impl.h index 61a645b5..8d40cb54 100644 --- a/src/checkpoint/dispatch/dispatch.impl.h +++ b/src/checkpoint/dispatch/dispatch.impl.h @@ -91,6 +91,14 @@ SerialSizeType Standard::size(T& target, Args&&... args) { return sizer.getSize(); } +template +SerialSizeType Standard::footprint(T& target, Args&&... args) { + auto footprinter = Traverse::with( + target, std::forward(args)... + ); + return footprinter.getMemoryFootprint(); +} + template PackerT Standard::pack(T& target, SerialSizeType const& size, Args&&... args) { return Traverse::with(target, size, std::forward(args)...); diff --git a/src/checkpoint/serializers/base_serializer.h b/src/checkpoint/serializers/base_serializer.h index 2ddaf831..d47f6610 100644 --- a/src/checkpoint/serializers/base_serializer.h +++ b/src/checkpoint/serializers/base_serializer.h @@ -57,6 +57,7 @@ enum struct eSerializationMode : int8_t { Unpacking = 1, Packing = 2, Sizing = 3, + Footprinting = 4, Invalid = -1 }; @@ -79,6 +80,10 @@ struct Serializer { bool isSizing() const { return cur_mode_ == ModeType::Sizing; } bool isPacking() const { return cur_mode_ == ModeType::Packing; } bool isUnpacking() const { return cur_mode_ == ModeType::Unpacking; } + bool isFootprinting() const { return cur_mode_ == ModeType::Footprinting; } + + template + void countBytes(const T& t) {} template void contiguousTyped(SerializerT& serdes, T* ptr, SerialSizeType num_elms) { diff --git a/src/checkpoint/serializers/footprinter.h b/src/checkpoint/serializers/footprinter.h new file mode 100644 index 00000000..1c301310 --- /dev/null +++ b/src/checkpoint/serializers/footprinter.h @@ -0,0 +1,74 @@ +/* +//@HEADER +// ***************************************************************************** +// +// footprinter.h +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_CHECKPOINT_SERIALIZERS_FOOTPRINTER_H +#define INCLUDED_CHECKPOINT_SERIALIZERS_FOOTPRINTER_H + +#include "checkpoint/common.h" +#include "checkpoint/serializers/base_serializer.h" + +namespace checkpoint { + +struct Footprinter : Serializer { + Footprinter() : Serializer(ModeType::Footprinting) { } + + SerialSizeType getMemoryFootprint() const { + return num_bytes_; + } + void contiguousBytes(void*, SerialSizeType size, SerialSizeType num_elms) { + num_bytes_ += size * num_elms; + } + + template + void countBytes(const T& t) { // this needs a better name + num_bytes_ += sizeof(t); + } + +private: + SerialSizeType num_bytes_ = 0; +}; + +} /* end namespace checkpoint */ + +#endif /*INCLUDED_CHECKPOINT_SERIALIZERS_FOOTPRINTER_H*/ diff --git a/src/checkpoint/serializers/serializers_headers.h b/src/checkpoint/serializers/serializers_headers.h index 92b09b7b..b08569f6 100644 --- a/src/checkpoint/serializers/serializers_headers.h +++ b/src/checkpoint/serializers/serializers_headers.h @@ -47,6 +47,7 @@ #include "checkpoint/common.h" #include "checkpoint/serializers/base_serializer.h" +#include "checkpoint/serializers/footprinter.h" #include "checkpoint/serializers/sizer.h" #include "checkpoint/serializers/packer.h" #include "checkpoint/serializers/unpacker.h" diff --git a/tests/unit/test_footprinter.cc b/tests/unit/test_footprinter.cc new file mode 100644 index 00000000..4e5a7ced --- /dev/null +++ b/tests/unit/test_footprinter.cc @@ -0,0 +1,392 @@ +/* +//@HEADER +// ***************************************************************************** +// +// test_footprinter.cc +// DARMA Toolkit v. 1.0.0 +// DARMA/checkpoint => Serialization Library +// +// Copyright 2019 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#include + +#include "test_harness.h" + +#include + +namespace checkpoint { namespace tests { namespace unit { + +struct TestFootprinter : TestHarness { }; + +struct Test1 { + double d; + + template + void serialize(Serializer& s) { + s | d; + } +}; + +struct Test2 { + float f; +}; + +template +void serialize(Serializer& s, Test2 t) { + s | t.f; +} + +TEST_F(TestFootprinter, test_fundamental_types) { + int i; + EXPECT_EQ(checkpoint::getMemoryFootprint(i), sizeof(i)); + + double d; + EXPECT_EQ(checkpoint::getMemoryFootprint(d), sizeof(d)); + + auto str = "12345"; + EXPECT_EQ( + checkpoint::getMemoryFootprint(str), + sizeof(str) + sizeof(*str) + // actually: sizeof(str) + 6 * sizeof(str[0]) + ); + + { + int* ptr = nullptr; + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr)); + } + + { + int* ptr = new int(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(ptr), + sizeof(ptr) + sizeof(*ptr) + ); + delete ptr; + } + + { + int *arr = new int[5]; + EXPECT_EQ( + checkpoint::getMemoryFootprint(arr), + sizeof(arr) + sizeof(*arr) + // actually: sizeof(arr) + 5 * sizeof(arr[0]) + ); + delete [] arr; + } + + { + int arr[] = {1, 2, 3, 4, 5}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(arr), + sizeof(arr) + // actually: sizeof(&arr) + sizeof(arr) + ); + } +} + +TEST_F(TestFootprinter, test_array) { + std::array a = {1, 2, 3}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(a), + a.size() * sizeof(a[0]) + ); +} + +TEST_F(TestFootprinter, test_chrono_duration) { + std::chrono::seconds sec(1); + EXPECT_EQ( + checkpoint::getMemoryFootprint(sec), + sizeof(sec) + ); + + { + std::chrono::milliseconds milisec{300}; + auto ms_ret = checkpoint::serialize(milisec); + auto deserialized = + checkpoint::deserialize(ms_ret->getBuffer()); + EXPECT_EQ(milisec, *deserialized); + } + + { + std::chrono::hours hrs{24}; + auto hrs_ret = checkpoint::serialize(hrs); + auto deserialized = + checkpoint::deserialize(hrs_ret->getBuffer()); + EXPECT_EQ(hrs, *deserialized); + } +} + +TEST_F(TestFootprinter, test_deque) { + std::deque d = {1, 2, 3}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(d), + sizeof(d) + d.size() * sizeof(d.front()) + ); +} + +TEST_F(TestFootprinter, test_enum) { + { + enum color + { + red, + green, + blue + }; + color col = red; + + EXPECT_EQ(checkpoint::getMemoryFootprint(col), sizeof(col)); + } + + { + enum struct color : char + { + red = 'r', + green = 'g', + blue = 'b' + }; + color col = color::green; + + EXPECT_EQ(checkpoint::getMemoryFootprint(col), sizeof(char)); + } +} + +void fn() { + std::cout << 1; +} + +TEST_F(TestFootprinter, test_function) { + std::function f = fn; + auto ptr = f.target(); + EXPECT_NE(ptr, nullptr); + EXPECT_EQ( + checkpoint::getMemoryFootprint(f), + sizeof(f) + ); +} + +TEST_F(TestFootprinter, test_list) { + std::list l = {1, 2, 3}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(l), + sizeof(l) + l.size() * sizeof(l.front()) + ); +} + +TEST_F(TestFootprinter, test_map) { + { + std::map m = {{1, 'a'}, {2, 'b'}, {3, 'c'}}; + auto p = *m.begin(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(m), + sizeof(m) + m.size() * (sizeof(p) + sizeof(p.first) + sizeof(p.second)) + ); + } + + { + std::unordered_map m = {{1, 'a'}, {2, 'b'}, {3, 'c'}}; + auto p = *m.begin(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(m), + sizeof(m) + m.size() * (sizeof(p) + sizeof(p.first) + sizeof(p.second)) + ); + } + + { + std::multimap m = {{1, 'a'}, {1, 'b'}, {1, 'c'}}; + auto p = *m.begin(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(m), + sizeof(m) + m.size() * (sizeof(p) + sizeof(p.first) + sizeof(p.second)) + ); + } + + { + std::unordered_multimap m = {{1, 'a'}, {1, 'b'}, {1, 'c'}}; + auto p = *m.begin(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(m), + sizeof(m) + m.size() * (sizeof(p) + sizeof(p.first) + sizeof(p.second)) + ); + } +} + +TEST_F(TestFootprinter, test_shared_ptr) { + { + std::shared_ptr ptr; + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr)); + } + + { + auto ptr = std::make_shared(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(ptr), + sizeof(ptr) + sizeof(*ptr) + ); + } +} + +TEST_F(TestFootprinter, test_queue) { + { + std::queue q; + q.push(1); + q.push(2); + q.push(3); + EXPECT_EQ( + checkpoint::getMemoryFootprint(q), + sizeof(q) + q.size() * sizeof(q.front()) + ); + EXPECT_EQ(q.size(), std::size_t{3}); + } + + { + std::priority_queue q; + q.push(1); + q.push(2); + q.push(3); + EXPECT_EQ( + checkpoint::getMemoryFootprint(q), + sizeof(q) + q.size() * sizeof(q.top()) + ); + EXPECT_EQ(q.size(), std::size_t{3}); + } +} + +TEST_F(TestFootprinter, test_set) { + { + std::set s = {1, 2, 3}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(s), + sizeof(s) + s.size() * sizeof(*s.begin()) + ); + } + + { + std::unordered_set s = {1, 2, 3}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(s), + sizeof(s) + s.size() * sizeof(*s.begin()) + ); + } + + { + std::multiset s = {1, 1, 1}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(s), + sizeof(s) + s.size() * sizeof(*s.begin()) + ); + } + + { + std::unordered_multiset s = {1, 1, 1}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(s), + sizeof(s) + s.size() * sizeof(*s.begin()) + ); + } +} + +// does not account for small string optimisation +TEST_F(TestFootprinter, test_string) { + std::string s = "123456789"; + EXPECT_EQ( + checkpoint::getMemoryFootprint(s), + sizeof(s) + s.capacity() * sizeof(s[0])); +} + +// naive approach, just sum memory usage of all elements +TEST_F(TestFootprinter, test_tuple) { + int i = 1; + double d = 3.8; + std::string s = "123456789"; + auto t = std::make_tuple(i, d, s); + EXPECT_EQ( + checkpoint::getMemoryFootprint(t), + sizeof(i) + sizeof(d) + sizeof(s) + s.capacity() * sizeof(s[0]) + ); +} + +TEST_F(TestFootprinter, test_unique_ptr) { + { + std::unique_ptr ptr; + EXPECT_EQ(checkpoint::getMemoryFootprint(ptr), sizeof(ptr)); + } + + { + auto ptr = std::make_unique(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(ptr), + sizeof(ptr) + sizeof(*ptr) + ); + } + + { + auto ptr = std::make_unique(); + EXPECT_EQ( + checkpoint::getMemoryFootprint(ptr), + sizeof(ptr) + sizeof(*ptr) + ); + } +} + +TEST_F(TestFootprinter, test_vector) { + { + std::vector v = {1, 2, 3, 4, 5}; + EXPECT_EQ( + checkpoint::getMemoryFootprint(v), + sizeof(v) + v.capacity() * sizeof(v[0]) + ); + } + + { + std::vector v = { new Test1(), nullptr }; + EXPECT_EQ( + checkpoint::getMemoryFootprint(v), + sizeof(v) + v.capacity() * sizeof(v[0]) + sizeof(*v[0]) + ); + delete v[0]; + } + + { + std::vector v = { false, true, false, true, true }; + EXPECT_EQ( + checkpoint::getMemoryFootprint(v), + sizeof(v) + ); + } +} + +}}} // end namespace checkpoint::tests::unit diff --git a/tests/unit/test_traversal.cc b/tests/unit/test_traversal.cc index a457c439..316a9e19 100644 --- a/tests/unit/test_traversal.cc +++ b/tests/unit/test_traversal.cc @@ -159,7 +159,7 @@ TEST_F(TestTraversal, test_traversal) { auto traverse = checkpoint::dispatch::Traverse::with(t); - EXPECT_EQ(traverse.bytes_, 10*4 + 10*8 + 4 + 4 + 100*4 + 8*3); + EXPECT_EQ(traverse.bytes_, std::size_t{10*4 + 10*8 + 4 + 4 + 100*4 + 8*3}); } TEST_F(TestTraversal, test_traversal_dispatcher) {