diff --git a/CMakeLists.txt b/CMakeLists.txt index c35d7c6cc..0931d424c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ project(podio) #--- Version ------------------------------------------------------------------- SET( ${PROJECT_NAME}_VERSION_MAJOR 1 ) SET( ${PROJECT_NAME}_VERSION_MINOR 1 ) -SET( ${PROJECT_NAME}_VERSION_PATCH 0 ) +SET( ${PROJECT_NAME}_VERSION_PATCH 99 ) SET( ${PROJECT_NAME}_VERSION "${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}.${${PROJECT_NAME}_VERSION_PATCH}" ) diff --git a/doc/links.md b/doc/links.md new file mode 100644 index 000000000..c7973ca38 --- /dev/null +++ b/doc/links.md @@ -0,0 +1,224 @@ +# Linking unrelated objects with each other +Sometimes it is necessary to build links between objects whose datatypes are +not related via a `OneToOneRelation` or a `OneToManyRelation`. These *external +relations* are called *Links* in podio, and they are implemented as a +templated version of the code that would be generated by the following yaml +snippet (in this case between generic `FromT` and `ToT` datatypes): + +```yaml +Link: + Description: "A weighted link between a FromT and a ToT" + Author: "P. O. Dio" + Members: + - float weight // the weight of the link + OneToOneRelations: + - FromT from // reference to the FromT + - ToT to // reference to the ToT +``` + +## `Link` basics +`Link`s are implemented as templated classes forming a similar structure +as other podio generated classes, with several layers of which users only ever +interact with the *User layer*. This layer has the following basic classes +```cpp +/// The collection class that forms the basis of I/O and also is the main entry point +template +class LinkCollection; + +/// The default (immutable) class that one gets after reading a collection +template +class Link; + +/// The mutable class for creating links before writing them +template +class MutableLink; +``` + +Although the names of the template parameters, `FromT` and `ToT` imply a +direction of the link, from a technical point of view nothing actually +enforces this direction, unless `FromT` and `ToT` are both of the same type. +Hence, links can effectively be treated as bi-directional, and one +combination of `FromT` and `ToT` should be enough for all use cases (see also +the [usage section](#how-to-use-links)). + +For a more detailed explanation of the internals and the actual implementation +see [the implementation details](#implementation-details). + +## How to use `Link`s +Using `Link`s is quite simple. In line with other datatypes that are +generated by podio all the functionality can be gained by including the +corresponding `Collection` header. After that it is generally recommended to +introduce a type alias for easier usage. **As a general rule `Links` need +to be declared with the default (immutable) types.** Trying to instantiate them +with `Mutable` types will result in a compilation error. + +```cpp +#include "podio/LinkCollection.h" + +#include "edm4hep/MCParticleCollection.h" +#include "edm4hep/ReconstructedParticleCollection.h" + +// declare a new link type +using MCRecoParticleLinkCollection = podio::LinkCollection; +``` + +This can now be used exactly as any other podio generated collection, i.e. +```cpp +edm4hep::MCParticle mcParticle{}; +edm4hep::ReconstructedParticle recoParticle{}; + +auto mcRecoLinks = MCRecoParticleLinkCollection{}; +auto link = mcRecoLinks.create(); // create an link; +link.setFrom(mcParticle); +link.setTo(recoParticle); +link.setWeight(1.0); // This is also the default value! +``` + +and similar for getting the linked objects +```cpp +auto mcP = link.getFrom(); +auto recoP = link.getTo(); +auto weight = link.getWeight(); +``` + +In the above examples the `From` and `To` in the method names imply a direction, +but it is also possible to use a templated `get` and `set` method to retrieve +the linked objects via their type: + +```cpp +link.set(mcParticle); +link.set(recoParticle); + +auto mcP = link.get(); +auto recoP = link.get(); +auto weight = link.getWeight(); +``` + +It is also possible to access the elements of a link via an index based +`get` (similar to `std::tuple`). In this case `0` corresponds to `getFrom`, `1` +corresponds to `getTo` and `2` corresponds to the weight. The main purpose of +this feature is to enable structured bindings: + +```cpp +const auto& [mcP, recoP, weight] = link; +``` + +The above three examples are three equivalent ways of retrieving the same things +from an `Link`. **The templated `get` and `set` methods are only available +if `FromT` and `ToT` are not the same type** and will lead to a compilation +error otherwise. + +### Enabling I/O capabilities for `Link`s + +`Link`s do not have I/O support out of the box. This has to be enabled via +the `PODIO_DECLARE_LINK` macro (defined in the `LinkCollection.h` +header). If you simply want to be able to read / write `Link`s in a +standalone executable, it is enough to use this macro somewhere in the +executable, e.g. to enable I/O capabilities for the `MCRecoParticleLink`s +used above this would look like: + +```cpp +PODIO_DECLARE_LINK(edm4hep::MCParticle, edm4hep::ReconstructedParticle) +``` + +The macro will also enable SIO support if the `PODIO_ENABLE_SIO=1` is passed to +the compiler. This is done by default when linking against the +`podio::podioSioIO` library in CMake. + +For enabling I/O support for shared datamodel libraries, it is necessary to have +all the necessary combinations of types declared via `PODIO_DECLARE_LINK` +and have that compiled into the library. This is necessary if you want to use +the python bindings, since they rely on dynamically loading the datamodel +libraries. + +## Implementation details + +In order to give a slightly easier entry to the details of the implementation +and also to make it easier to find where things in the generated documentation, +we give a brief description of the main ideas and design choices here. With +those it should be possible to dive deeper if necessary or to understand the +template structure that is visible in the documentation, but should be fairly +invisible in usage. We will focus mainly on the user facing classes, as those +deal with the most complexity, the underlying layers are more or less what could +be obtained by generating them via the yaml snippet above and sprinkling some +`` templates where necessary. + +### File structure + +The user facing `"podio/LinkCollection.h"` header essentially just +defines the `PODIO_DECLARE_LINK` macro (depending on whether SIO support +is desired and possible or not). All the actual implementation is done in the +following files: + +- [`"podio/detail/LinkCollectionImpl.h"`](https://github.com/AIDASoft/podio/blob/master/include/podio/detail/LinkCollectionImpl.h): + for the collection functionality +- [`"podio/detail/Link.h"`](https://github.com/AIDASoft/podio/blob/master/include/podio/detail/Link.h): + for the functionality of single link +- [`"podio/detail/LinkCollectionIterator.h"`](https://github.com/AIDASoft/podio/blob/master/include/podio/detail/LinkCollectionIterator.h): + for the collection iterator functionality +- [`"podio/detail/LinkObj.h"`](https://github.com/AIDASoft/podio/blob/master/include/podio/detail/LinkObj.h): + for the object layer functionality + - [`"podio/detail/LinkCollectionData.h"`](https://github.com/AIDASoft/podio/blob/master/include/podio/detail/LinkCollectionData.h): + for the collection data functionality +- [`"podio/detail/LinkFwd.h"`](https://github.com/AIDASoft/podio/blob/master/include/podio/detail/LinkFwd.h): + for some type helper functionality and some forward declarations that are used + throughout the other headers +- [`"podio/detail/LinkSIOBlock.h"`](https://github.com/AIDASoft/podio/blob/master/include/podio/detail/LinkSIOBlock.h): + for defining the SIOBlocks that are necessary to use SIO + +As is visible from this structure, we did not introduce an `LinkData` +class, since that would effectively just be a `float` wrapped inside a `struct`. + +### Default and `Mutable` `Link` classes + +A quick look into the `LinkFwd.h` header will reveal that the default and +`Mutable` `Link` classes are in fact just partial specialization of the +`LinkT` class that takes a `bool Mutable` as third template argument. The +same approach is also followed by the `LinkCollectionIterator`s: + +```cpp +template +class LinkT; + +template +using Link = LinkT; + +template +using MutableLink = LinkT; +``` + +Throughout the implementation it is assumed that `FromT` and `ToT` always are the +default handle types. This is ensured through `static_assert`s in the +`LinkCollection` to make sure it can only be instantiated with those. The +`GetDefaultHandleType` helper templates are used to retrieve the correct type +from any `FromT` regardless of whether it is a mutable or a default handle type +With this in mind, effectively all mutating operations on `Link`s are +defined using [*SFINAE*](https://en.cppreference.com/w/cpp/language/sfinae) +using the following template structure (taking here `setFrom` as an example) + +```cpp +template , FromT>>> +void setFrom(FromU value); +``` + +This is a SFINAE friendly way to ensure that this definition is only viable if +the following conditions are met +- The object this method is called on has to be `Mutable`. (first part inside the `std::enable_if`) +- The passed in `value` is either a `Mutable` or default class of type `FromT`. (second part inside the `std::enable_if`) + +In some cases the template signature looks like this + +```cpp +template> +void setWeight(float weight); +``` + +The reason to have a defaulted `bool` template parameter here is the same as the +one for having a `typename FromU` template parameter above: SFINAE only works +with deduced types. Using `Mut && Mutable` in the `std::enable_if` makes sure +that users cannot bypass the immutability by specifying a template parameter +themselves. diff --git a/include/podio/LinkCollection.h b/include/podio/LinkCollection.h new file mode 100644 index 000000000..ae2337d84 --- /dev/null +++ b/include/podio/LinkCollection.h @@ -0,0 +1,37 @@ +#ifndef PODIO_LINKCOLLECTION_H +#define PODIO_LINKCOLLECTION_H + +#include "podio/detail/LinkCollectionImpl.h" +#include "podio/detail/PreprocessorMacros.h" + +#ifndef PODIO_ENABLE_SIO + #define PODIO_ENABLE_SIO 0 +#endif + +/// Macro for registering links at the CollectionBufferFactory by injecting the +/// corresponding buffer creation function. +#define PODIO_REGISTER_LINK_BUFFERFACTORY(FromT, ToT) \ + const static auto PODIO_PP_CONCAT(REGISTERED_LINK_, __COUNTER__) = \ + podio::detail::registerLinkCollection(podio::detail::linkCollTypeName()); + +/// Macro for registering the necessary SIOBlock for a Link with the SIOBlock factory +#define PODIO_REGISTER_LINK_SIOFACTORY(FromT, ToT) \ + const static auto PODIO_PP_CONCAT(LINK_SIO_BLOCK_, __COUNTER__) = podio::LinkSIOBlock{}; + +#if PODIO_ENABLE_SIO && __has_include("podio/detail/LinkSIOBlock.h") + #include "podio/detail/LinkSIOBlock.h" + /// Main macro for declaring links. Takes care of the following things: + /// - Registering the necessary buffer creation functionality with the + /// CollectionBufferFactory. + /// - Registering the necessary SIOBlock with the SIOBlock factory + #define PODIO_DECLARE_LINK(FromT, ToT) \ + PODIO_REGISTER_LINK_BUFFERFACTORY(FromT, ToT) \ + PODIO_REGISTER_LINK_SIOFACTORY(FromT, ToT) +#else + /// Main macro for declaring links. Takes care of the following things: + /// - Registering the necessary buffer creation functionality with the + /// CollectionBufferFactory. + #define PODIO_DECLARE_LINK(FromT, ToT) PODIO_REGISTER_LINK_BUFFERFACTORY(FromT, ToT) +#endif + +#endif // PODIO_LINKCOLLECTION_H diff --git a/include/podio/detail/Link.h b/include/podio/detail/Link.h new file mode 100644 index 000000000..0cb50135e --- /dev/null +++ b/include/podio/detail/Link.h @@ -0,0 +1,357 @@ +#ifndef PODIO_DETAIL_LINK_H +#define PODIO_DETAIL_LINK_H + +#include "podio/detail/LinkFwd.h" +#include "podio/detail/LinkObj.h" +#include "podio/utilities/MaybeSharedPtr.h" +#include "podio/utilities/TypeHelpers.h" + +#ifdef PODIO_JSON_OUTPUT + #include "nlohmann/json.hpp" +#endif + +#include +#include // std::swap + +namespace podio { + +/// Generalized Link type for both Mutable and immutable (default) +/// versions. User facing classes with the expected naming scheme are defined via +/// template aliases in LinkFwd.h +template +class LinkT { + // The typedefs in LinkFwd.h should make sure that at this point + // Mutable classes are stripped, i.e. the user should never be able to trigger + // these! + static_assert(std::is_same_v, FromT>, + "Links need to be instantiated with the default types!"); + static_assert(std::is_same_v, ToT>, + "Links need to be instantiated with the default types!"); + + using LinkObjT = LinkObj; + friend LinkCollection; + friend LinkCollectionIteratorT; + friend LinkT; + + /// Helper member variable to check whether FromU and ToU can be used for this + /// Link. We need this to make SFINAE trigger in some cases below + template + constexpr static bool sameTypes = std::is_same_v && std::is_same_v; + + /// Variable template to for determining whether T is either FromT or ToT. + /// Mainly defined for convenience + template + static constexpr bool isFromOrToT = detail::isInTuple>; + + /// Variable template to for determining whether T is either FromT or ToT or + /// any of their mutable versions. + template + static constexpr bool isMutableFromOrToT = + detail::isInTuple, detail::GetMutableHandleType>>; + +public: + using mutable_type = podio::MutableLink; + using value_type = podio::Link; + using collection_type = podio::LinkCollection; + + /// Constructor + LinkT() : m_obj(new LinkObjT{}, podio::utils::MarkOwned) { + } + + /// Constructor with weight + LinkT(float weight) : m_obj(new LinkObjT{}, podio::utils::MarkOwned) { + m_obj->data.weight = weight; + } + + /// Copy constructor + LinkT(const LinkT& other) = default; + + /// Assignment operator + LinkT& operator=(LinkT other) { + swap(*this, other); + return *this; + } + + /// Implicit conversion of mutable to immutable links + template >> + operator LinkT() const { + return Link(m_obj); + } + + /// Create a mutable deep-copy + /// + /// @param cloneRelations if set to false only the weight will be cloned but + /// not the relations to the objects this link points + /// to. Defaults to true + /// + /// @returns A mutable deep-copy of this link which is independent of the + /// original one + template >> + MutableLink clone(bool cloneRelations = true) const { + auto tmp = new LinkObjT(podio::ObjectID{}, m_obj->data); + if (cloneRelations) { + if (m_obj->m_from) { + tmp->m_from = std::make_unique(*m_obj->m_from); + } + if (m_obj->m_to) { + tmp->m_to = std::make_unique(*m_obj->m_to); + } + } + return MutableLink(podio::utils::MaybeSharedPtr(tmp, podio::utils::MarkOwned)); + } + + /// Create an empty Link handle + template > + static Link makeEmpty() { + return {nullptr}; + } + + /// Destructor + ~LinkT() = default; + + /// Get the weight of the link + float getWeight() const { + return m_obj->data.weight; + } + + /// Set the weight of the link + template > + void setWeight(float value) { + m_obj->data.weight = value; + } + + /// Access the related-from object + const FromT getFrom() const { + if (!m_obj->m_from) { + return FromT::makeEmpty(); + } + return FromT(*(m_obj->m_from)); + } + + /// Set the related-from object + /// + /// @note All setFrom overloads are equivalent and the correct one is selected + /// at compile time. We need to differentiate between the handles, only to + /// make the python bindings work + template , FromT> && + detail::isDefaultHandleType, + int> = 0> + void setFrom(FromU value) { + m_obj->m_from = std::make_unique>(value); + } + + /// Set the related-from object + /// + /// @note All setFrom overloads are equivalent and the correct one is selected + /// at compile time. We need to differentiate between the handles, only to + /// make the python bindings work + template , FromU> && + detail::isMutableHandleType, + int> = 0> + void setFrom(FromU value) { + setFrom(detail::GetDefaultHandleType(value)); + } + + /// Set the related-from object + /// + /// @note All setFrom overloads are equivalent and the correct one is selected + /// at compile time. We need this overload to allow for an implicit conversion + /// to interface types in case the relation contains an interface type. + template , int> = 0> + void setFrom(FromU value) { + setFrom(FromT(value)); + } + + /// Access the related-to object + const ToT getTo() const { + if (!m_obj->m_to) { + return ToT::makeEmpty(); + } + return ToT(*(m_obj->m_to)); + } + + /// Set the related-to object + /// + /// @note All setTo overloads are equivalent and the correct one is selected + /// at compile time. We need to differentiate between the handles, only to + /// make the python bindings work + template , ToT> && + detail::isDefaultHandleType, + int> = 0> + void setTo(ToU value) { + m_obj->m_to = std::make_unique>(value); + } + + /// Set the related-to object + /// + /// @note All setTo overloads are equivalent and the correct one is selected + /// at compile time. We need to differentiate between the handles, only to + /// make the python bindings work + template , ToU> && + detail::isMutableHandleType, + int> = 0> + void setTo(ToU value) { + setTo(detail::GetDefaultHandleType(value)); + } + + /// Set the related-to object + /// + /// @note All setTo overloads are equivalent and the correct one is selected + /// at compile time. We need this overload to allow for an implicit conversion + /// to interface types in case the relation contains an interface type. + template , int> = 0> + void setTo(ToU value) { + setTo(ToT(value)); + } + + /// Templated version for getting an element of the link by type. Only + /// available for Links where FromT and ToT are **not the same type**, + /// and if the requested type is actually part of the Link. It is only + /// possible to get the immutable types from this. Will result in a + /// compilation error if any of these conditions is not met. + /// + /// @tparam T the desired type + /// @returns T the element of the Link + template && isFromOrToT>> + T get() const { + if constexpr (std::is_same_v) { + return getFrom(); + } else { + return getTo(); + } + } + + /// Tuple like index based access to the elements of the Link. Returns + /// only immutable types of the links. This method enables structured + /// bindings for Links. + /// + /// @tparam Index an index (smaller than 3) to access an element of the Link + /// @returns Depending on the value of Index: + /// - 0: The From element of the Link + /// - 1: The To element of the Link + /// - 2: The weight of the Link + template > + auto get() const { + if constexpr (Index == 0) { + return getFrom(); + } else if constexpr (Index == 1) { + return getTo(); + } else { + return getWeight(); + } + } + + /// Templated version for setting an element of the link by type. Only + /// available for Links where FromT and ToT are **not the same type**, + /// and if the requested type is actually part of the Link. Will result + /// in a compilation error if any of these conditions is not met. + /// + /// @tparam T type of value (**inferred!**) + /// @param value the element to set for this link. + template && + (isMutableFromOrToT || detail::isInterfaceInitializableFrom || + detail::isInterfaceInitializableFrom), + int> = 0> + void set(T value) { + if constexpr (std::is_same_v) { + setFrom(std::move(value)); + } else { + setTo(std::move(value)); + } + } + + /// check whether the object is actually available + bool isAvailable() const { + return m_obj; + } + + /// disconnect from Link instance + void unlink() { + m_obj = podio::utils::MaybeSharedPtr(nullptr); + } + + /// Get the ObjectID + podio::ObjectID getObjectID() const { + if (m_obj) { + return m_obj->id; + } + return podio::ObjectID{}; + } + + podio::ObjectID id() const { + return getObjectID(); + } + + bool operator==(const LinkT& other) const { + return m_obj == other.m_obj; + } + + bool operator!=(const LinkT& other) const { + return !(*this == other); + } + + template >> + bool operator==(const LinkT& other) const { + return m_obj == other.m_obj; + } + + template >> + bool operator!=(const LinkT& other) const { + return !(*this == other); + } + + bool operator<(const LinkT& other) const { + return m_obj < other.m_obj; + } + + friend void swap(LinkT& a, LinkT& b) { + using std::swap; + swap(a.m_obj, b.m_obj); // swap out the internal pointers + } + +private: + /// Constructor from existing LinkObj + explicit LinkT(podio::utils::MaybeSharedPtr obj) : m_obj(std::move(obj)) { + } + + template > + LinkT(LinkObjT* obj) : m_obj(podio::utils::MaybeSharedPtr(obj)) { + } + + podio::utils::MaybeSharedPtr m_obj{nullptr}; +}; + +template +std::ostream& operator<<(std::ostream& os, const Link& link) { + if (!link.isAvailable()) { + return os << "[not available]"; + } + + return os << " id: " << link.id() << '\n' + << " weight: " << link.getWeight() << '\n' + << " from: " << link.getFrom().id() << '\n' + << " to: " << link.getTo().id() << '\n'; +} + +#if defined(PODIO_JSON_OUTPUT) && !defined(__CLING__) +template +void to_json(nlohmann::json& j, const podio::LinkT& link) { + j = nlohmann::json{{"weight", link.getWeight()}}; + + j["from"] = nlohmann::json{{"collectionID", link.getFrom().getObjectID().collectionID}, + {"index", link.getFrom().getObjectID().index}}; + + j["to"] = nlohmann::json{{"collectionID", link.getTo().getObjectID().collectionID}, + {"index", link.getTo().getObjectID().index}}; +} +#endif + +} // namespace podio + +#endif // PODIO_DETAIL_LINK_H diff --git a/include/podio/detail/LinkCollectionData.h b/include/podio/detail/LinkCollectionData.h new file mode 100644 index 000000000..948d7247d --- /dev/null +++ b/include/podio/detail/LinkCollectionData.h @@ -0,0 +1,201 @@ +#ifndef PODIO_DETAIL_LINKCOLLECTIONDATA_H +#define PODIO_DETAIL_LINKCOLLECTIONDATA_H + +#include "podio/detail/LinkFwd.h" +#include "podio/detail/LinkObj.h" + +#include "podio/CollectionBase.h" +#include "podio/CollectionBuffers.h" +#include "podio/ICollectionProvider.h" +#include "podio/detail/RelationIOHelpers.h" + +#include +#include +#include + +namespace podio { + +template +class LinkCollectionData { +public: + LinkObjPointerContainer entries{}; + + LinkCollectionData() : + m_rel_from(new std::vector()), m_rel_to(new std::vector()), m_data(new LinkDataContainer()) { + m_refCollections.reserve(2); + m_refCollections.emplace_back(std::make_unique>()); + m_refCollections.emplace_back(std::make_unique>()); + } + + LinkCollectionData(podio::CollectionReadBuffers buffers, bool isSubsetColl) : + m_rel_from(new std::vector()), + m_rel_to(new std::vector()), + m_refCollections(std::move(*buffers.references)) { + if (!isSubsetColl) { + m_data.reset(buffers.dataAsVector()); + } + } + + LinkCollectionData(const LinkCollectionData&) = delete; + LinkCollectionData& operator=(const LinkCollectionData&) = delete; + LinkCollectionData(LinkCollectionData&&) = default; + LinkCollectionData& operator=(LinkCollectionData&&) = default; + ~LinkCollectionData() = default; + + podio::CollectionWriteBuffers getCollectionBuffers(bool isSubsetColl) { + return {isSubsetColl ? nullptr : (void*)&m_data, (void*)m_data.get(), &m_refCollections, &m_vecInfo}; + } + + void clear(bool isSubsetColl) { + if (isSubsetColl) { + // We don't own the objects so no cleanup to do here + entries.clear(); + // Clear the ObjectID I/O buffer + for (auto& pointer : m_refCollections) { + pointer->clear(); + } + return; + } + + // Normal collections manage a bit more and have to clean up a bit more + if (m_data) { + m_data->clear(); + } + for (auto& pointer : m_refCollections) { + pointer->clear(); + } + if (m_rel_from) { + for (auto& item : (*m_rel_from)) { + item.unlink(); + } + m_rel_from->clear(); + } + + if (m_rel_to) { + for (auto& item : (*m_rel_to)) { + item.unlink(); + } + m_rel_to->clear(); + } + + for (auto& obj : entries) { + delete obj; + } + entries.clear(); + } + + void prepareForWrite(bool isSubsetColl) { + for (auto& pointer : m_refCollections) { + pointer->clear(); + } + + // If this is a subset collection use the relation storing mechanism to + // store the ObjectIDs of all reference objects and nothing else + if (isSubsetColl) { + for (const auto* obj : entries) { + m_refCollections[0]->emplace_back(obj->id); + } + return; + } + + m_data->reserve(entries.size()); + for (const auto obj : entries) { + m_data->push_back(obj->data); + + if (obj->m_from) { + m_refCollections[0]->emplace_back(obj->m_from->getObjectID()); + } else { + m_refCollections[0]->push_back({podio::ObjectID::invalid, 0}); + } + + if (obj->m_to) { + m_refCollections[1]->emplace_back(obj->m_to->getObjectID()); + } else { + m_refCollections[1]->push_back({podio::ObjectID::invalid, 0}); + } + } + } + + void prepareAfterRead(uint32_t collectionID) { + int index = 0; + for (const auto data : *m_data) { + auto obj = new LinkObj({index++, collectionID}, data); + entries.emplace_back(obj); + } + + // Keep the I/O data buffer to keep the preparedForWrite state intact + } + + bool setReferences(const podio::ICollectionProvider* collectionProvider, bool isSubsetColl) { + if (isSubsetColl) { + for (const auto& id : *m_refCollections[0]) { + podio::CollectionBase* coll{nullptr}; + LinkObj* obj{nullptr}; + if (collectionProvider->get(id.collectionID, coll)) { + auto* tmp_coll = static_cast*>(coll); + obj = tmp_coll->m_storage.entries[id.index]; + } + entries.push_back(obj); + } + return true; // TODO: check success, how? + } + + // Normal collections have to resolve all relations + for (size_t i = 0; i < entries.size(); ++i) { + const auto id = (*m_refCollections[0])[i]; + if (id.index != podio::ObjectID::invalid) { + podio::CollectionBase* coll = nullptr; + if (!collectionProvider->get(id.collectionID, coll)) { + entries[i]->m_from = nullptr; + continue; + } + podio::detail::addSingleRelation(entries[i]->m_from, coll, id); + } else { + entries[i]->m_from = nullptr; + } + } + + for (size_t i = 0; i < entries.size(); ++i) { + const auto id = (*m_refCollections[1])[i]; + if (id.index != podio::ObjectID::invalid) { + podio::CollectionBase* coll = nullptr; + if (!collectionProvider->get(id.collectionID, coll)) { + entries[i]->m_to = nullptr; + continue; + } + podio::detail::addSingleRelation(entries[i]->m_to, coll, id); + } else { + entries[i]->m_to = nullptr; + } + } + + return true; // TODO: check success, how? + } + + void makeSubsetCollection() { + // Subset collections do not need all the data buffers that normal + // collections need, so we can free them here + m_data.reset(nullptr); + + m_rel_from.reset(nullptr); + m_rel_to.reset(nullptr); + + // Subset collections need one vector of ObjectIDs for I/O purposes. + m_refCollections.resize(1); + m_refCollections[0] = std::make_unique>(); + } + +private: + // members to handle relations + podio::UVecPtr m_rel_from{nullptr}; + podio::UVecPtr m_rel_to{nullptr}; + + // I/O related buffers (as far as necessary) + podio::CollRefCollection m_refCollections{}; + podio::VectorMembersInfo m_vecInfo{}; + std::unique_ptr m_data{nullptr}; +}; + +} // namespace podio + +#endif // PODIO_DETAIL_LINKCOLLECTIONDATA_H diff --git a/include/podio/detail/LinkCollectionImpl.h b/include/podio/detail/LinkCollectionImpl.h new file mode 100644 index 000000000..fe1a61ace --- /dev/null +++ b/include/podio/detail/LinkCollectionImpl.h @@ -0,0 +1,372 @@ +#ifndef PODIO_DETAIL_LINKCOLLECTIONIMPL_H +#define PODIO_DETAIL_LINKCOLLECTIONIMPL_H + +#include "podio/detail/Link.h" +#include "podio/detail/LinkCollectionData.h" +#include "podio/detail/LinkCollectionIterator.h" +#include "podio/detail/LinkFwd.h" +#include "podio/detail/LinkObj.h" + +#include "podio/CollectionBase.h" +#include "podio/CollectionBufferFactory.h" +#include "podio/CollectionBuffers.h" +#include "podio/DatamodelRegistry.h" +#include "podio/ICollectionProvider.h" +#include "podio/SchemaEvolution.h" +#include "podio/utilities/MaybeSharedPtr.h" +#include "podio/utilities/TypeHelpers.h" + +#ifdef PODIO_JSON_OUTPUT + #include "nlohmann/json.hpp" +#endif + +#include +#include +#include +#include +#include +#include + +namespace podio { + +template +class LinkCollection : public podio::CollectionBase { + static_assert(std::is_same_v>, + "Links need to be instantiated with the default types!"); + static_assert(std::is_same_v>, + "Links need to be instantiated with the default types!"); + + // convenience typedefs + using CollectionDataT = podio::LinkCollectionData; + +public: + using value_type = Link; + using mutable_type = MutableLink; + using const_iterator = LinkCollectionIterator; + using iterator = LinkMutableCollectionIterator; + using difference_type = ptrdiff_t; + using size_type = size_t; + + LinkCollection() = default; + + LinkCollection(CollectionDataT&& data, bool isSubsetColl) : + m_isSubsetColl(isSubsetColl), m_collectionID(0), m_storage(std::move(data)) { + } + + // Move-only type + LinkCollection(const LinkCollection&) = delete; + LinkCollection& operator=(const LinkCollection&) = delete; + LinkCollection(LinkCollection&&) = default; + LinkCollection& operator=(LinkCollection&&) = default; + + ~LinkCollection() { + // Need to tell the storage how to clean up + m_storage.clear(m_isSubsetColl); + } + + /// Append a new link to the collection and return this object + mutable_type create() { + if (m_isSubsetColl) { + throw std::logic_error("Cannot create new elements on a subset collection"); + } + + auto obj = m_storage.entries.emplace_back(new LinkObj()); + obj->id = {int(m_storage.entries.size() - 1), m_collectionID}; + return mutable_type(podio::utils::MaybeSharedPtr(obj)); + } + + /// Returns the immutable object of given index + value_type operator[](unsigned int index) const { + return value_type(m_storage.entries[index]); + } + /// Returns the mutable object of given index + mutable_type operator[](unsigned int index) { + return mutable_type(podio::utils::MaybeSharedPtr(m_storage.entries[index])); + } + /// Returns the immutable object of given index + value_type at(unsigned int index) const { + return value_type(m_storage.entries.at(index)); + } + /// Returns the mutable object of given index + mutable_type at(unsigned int index) { + return mutable_type(podio::utils::MaybeSharedPtr(m_storage.at(index))); + } + + void push_back(mutable_type object) { + // We have to do different things here depending on whether this is a + // subset collection or not. A normal collection cannot collect objects + // that are already part of another collection, while a subset collection + // can only collect such objects + if (!m_isSubsetColl) { + auto obj = object.m_obj; + if (obj->id.index == podio::ObjectID::untracked) { + const auto size = m_storage.entries.size(); + obj->id = {(int)size, m_collectionID}; + m_storage.entries.push_back(obj.release()); + } else { + throw std::invalid_argument("Object already in a collection. Cannot add it to a second collection"); + } + + } else { + push_back(value_type(object)); + } + } + + void push_back(value_type object) { + if (!m_isSubsetColl) { + throw std::invalid_argument("Can only add immutable objects to subset collections"); + } + auto obj = object.m_obj; + if (obj->id.index < 0) { + throw std::invalid_argument( + "Object needs to be tracked by another collection in order for it to be storable in a subset collection"); + } + m_storage.entries.push_back(obj.release()); + } + + /// Number of elements in the collection + size_t size() const override { + return m_storage.entries.size(); + } + + /// maximal number of elements in the collection + std::size_t max_size() const override { + return m_storage.entries.max_size(); + } + + /// Is the collection empty + bool empty() const override { + return m_storage.entries.empty(); + } + + void clear() override { + m_storage.clear(m_isSubsetColl); + m_isPrepared = false; + } + + void print(std::ostream& os = std::cout, bool flush = true) const override { + os << *this; + if (flush) { + os.flush(); + } + } + + // support for the iterator protocol + const_iterator begin() const { + return const_iterator(0, &m_storage.entries); + } + const_iterator cbegin() const { + return begin(); + } + const_iterator end() const { + return const_iterator(m_storage.entries.size(), &m_storage.entries); + } + const_iterator cend() const { + return end(); + } + iterator begin() { + return iterator(0, &m_storage.entries); + } + iterator end() { + return iterator(m_storage.entries.size(), &m_storage.entries); + } + + bool isValid() const override { + return m_isValid; + } + + podio::CollectionWriteBuffers getBuffers() override { + return m_storage.getCollectionBuffers(m_isSubsetColl); + } + + const std::string_view getTypeName() const override { + return podio::detail::linkCollTypeName(); + } + + const std::string_view getValueTypeName() const override { + return podio::detail::linkTypeName(); + } + + const std::string_view getDataTypeName() const override { + return "float"; + } + + bool isSubsetCollection() const override { + return m_isSubsetColl; + } + + void setSubsetCollection(bool setSubset = true) override { + if (m_isSubsetColl != setSubset && !m_storage.entries.empty()) { + throw std::logic_error("Cannot change the character of a collection that already contains elements"); + } + + if (setSubset) { + m_storage.makeSubsetCollection(); + } + m_isSubsetColl = setSubset; + } + + void setID(unsigned id) override { + m_collectionID = id; + } + + unsigned getID() const override { + return m_collectionID; + } + + void prepareForWrite() const override { + // TODO: Replace this double locking pattern with an atomic and only one + // lock. Problem: std::atomic is not default movable + { + std::lock_guard lock{*m_storageMtx}; + // If the collection has been prepared already there is nothing to do + if (m_isPrepared) { + return; + } + } + + std::lock_guard lock{*m_storageMtx}; + // by the time we acquire the lock another thread might have already + // succeeded, so we need to check again now + if (m_isPrepared) { + return; + } + m_storage.prepareForWrite(m_isSubsetColl); + m_isPrepared = true; + } + + void prepareAfterRead() override { + // No need to go through this again if we have already done it + if (m_isPrepared) { + return; + } + + if (!m_isSubsetColl) { + // Subset collections do not store any data that would require post-processing + m_storage.prepareAfterRead(m_collectionID); + } + // Preparing a collection doesn't affect the underlying I/O buffers, so this + // collection is still prepared + m_isPrepared = true; + } + + bool setReferences(const ICollectionProvider* collectionProvider) override { + return m_storage.setReferences(collectionProvider, m_isSubsetColl); + } + + static constexpr SchemaVersionT schemaVersion = 1; + + SchemaVersionT getSchemaVersion() const override { + return schemaVersion; + } + + size_t getDatamodelRegistryIndex() const override { + return podio::DatamodelRegistry::NoDefinitionNecessary; + } + +private: + // For setReferences, we need to give our own CollectionData access to our + // private entries. Otherwise we would need to expose a public member function + // that gives access to the Obj* which is definitely not what we want + friend CollectionDataT; + + bool m_isValid{false}; + mutable bool m_isPrepared{false}; + bool m_isSubsetColl{false}; + uint32_t m_collectionID{0}; + mutable std::unique_ptr m_storageMtx{std::make_unique()}; + mutable CollectionDataT m_storage{}; +}; + +template +std::ostream& operator<<(std::ostream& o, const LinkCollection& v) { + const auto old_flags = o.flags(); + o << " id: weight:" << '\n'; + for (const auto&& el : v) { + o << std::scientific << std::showpos << std::setw(12) << el.id() << " " << std::setw(12) << " " << el.getWeight() + << '\n'; + + o << " from : "; + o << el.getFrom().id() << std::endl; + o << " to : "; + o << el.getTo().id() << std::endl; + } + + o.flags(old_flags); + return o; +} + +namespace detail { + template + podio::CollectionReadBuffers createLinkBuffers(bool subsetColl) { + auto readBuffers = podio::CollectionReadBuffers{}; + readBuffers.type = podio::detail::linkCollTypeName(); + readBuffers.schemaVersion = podio::LinkCollection::schemaVersion; + readBuffers.data = subsetColl ? nullptr : new LinkDataContainer(); + + // Either it is a subset collection or we have two relations + const auto nRefs = subsetColl ? 1 : 2; + readBuffers.references = new podio::CollRefCollection(nRefs); + for (auto& ref : *readBuffers.references) { + // Make sure to place usable buffer pointers here + ref = std::make_unique>(); + } + + readBuffers.createCollection = [](podio::CollectionReadBuffers buffers, bool isSubsetColl) { + LinkCollectionData data(buffers, isSubsetColl); + return std::make_unique>(std::move(data), isSubsetColl); + }; + + readBuffers.recast = [](podio::CollectionReadBuffers& buffers) { + if (buffers.data) { + buffers.data = podio::CollectionWriteBuffers::asVector(buffers.data); + } + }; + + readBuffers.deleteBuffers = [](podio::CollectionReadBuffers& buffers) { + if (buffers.data) { + // If we have data then we are not a subset collection and we have + // to clean up all type erased buffers by casting them back to + // something that we can delete + delete static_cast(buffers.data); + } + delete buffers.references; + delete buffers.vectorMembers; + }; + + return readBuffers; + } + + template + bool registerLinkCollection(const std::string_view linkTypeName) { + const static auto reg = [&linkTypeName]() { + const auto schemaVersion = LinkCollection::schemaVersion; + + auto& factory = CollectionBufferFactory::mutInstance(); + factory.registerCreationFunc(std::string(linkTypeName), schemaVersion, createLinkBuffers); + + // For now passing the same schema version for from and current version + // simply to make SchemaEvolution aware of LinkCollections + podio::SchemaEvolution::mutInstance().registerEvolutionFunc(std::string(linkTypeName), schemaVersion, + schemaVersion, SchemaEvolution::noOpSchemaEvolution, + SchemaEvolution::Priority::AutoGenerated); + + return true; + }(); + return reg; + } +} // namespace detail + +#if defined(PODIO_JSON_OUTPUT) && !defined(__CLING__) +template +void to_json(nlohmann::json& j, const podio::LinkCollection& collection) { + j = nlohmann::json::array(); + for (auto&& elem : collection) { + j.emplace_back(elem); + } +} +#endif + +} // namespace podio + +#endif // PODIO_DETAIL_LINKCOLLECTIONIMPL_H diff --git a/include/podio/detail/LinkCollectionIterator.h b/include/podio/detail/LinkCollectionIterator.h new file mode 100644 index 000000000..35fcba8d0 --- /dev/null +++ b/include/podio/detail/LinkCollectionIterator.h @@ -0,0 +1,47 @@ +#ifndef PODIO_DETAIL_LINKCOLLECTIONITERATOR_H +#define PODIO_DETAIL_LINKCOLLECTIONITERATOR_H + +#include "podio/detail/LinkFwd.h" +#include "podio/utilities/MaybeSharedPtr.h" + +namespace podio { +template +class LinkCollectionIteratorT { + using LinkType = LinkT; + using LinkObjT = LinkObj; + +public: + LinkCollectionIteratorT(size_t index, const LinkObjPointerContainer* coll) : + m_index(index), m_object(podio::utils::MaybeSharedPtr{nullptr}), m_collection(coll) { + } + + LinkCollectionIteratorT(const LinkCollectionIteratorT&) = delete; + LinkCollectionIteratorT& operator=(const LinkCollectionIteratorT&) = delete; + + bool operator!=(const LinkCollectionIteratorT& other) const { + return m_index != other.m_index; // TODO: may not be complete + } + + LinkType operator*() { + m_object.m_obj = podio::utils::MaybeSharedPtr((*m_collection)[m_index]); + return m_object; + } + + LinkType* operator->() { + m_object.m_obj = podio::utils::MaybeSharedPtr((*m_collection)[m_index]); + return &m_object; + } + + LinkCollectionIteratorT& operator++() { + ++m_index; + return *this; + } + +private: + size_t m_index; + LinkType m_object; + const LinkObjPointerContainer* m_collection; +}; +} // namespace podio + +#endif // PODIO_DETAIL_LINKCOLLECTIONITERATOR_H diff --git a/include/podio/detail/LinkFwd.h b/include/podio/detail/LinkFwd.h new file mode 100644 index 000000000..254447ef7 --- /dev/null +++ b/include/podio/detail/LinkFwd.h @@ -0,0 +1,104 @@ +#ifndef PODIO_DETAIL_LINKFWD_H +#define PODIO_DETAIL_LINKFWD_H + +#include "podio/utilities/TypeHelpers.h" + +#include +#include +#include +#include +#include + +namespace podio { +namespace detail { + + /// Get the collection type name for a LinkCollection + /// + /// @tparam FromT the From type of the link + /// @tparam ToT the To type of the link + /// @returns a type string that is a valid c++ template instantiation + template + inline const std::string_view linkCollTypeName() { + const static std::string typeName = + std::string("podio::LinkCollection<") + FromT::typeName + "," + ToT::typeName + ">"; + return std::string_view{typeName}; + } + + /// Get the value type name for a LinkCollection + /// + /// @tparam FromT the From type of the link + /// @tparam ToT the To type of the link + /// @returns a type string that is a valid c++ template instantiation + template + inline const std::string_view linkTypeName() { + const static std::string typeName = std::string("podio::Link<") + FromT::typeName + "," + ToT::typeName + ">"; + return std::string_view{typeName}; + } + + /// Get an SIO friendly type name for a LinkCollection (necessary for + /// registration in the SIOBlockFactory) + /// + /// @tparam FromT the From type of the link + /// @tparam ToT the To type of the link + /// @returns a string that uniquely identifies this combination of From and To + /// types + template + inline const std::string& linkSIOName() { + static auto n = std::string("LINK_FROM_") + FromT::typeName + "_TO_" + ToT::typeName; + std::replace(n.begin(), n.end(), ':', '_'); + return n; + } +} // namespace detail + +// Forward declarations and typedefs used throughout the whole Link business +template +class LinkObj; + +template +using LinkObjPointerContainer = std::deque*>; + +/// Simple struct to keep implementation more in line with generated links and +/// to ease evolution of generated links into templated ones +struct LinkData { + float weight{}; +}; + +using LinkDataContainer = std::vector; + +template +class LinkT; + +template +using Link = LinkT, detail::GetDefaultHandleType, false>; + +template +using MutableLink = LinkT, detail::GetDefaultHandleType, true>; + +template +class LinkCollection; + +template +class LinkCollectionData; + +template +class LinkCollectionIteratorT; + +template +using LinkCollectionIterator = LinkCollectionIteratorT; + +template +using LinkMutableCollectionIterator = LinkCollectionIteratorT; + +} // namespace podio + +namespace std { +/// Specialization for enabling structured bindings for Links +template +struct tuple_size> : std::integral_constant {}; + +/// Specialization for enabling structured bindings for Links +template +struct tuple_element> : tuple_element> {}; +} // namespace std + +#endif // PODIO_DETAIL_LINKFWD_H diff --git a/include/podio/detail/LinkObj.h b/include/podio/detail/LinkObj.h new file mode 100644 index 000000000..ef1860514 --- /dev/null +++ b/include/podio/detail/LinkObj.h @@ -0,0 +1,53 @@ +#ifndef PODIO_DETAIL_LINKOBJ_H +#define PODIO_DETAIL_LINKOBJ_H + +#include "podio/detail/LinkFwd.h" + +#include "podio/ObjectID.h" + +#include + +namespace podio { + +template +class LinkObj { + + friend Link; + friend MutableLink; + +public: + /// Constructor + LinkObj() : id(), data(LinkData{1.0f}), m_from(nullptr), m_to(nullptr) { + } + + /// Constructor from ObjectID and data (does not initialize relations yet!) + LinkObj(const podio::ObjectID id_, LinkData data_) : id(id_), data(data_) { + } + + /// Copy constructor (deep-copy of relations) + LinkObj(const LinkObj& other) : id(), data(other.data), m_from(nullptr), m_to(nullptr) { + if (other.m_from) { + m_from = new FromT(*other.m_from); + } + if (other.m_to) { + m_to = new ToT(*other.m_to); + } + } + + /// No assignment operator + LinkObj& operator=(const LinkObj&) = delete; + + /// Destructor + ~LinkObj() = default; + +public: + podio::ObjectID id{}; + LinkData data{1.0f}; + + std::unique_ptr m_from{nullptr}; + std::unique_ptr m_to{nullptr}; +}; + +} // namespace podio + +#endif // PODIO_DETAIL_LINKOBJ_H diff --git a/include/podio/detail/LinkSIOBlock.h b/include/podio/detail/LinkSIOBlock.h new file mode 100644 index 000000000..de4f7082b --- /dev/null +++ b/include/podio/detail/LinkSIOBlock.h @@ -0,0 +1,81 @@ +#ifndef PODIO_DETAIL_LINKSIOBLOCK_H +#define PODIO_DETAIL_LINKSIOBLOCK_H + +#include "podio/detail/LinkCollectionImpl.h" + +#include "podio/CollectionBufferFactory.h" +#include "podio/CollectionBuffers.h" +#include "podio/SIOBlock.h" + +#include +#include +#include + +#include + +namespace podio { +template +class LinkSIOBlock : public podio::SIOBlock { +public: + LinkSIOBlock() : + SIOBlock(podio::detail::linkSIOName(), + sio::version::encode_version(LinkCollection::schemaVersion, 0)) { + podio::SIOBlockFactory::instance().registerBlockForCollection( + std::string(podio::detail::linkTypeName()), this); + } + + LinkSIOBlock(const std::string& name) : + SIOBlock(name, sio::version::encode_version(LinkCollection::schemaVersion, 0)) { + } + + void read(sio::read_device& device, sio::version_type version) override { + auto& bufferFactory = podio::CollectionBufferFactory::instance(); + // TODO: + // - Error handling of empty optional + auto maybeBuffers = bufferFactory.createBuffers(std::string(podio::detail::linkCollTypeName()), + sio::version::major_version(version), m_subsetColl); + m_buffers = maybeBuffers.value(); + + if (!m_subsetColl) { + unsigned size{0}; + device.data(size); + auto* dataVec = m_buffers.dataAsVector(); + dataVec->resize(size); + podio::handlePODDataSIO(device, dataVec->data(), size); + } + + // ---- read ref collections + auto* refColls = m_buffers.references; + for (auto& refC : *refColls) { + unsigned size{0}; + device.data(size); + refC->resize(size); + podio::handlePODDataSIO(device, refC->data(), size); + } + } + + void write(sio::write_device& device) override { + if (!m_subsetColl) { + auto* dataVec = podio::CollectionWriteBuffers::asVector(m_buffers.data); + unsigned size = dataVec->size(); + device.data(size); + podio::handlePODDataSIO(device, dataVec->data(), size); + } + + // ---- write ref collections ------ + auto* refColls = m_buffers.references; + for (auto& refC : *refColls) { + unsigned size = refC->size(); + device.data(size); + podio::handlePODDataSIO(device, refC->data(), size); + } + } + + SIOBlock* create(const std::string& name) const override { + return new LinkSIOBlock(name); + } +}; + +} // namespace podio + +#endif // PODIO_DETAIL_LINKSIOBLOCK_H diff --git a/include/podio/detail/PreprocessorMacros.h b/include/podio/detail/PreprocessorMacros.h new file mode 100644 index 000000000..3a038a9c2 --- /dev/null +++ b/include/podio/detail/PreprocessorMacros.h @@ -0,0 +1,10 @@ +#ifndef PODIO_DETAIL_PREPROCESSORMACROS_H +#define PODIO_DETAIL_PREPROCESSORMACROS_H + +// Preprocessor helper macros for concatenating tokens at preprocessing times +// Necessary because we use __COUNTER__ for unique names of static +// variables for values returned by registration function calls +#define PODIO_PP_CONCAT_IMPL(x, y) x##y +#define PODIO_PP_CONCAT(x, y) PODIO_PP_CONCAT_IMPL(x, y) + +#endif // PODIO_DETAIL_PREPROCESSORMACROS_H diff --git a/include/podio/utilities/TypeHelpers.h b/include/podio/utilities/TypeHelpers.h index 9b5ff4640..ebff8e3be 100644 --- a/include/podio/utilities/TypeHelpers.h +++ b/include/podio/utilities/TypeHelpers.h @@ -45,6 +45,9 @@ namespace det { template typename Op, typename... Args> using detected_or = detail::detector; + + template