From 2e38f24868cb739e5562a21cedd7a1265b660ff6 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Tue, 6 Dec 2022 18:23:21 +0100 Subject: [PATCH 01/19] Split model reading into file reading and parsing --- python/podio/podio_config_reader.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/python/podio/podio_config_reader.py b/python/podio/podio_config_reader.py index a72bcce06..3992a3aa9 100644 --- a/python/podio/podio_config_reader.py +++ b/python/podio/podio_config_reader.py @@ -407,24 +407,21 @@ def _read_datatype(cls, value): return datatype @classmethod - def read(cls, yamlfile, package_name, upstream_edm=None): - """Read the datamodel definition from the yamlfile.""" - with open(yamlfile, "r", encoding='utf-8') as stream: - content = yaml.load(stream, yaml.SafeLoader) - + def parse_model(cls, model_dict, package_name, upstream_edm=None): + """Parse a model from the dictionary, e.g. read from a yaml file.""" components = {} - if "components" in content: - for klassname, value in content["components"].items(): + if "components" in model_dict: + for klassname, value in model_dict["components"].items(): components[klassname] = cls._read_component(value) datatypes = {} - if "datatypes" in content: - for klassname, value in content["datatypes"].items(): + if "datatypes" in model_dict: + for klassname, value in model_dict["datatypes"].items(): datatypes[klassname] = cls._read_datatype(value) options = copy.deepcopy(cls.options) - if "options" in content: - for option, value in content["options"].items(): + if "options" in model_dict: + for option, value in model_dict["options"].items(): options[option] = value # Normalize the includeSubfoler internally already here @@ -438,3 +435,11 @@ def read(cls, yamlfile, package_name, upstream_edm=None): datamodel = DataModel(datatypes, components, options) validator.validate(datamodel, upstream_edm) return datamodel + + @classmethod + def read(cls, yamlfile, package_name, upstream_edm=None): + """Read the datamodel definition from the yamlfile.""" + with open(yamlfile, "r", encoding='utf-8') as stream: + content = yaml.load(stream, yaml.SafeLoader) + + return cls.parse_model(content, package_name, upstream_edm) From e85b0d7cd6fec6c2779ea59c4f1e9763cae1b003 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Tue, 6 Dec 2022 18:23:33 +0100 Subject: [PATCH 02/19] Add possibility to dump parsed EDMs to JSON --- python/podio/generator_utils.py | 26 +++++++++ python/podio/test_DataModelJSONEncoder.py | 71 +++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 python/podio/test_DataModelJSONEncoder.py diff --git a/python/podio/generator_utils.py b/python/podio/generator_utils.py index 9711dcdf1..e267772f8 100644 --- a/python/podio/generator_utils.py +++ b/python/podio/generator_utils.py @@ -4,6 +4,7 @@ """ import re +import json def _get_namespace_class(full_type): @@ -183,6 +184,14 @@ def setter_name(self, get_syntax, is_relation=False): return self.name return _prefix_name(self.name, 'set') + def _to_json(self): + """Return a string representation that can be parsed again.""" + # The __str__ method is geared towards c++ too much, so we have to build + # things again here from available information + def_val = f'{{{self.default_val}}}' if self.default_val else '' + description = f' // {self.description}' if self.description else '' + return f'{self.full_type} {self.name}{def_val}{description}' + class DataModel: # pylint: disable=too-few-public-methods """A class for holding a complete datamodel read from a configuration file""" @@ -197,3 +206,20 @@ def __init__(self, datatypes=None, components=None, options=None): # use subfolder when including package header files "includeSubfolder": False, } + + def _to_json(self): + """Return the dictionary, so that we can easily hook this into the pythons + JSON ecosystem""" + return self.__dict__ + + +class DataModelJSONEncoder(json.JSONEncoder): + """A JSON encoder for DataModels, resp. anything hat has a _to_json method.""" + + def default(self, o): + """The override for the default, first trying to call _to_json, otherwise + handing off to the default JSONEncoder""" + try: + return o._to_json() # pylint: disable=protected-access + except AttributeError: + return super().default(o) diff --git a/python/podio/test_DataModelJSONEncoder.py b/python/podio/test_DataModelJSONEncoder.py new file mode 100644 index 000000000..b63ff22f0 --- /dev/null +++ b/python/podio/test_DataModelJSONEncoder.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Unit tests for the JSON encoding of data models""" + +import unittest + +from podio.generator_utils import DataModelJSONEncoder +from podio.podio_config_reader import MemberParser + + +def get_member_var_json(string): + """Get a MemberVariable encoded as JSON from the passed string. + + Passes through the whole chain of parsing and JSON encoding, as it is done + during data model encoding. + + Args: + string (str): The member variable definition as a string. NOTE: here it is + assumed that this is a valid string that can be parsed. + + Returns: + str: The json encoded member variable + """ + parser = MemberParser() + member_var = parser.parse(string, False) # be lenient with missing descriptions + return DataModelJSONEncoder().encode(member_var).strip('"') # strip quotes from JSON + + +class DataModelJSONEncoderTest(unittest.TestCase): + """Unit tests for the DataModelJSONEncoder and the utility functionality in MemberVariable""" + + def test_encode_only_types(self): + """Test that encoding works for type declarations only""" + for mdef in (r"float someFloat", + r"ArbitraryType name", + r"std::int16_t fixedWidth", + r"namespace::Type type"): + self.assertEqual(get_member_var_json(mdef), mdef) + + # Fixed with without std are encoded with std namespace + fixed_w = r"int32_t fixedWidth" + self.assertEqual(get_member_var_json(fixed_w), f"std::{fixed_w}") + + def test_encode_array_types(self): + """Test that encoding array member variable declarations work""" + for mdef in (r"std::array anArray", + r"std::array fwArr", + r"std::array typeArr", + r"std::array namespacedTypeArr"): + self.assertEqual(get_member_var_json(mdef), mdef) + + def test_encode_default_vals(self): + """Test that encoding definitions with default values works""" + for mdef in (r"int i{42}", + r"std::uint32_t uint{64}", + r"ArbType a{123}", + r"namespace::Type t{whatever}", + r"std::array fs{3.14f, 6.28f}", + r"std::array typeArr{1, 2, 3}"): + self.assertEqual(get_member_var_json(mdef), mdef) + + def test_encode_with_description(self): + """Test that encoding definitions that contain a description works""" + for mdef in (r"int i // an unitialized int", + r"std::uint32_t ui{42} // an initialized unsigned int", + r"std::array fs // a float array", + r"std::array tA{1, 2, 3} // an initialized array of namespaced types", + r"AType type // a very special type", + r"nsp::Type nspT // a namespaced type", + r"nsp::Type nspT{with init} // an initialized namespaced type", + r"ArbitratyType arbT{42} // an initialized type"): + self.assertEqual(get_member_var_json(mdef), mdef) From daed17e70f5a2d7d335a6efe10b7aa07460e9350 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 8 Dec 2022 17:17:25 +0100 Subject: [PATCH 03/19] Add registry for datamodel JSON defintions - Generate model definitions in JSON format as constexpr string literals - Register constexpr string literals in registry --- include/podio/EDMDefinitionRegistry.h | 46 +++++++++++++++++++ python/podio_class_generator.py | 19 +++++++- python/templates/CMakeLists.txt | 1 + python/templates/DatamodelDefinition.h.jinja2 | 16 +++++++ src/CMakeLists.txt | 5 +- src/EDMDefinitionRegistry.cc | 44 ++++++++++++++++++ 6 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 include/podio/EDMDefinitionRegistry.h create mode 100644 python/templates/DatamodelDefinition.h.jinja2 create mode 100644 src/EDMDefinitionRegistry.cc diff --git a/include/podio/EDMDefinitionRegistry.h b/include/podio/EDMDefinitionRegistry.h new file mode 100644 index 000000000..bc3e42d72 --- /dev/null +++ b/include/podio/EDMDefinitionRegistry.h @@ -0,0 +1,46 @@ +#ifndef PODIO_EDMDEFINITIONREGISTRY_H +#define PODIO_EDMDEFINITIONREGISTRY_H + +#include +#include +#include +#include + +namespace podio { +class EDMDefinitionRegistry { +public: + static EDMDefinitionRegistry& instance(); + ~EDMDefinitionRegistry() = default; + EDMDefinitionRegistry(const EDMDefinitionRegistry&) = delete; + EDMDefinitionRegistry& operator=(const EDMDefinitionRegistry&) = delete; + EDMDefinitionRegistry(EDMDefinitionRegistry&&) = delete; + EDMDefinitionRegistry& operator=(const EDMDefinitionRegistry&&) = delete; + + /** + * Get the definition (in JSON format) of the EDM with the given edm_name. If + * no EDM under the given name can be found, an empty model definition is + * returned + */ + const std::string_view getDefinition(std::string_view edm_name) const; + + /** + * Get the defintion (in JSON format) of the EDM wth the given index. If no + * EDM is found under the given index, an empty model definition is returned. + */ + const std::string_view getDefinition(size_t index) const; + + /** + * Register a definition and return the index in the registry. If a definition + * already exists under the given name, then the index of the existing + * definition is returned + */ + size_t registerEDM(std::string name, std::string_view definition); + +private: + EDMDefinitionRegistry() = default; + + std::vector> m_definitions{}; +}; +} // namespace podio + +#endif // PODIO_EDMDEFINITIONREGISTRY_H diff --git a/python/podio_class_generator.py b/python/podio_class_generator.py index 7c6fc5b40..ec015230c 100755 --- a/python/podio_class_generator.py +++ b/python/podio_class_generator.py @@ -17,7 +17,7 @@ import jinja2 from podio.podio_config_reader import PodioConfigReader -from podio.generator_utils import DataType, DefinitionError +from podio.generator_utils import DataType, DefinitionError, DataModelJSONEncoder THIS_DIR = os.path.dirname(os.path.abspath(__file__)) TEMPLATE_DIR = os.path.join(THIS_DIR, 'templates') @@ -113,6 +113,8 @@ def process(self): for name, datatype in self.datamodel.datatypes.items(): self._process_datatype(name, datatype) + self._write_edm_def_file() + if 'ROOT' in self.io_handlers: self._create_selection_xml() self.print_report() @@ -203,6 +205,9 @@ def _fill_templates(self, template_base, data): def _process_component(self, name, component): """Process one component""" + # Make a copy here and add the preprocessing steps to that such that the + # original definition can be left untouched + component = deepcopy(component) includes = set() includes.update(*(m.includes for m in component['Members'])) @@ -368,6 +373,18 @@ def _preprocess_datatype(self, name, definition): return data + def _write_edm_def_file(self): + """Write the edm definition to a compile time string""" + model_encoder = DataModelJSONEncoder() + data = { + 'package_name': self.package_name, + 'edm_definition': model_encoder.encode(self.datamodel), + 'incfolder': self.incfolder, + } + + self._write_file('DatamodelDefinition.h', + self._eval_template('DatamodelDefinition.h.jinja2', data)) + def _get_member_includes(self, members): """Process all members and gather the necessary includes""" includes = set() diff --git a/python/templates/CMakeLists.txt b/python/templates/CMakeLists.txt index c3c382ad5..be5f4b307 100644 --- a/python/templates/CMakeLists.txt +++ b/python/templates/CMakeLists.txt @@ -14,6 +14,7 @@ set(PODIO_TEMPLATES ${CMAKE_CURRENT_LIST_DIR}/selection.xml.jinja2 ${CMAKE_CURRENT_LIST_DIR}/SIOBlock.cc.jinja2 ${CMAKE_CURRENT_LIST_DIR}/SIOBlock.h.jinja2 + ${CMAKE_CURRENT_LIST_DIR}/DatamodelDefinition.h.jinja2 ${CMAKE_CURRENT_LIST_DIR}/macros/collections.jinja2 ${CMAKE_CURRENT_LIST_DIR}/macros/declarations.jinja2 ${CMAKE_CURRENT_LIST_DIR}/macros/implementations.jinja2 diff --git a/python/templates/DatamodelDefinition.h.jinja2 b/python/templates/DatamodelDefinition.h.jinja2 new file mode 100644 index 000000000..2b56a5b87 --- /dev/null +++ b/python/templates/DatamodelDefinition.h.jinja2 @@ -0,0 +1,16 @@ +// AUTOMATICALLY GENERATED FILE - DO NOT EDIT + +#include "podio/EDMDefinitionRegistry.h" + +namespace {{ package_name }}::meta { +/** + * The complete definition of the datamodel at generation time in JSON format. + */ +static constexpr auto {{ package_name }}__JSONDefinition = R"EDMDEFINITION({{ edm_definition }})EDMDEFINITION"; + +/** + * The index of the definition in the registry + */ +static const auto definitionRegistryIndex = podio::EDMDefinitionRegistry::instance().registerEDM("{{ package_edm_edm_name }}", {{ package_name }}__JSONDefinition); + +} // namespace {{ package_name }}::meta diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d0017cba7..956903316 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,7 +48,9 @@ SET(core_sources CollectionIDTable.cc GenericParameters.cc ASCIIWriter.cc - EventStore.cc) + EventStore.cc + EDMDefinitionRegistry.cc + ) SET(core_headers ${CMAKE_SOURCE_DIR}/include/podio/CollectionBase.h @@ -59,6 +61,7 @@ SET(core_headers ${CMAKE_SOURCE_DIR}/include/podio/ObjectID.h ${CMAKE_SOURCE_DIR}/include/podio/UserDataCollection.h ${CMAKE_SOURCE_DIR}/include/podio/podioVersion.h + ${CMAKE_SOURCE_DIR}/include/podio/EDMDefinitionRegistry.h ) PODIO_ADD_LIB_AND_DICT(podio "${core_headers}" "${core_sources}" selection.xml) diff --git a/src/EDMDefinitionRegistry.cc b/src/EDMDefinitionRegistry.cc new file mode 100644 index 000000000..b1048ead1 --- /dev/null +++ b/src/EDMDefinitionRegistry.cc @@ -0,0 +1,44 @@ +#include "podio/EDMDefinitionRegistry.h" + +#include +#include +#include + +namespace podio { +EDMDefinitionRegistry& EDMDefinitionRegistry::instance() { + static EDMDefinitionRegistry registryInstance; + return registryInstance; +} + +size_t EDMDefinitionRegistry::registerEDM(std::string name, std::string_view definition) { + const auto it = std::find_if(m_definitions.cbegin(), m_definitions.cend(), + [&name](const auto& kvPair) { return kvPair.first == name; }); + + if (it == m_definitions.cend()) { + int index = m_definitions.size(); + m_definitions.emplace_back(name, definition); + return index; + } + + // TODO: Output? + return std::distance(m_definitions.cbegin(), it); +} + +const std::string_view EDMDefinitionRegistry::getDefinition(std::string_view edm_name) const { + const auto it = std::find_if(m_definitions.cbegin(), m_definitions.cend(), + [&edm_name](const auto& kvPair) { return kvPair.first == edm_name; }); + + // TODO: Output when not found + return getDefinition(std::distance(m_definitions.cbegin(), it)); +} + +const std::string_view EDMDefinitionRegistry::getDefinition(size_t index) const { + if (index >= m_definitions.size()) { + static constexpr std::string_view emptyDef = "{}"; // valid empty JSON + return emptyDef; + } + + return m_definitions[index].second; +} + +} // namespace podio From 2b0a5a383809a9f533f2fa6aa31fcf04640238e1 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 8 Dec 2022 19:14:39 +0100 Subject: [PATCH 04/19] Populate definition registry via static variable initialization - Also make the collections query-able --- include/podio/CollectionBase.h | 2 ++ include/podio/EDMDefinitionRegistry.h | 7 ++++++- include/podio/UserDataCollection.h | 5 +++++ python/templates/Collection.cc.jinja2 | 5 +++++ python/templates/Collection.h.jinja2 | 2 ++ python/templates/DatamodelDefinition.h.jinja2 | 18 ++++++++++++++++-- src/EDMDefinitionRegistry.cc | 1 + 7 files changed, 37 insertions(+), 3 deletions(-) diff --git a/include/podio/CollectionBase.h b/include/podio/CollectionBase.h index 670291ba3..16efd8aa0 100644 --- a/include/podio/CollectionBase.h +++ b/include/podio/CollectionBase.h @@ -76,6 +76,8 @@ class CollectionBase { /// print this collection to the passed stream virtual void print(std::ostream& os = std::cout, bool flush = true) const = 0; + + virtual size_t getDefinitionRegistryIndex() const = 0; }; } // namespace podio diff --git a/include/podio/EDMDefinitionRegistry.h b/include/podio/EDMDefinitionRegistry.h index bc3e42d72..73c971a23 100644 --- a/include/podio/EDMDefinitionRegistry.h +++ b/include/podio/EDMDefinitionRegistry.h @@ -16,6 +16,11 @@ class EDMDefinitionRegistry { EDMDefinitionRegistry(EDMDefinitionRegistry&&) = delete; EDMDefinitionRegistry& operator=(const EDMDefinitionRegistry&&) = delete; + /// Dedicated index value for collections that don't need a definition (e.g. UserDataCollection) + static constexpr size_t NoDefinitionNecessary = -1; + /// Dedicated index value for error checking, used to default init the generated RegistryIndex + static constexpr size_t NoDefinitionAvailable = -2; + /** * Get the definition (in JSON format) of the EDM with the given edm_name. If * no EDM under the given name can be found, an empty model definition is @@ -38,7 +43,7 @@ class EDMDefinitionRegistry { private: EDMDefinitionRegistry() = default; - + /// The stored definitions std::vector> m_definitions{}; }; } // namespace podio diff --git a/include/podio/UserDataCollection.h b/include/podio/UserDataCollection.h index 2365c5094..81970c41f 100644 --- a/include/podio/UserDataCollection.h +++ b/include/podio/UserDataCollection.h @@ -3,6 +3,7 @@ #include "podio/CollectionBase.h" #include "podio/CollectionBuffers.h" +#include "podio/EDMDefinitionRegistry.h" #include "podio/utilities/TypeHelpers.h" #include @@ -172,6 +173,10 @@ class UserDataCollection : public CollectionBase { } } + size_t getDefinitionRegistryIndex() const override { + return EDMDefinitionRegistry::NoDefinitionNecessary; + } + // ----- some wrapers for std::vector and access to the complete std::vector (if really needed) typename std::vector::iterator begin() { diff --git a/python/templates/Collection.cc.jinja2 b/python/templates/Collection.cc.jinja2 index c265c29a7..2db4f6005 100644 --- a/python/templates/Collection.cc.jinja2 +++ b/python/templates/Collection.cc.jinja2 @@ -4,6 +4,7 @@ // AUTOMATICALLY GENERATED FILE - DO NOT EDIT #include "{{ incfolder }}{{ class.bare_type }}Collection.h" +#include "{{ incfolder }}DatamodelDefinition.h" {% for include in includes_coll_cc %} {{ include }} @@ -178,6 +179,10 @@ podio::CollectionReadBuffers {{ collection_type }}::createBuffers() /*const*/ { {{ macros.vectorized_access(class, member) }} {% endfor %} +size_t {{ collection_type }}::getDefinitionRegistryIndex() const { + return {{ package_name }}::meta::RegistryIndex::value(); +} + #ifdef PODIO_JSON_OUTPUT void to_json(nlohmann::json& j, const {{ collection_type }}& collection) { j = nlohmann::json::array(); diff --git a/python/templates/Collection.h.jinja2 b/python/templates/Collection.h.jinja2 index 049ecff79..6e42ff76e 100644 --- a/python/templates/Collection.h.jinja2 +++ b/python/templates/Collection.h.jinja2 @@ -130,6 +130,8 @@ public: return m_isValid; } + size_t getDefinitionRegistryIndex() const final; + // support for the iterator protocol iterator begin() { return iterator(0, &m_storage.entries); diff --git a/python/templates/DatamodelDefinition.h.jinja2 b/python/templates/DatamodelDefinition.h.jinja2 index 2b56a5b87..3ef6694aa 100644 --- a/python/templates/DatamodelDefinition.h.jinja2 +++ b/python/templates/DatamodelDefinition.h.jinja2 @@ -9,8 +9,22 @@ namespace {{ package_name }}::meta { static constexpr auto {{ package_name }}__JSONDefinition = R"EDMDEFINITION({{ edm_definition }})EDMDEFINITION"; /** - * The index of the definition in the registry + * The helper class that takes care of registering the EDM definition to the + * EDMDefinitionRegistry and to provide the index in that registry. + * + * Implemented as a singleton mainly to ensure only a single registration of + * each EDM, during the constructor */ -static const auto definitionRegistryIndex = podio::EDMDefinitionRegistry::instance().registerEDM("{{ package_edm_edm_name }}", {{ package_name }}__JSONDefinition); +class RegistryIndex { +public: + static size_t value() { + static auto index = RegistryIndex(podio::EDMDefinitionRegistry::instance().registerEDM("{{ package_name }}", {{ package_name }}__JSONDefinition)); + return index.m_value; + } + +private: + RegistryIndex(size_t v) : m_value(v) {} + size_t m_value{podio::EDMDefinitionRegistry::NoDefinitionAvailable}; +}; } // namespace {{ package_name }}::meta diff --git a/src/EDMDefinitionRegistry.cc b/src/EDMDefinitionRegistry.cc index b1048ead1..589d55891 100644 --- a/src/EDMDefinitionRegistry.cc +++ b/src/EDMDefinitionRegistry.cc @@ -1,6 +1,7 @@ #include "podio/EDMDefinitionRegistry.h" #include +#include #include #include From f24e18835744a554e0a8e64efdf362abc32f4513 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Fri, 16 Dec 2022 19:38:33 +0100 Subject: [PATCH 05/19] Write definition and read it back afterwards (ROOT) --- include/podio/EDMDefinitionRegistry.h | 6 ++++++ include/podio/ROOTFrameReader.h | 4 ++++ include/podio/ROOTFrameWriter.h | 10 ++++++++++ include/podio/SIOFrameReader.h | 4 ++++ src/EDMDefinitionRegistry.cc | 8 ++++++++ src/ROOTFrameReader.cc | 19 ++++++++++++++++++- src/ROOTFrameWriter.cc | 22 ++++++++++++++++++++++ src/rootUtils.h | 7 +++++++ src/selection.xml | 1 + tests/read_frame.h | 2 ++ 10 files changed, 82 insertions(+), 1 deletion(-) diff --git a/include/podio/EDMDefinitionRegistry.h b/include/podio/EDMDefinitionRegistry.h index 73c971a23..b1e377085 100644 --- a/include/podio/EDMDefinitionRegistry.h +++ b/include/podio/EDMDefinitionRegistry.h @@ -34,6 +34,12 @@ class EDMDefinitionRegistry { */ const std::string_view getDefinition(size_t index) const; + /** + * Get the name of the EDM that is stored under the given index. If no EDM is + * found under the given index, an empty string is returned + */ + const std::string& getEDMName(size_t index) const; + /** * Register a definition and return the index in the registry. If a definition * already exists under the given name, then the index of the existing diff --git a/include/podio/ROOTFrameReader.h b/include/podio/ROOTFrameReader.h index 1850a0b02..b8710b064 100644 --- a/include/podio/ROOTFrameReader.h +++ b/include/podio/ROOTFrameReader.h @@ -79,6 +79,8 @@ class ROOTFrameReader { /// Get the names of all the availalable Frame categories in the current file(s) std::vector getAvailableCategories() const; + const std::string_view getEDMDefinition(const std::string& edmName) const; + private: /** * Helper struct to group together all the necessary state to read / process a @@ -131,6 +133,8 @@ class ROOTFrameReader { std::unordered_map m_categories{}; ///< All categories std::vector m_availCategories{}; ///< All available categories from this file + std::vector> m_availEDMDefs{}; + podio::version::Version m_fileVersion{0, 0, 0}; }; diff --git a/include/podio/ROOTFrameWriter.h b/include/podio/ROOTFrameWriter.h index 9428ed929..5fa2a06cd 100644 --- a/include/podio/ROOTFrameWriter.h +++ b/include/podio/ROOTFrameWriter.h @@ -7,6 +7,7 @@ #include "TFile.h" #include +#include #include #include #include @@ -78,8 +79,17 @@ class ROOTFrameWriter { const std::vector& collections, /*const*/ podio::GenericParameters* parameters); + /// Register the EDM definition of this collection to be written to file + void registerEDMDef(const podio::CollectionBase* coll, const std::string& name); + std::unique_ptr m_file{nullptr}; ///< The storage file std::unordered_map m_categories{}; ///< All categories + + /** + * The indices in the EDM definition registry for all datamodels of which at + * least one collection was written. + */ + std::set m_edmDefRegistryIdcs{}; }; } // namespace podio diff --git a/include/podio/SIOFrameReader.h b/include/podio/SIOFrameReader.h index d7a2c5e8c..5b1adcd32 100644 --- a/include/podio/SIOFrameReader.h +++ b/include/podio/SIOFrameReader.h @@ -53,6 +53,10 @@ class SIOFrameReader { /// Get the names of all the availalable Frame categories in the current file(s) std::vector getAvailableCategories() const; + std::string_view getEDMDefinition(const std::string&) const { + return "{}"; + } + private: void readPodioHeader(); diff --git a/src/EDMDefinitionRegistry.cc b/src/EDMDefinitionRegistry.cc index 589d55891..f111b0644 100644 --- a/src/EDMDefinitionRegistry.cc +++ b/src/EDMDefinitionRegistry.cc @@ -42,4 +42,12 @@ const std::string_view EDMDefinitionRegistry::getDefinition(size_t index) const return m_definitions[index].second; } +const std::string& EDMDefinitionRegistry::getEDMName(size_t index) const { + if (index >= m_definitions.size()) { + static const std::string emptyName = ""; + return emptyName; + } + return m_definitions[index].first; +} + } // namespace podio diff --git a/src/ROOTFrameReader.cc b/src/ROOTFrameReader.cc index e3d6c6aba..421f13e56 100644 --- a/src/ROOTFrameReader.cc +++ b/src/ROOTFrameReader.cc @@ -179,6 +179,7 @@ std::vector getAvailableCategories(TChain* metaChain) { auto* branches = metaChain->GetListOfBranches(); std::vector brNames; brNames.reserve(branches->GetEntries()); + for (int i = 0; i < branches->GetEntries(); ++i) { const std::string name = branches->At(i)->GetName(); const auto fUnder = name.find("___"); @@ -189,7 +190,6 @@ std::vector getAvailableCategories(TChain* metaChain) { std::sort(brNames.begin(), brNames.end()); brNames.erase(std::unique(brNames.begin(), brNames.end()), brNames.end()); - return brNames; } @@ -217,6 +217,12 @@ void ROOTFrameReader::openFiles(const std::vector& filenames) { m_fileVersion = versionPtr ? *versionPtr : podio::version::Version{0, 0, 0}; delete versionPtr; + auto* edmDefs = &m_availEDMDefs; + if (auto* edmDefBranch = root_utils::getBranch(m_metaChain.get(), root_utils::edmDefBranchName)) { + edmDefBranch->SetAddress(&edmDefs); + edmDefBranch->GetEntry(0); + } + // Do some work up front for setting up categories and setup all the chains // and record the available categories. The rest of the setup follows on // demand when the category is first read @@ -229,6 +235,17 @@ void ROOTFrameReader::openFiles(const std::vector& filenames) { } } +const std::string_view ROOTFrameReader::getEDMDefinition(const std::string& edmName) const { + const auto it = std::find_if(m_availEDMDefs.cbegin(), m_availEDMDefs.cend(), + [&edmName](const auto& entry) { return std::get<0>(entry) == edmName; }); + + if (it != m_availEDMDefs.cend()) { + return std::get<1>(*it); + } + + return "{}"; +} + unsigned ROOTFrameReader::getEntries(const std::string& name) const { if (auto it = m_categories.find(name); it != m_categories.end()) { return it->second.chain->GetEntries(); diff --git a/src/ROOTFrameWriter.cc b/src/ROOTFrameWriter.cc index d98d6763a..b9438e5eb 100644 --- a/src/ROOTFrameWriter.cc +++ b/src/ROOTFrameWriter.cc @@ -1,5 +1,6 @@ #include "podio/ROOTFrameWriter.h" #include "podio/CollectionBase.h" +#include "podio/EDMDefinitionRegistry.h" #include "podio/Frame.h" #include "podio/GenericParameters.h" #include "podio/podioVersion.h" @@ -35,6 +36,8 @@ void ROOTFrameWriter::writeFrame(const podio::Frame& frame, const std::string& c for (const auto& name : catInfo.collsToWrite) { auto* coll = frame.getCollectionForWrite(name); collections.emplace_back(name, const_cast(coll)); + + registerEDMDef(coll, name); } // We will at least have a parameters branch, even if there are no @@ -58,6 +61,17 @@ ROOTFrameWriter::CategoryInfo& ROOTFrameWriter::getCategoryInfo(const std::strin return it->second; } +void ROOTFrameWriter::registerEDMDef(const podio::CollectionBase* coll, const std::string& name) { + const auto edmIndex = coll->getDefinitionRegistryIndex(); + if (edmIndex == EDMDefinitionRegistry::NoDefinitionAvailable) { + std::cerr << "No EDM definition available for collection " << name << std::endl; + } else { + if (edmIndex != EDMDefinitionRegistry::NoDefinitionNecessary) { + m_edmDefRegistryIdcs.insert(edmIndex); + } + } +} + void ROOTFrameWriter::initBranches(CategoryInfo& catInfo, const std::vector& collections, /*const*/ podio::GenericParameters& parameters) { catInfo.branches.reserve(collections.size() + 1); // collections + parameters @@ -129,6 +143,14 @@ void ROOTFrameWriter::finish() { auto podioVersion = podio::version::build_version; metaTree->Branch(root_utils::versionBranchName, &podioVersion); + std::vector> edmDefinitions; + edmDefinitions.reserve(m_edmDefRegistryIdcs.size()); + for (const auto& index : m_edmDefRegistryIdcs) { + const auto& edmRegistry = podio::EDMDefinitionRegistry::instance(); + edmDefinitions.emplace_back(edmRegistry.getEDMName(index), edmRegistry.getDefinition(index)); + } + metaTree->Branch(root_utils::edmDefBranchName, &edmDefinitions); + metaTree->Fill(); m_file->Write(); diff --git a/src/rootUtils.h b/src/rootUtils.h index 5bce3d702..215c7fea6 100644 --- a/src/rootUtils.h +++ b/src/rootUtils.h @@ -7,6 +7,7 @@ #include "podio/CollectionIDTable.h" #include "TBranch.h" +#include "TChain.h" #include "TClass.h" #include "TTree.h" @@ -35,6 +36,12 @@ constexpr static auto paramBranchName = "PARAMETERS"; */ constexpr static auto versionBranchName = "PodioBuildVersion"; +/** + * The name of the branch in which all the EDM names and their definitions are + * stored in the meta data tree. + */ +constexpr static auto edmDefBranchName = "EDMDefinitions"; + /** * Name of the branch for storing the idTable for a given category in the meta * data tree diff --git a/src/selection.xml b/src/selection.xml index 3c0be36e3..d198bfab6 100644 --- a/src/selection.xml +++ b/src/selection.xml @@ -15,6 +15,7 @@ + diff --git a/tests/read_frame.h b/tests/read_frame.h index 76d4f0057..0f8a724f2 100644 --- a/tests/read_frame.h +++ b/tests/read_frame.h @@ -145,6 +145,8 @@ int read_frames(const std::string& filename) { } } + std::cout << reader.getEDMDefinition("datamodel") << std::endl; + return 0; } From ebd9af621a75307df7e2b4a32d2580ff0f090b63 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Tue, 20 Dec 2022 17:35:05 +0100 Subject: [PATCH 06/19] Read and write EDM definitions in SIO --- include/podio/SIOBlock.h | 56 ++++++++++++++++++++++++++ include/podio/SIOFrameReader.h | 9 +++-- include/podio/SIOFrameWriter.h | 9 +++++ include/podio/utilities/TypeHelpers.h | 58 +++++++++++++++++++++++++++ src/SIOBlock.cc | 38 ++++-------------- src/SIOFrameReader.cc | 38 +++++++++++++++++- src/SIOFrameWriter.cc | 24 +++++++++++ src/sioUtils.h | 1 + tests/read_frame.h | 2 - 9 files changed, 199 insertions(+), 36 deletions(-) diff --git a/include/podio/SIOBlock.h b/include/podio/SIOBlock.h index 95e3de27f..36ee5b232 100644 --- a/include/podio/SIOBlock.h +++ b/include/podio/SIOBlock.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include namespace podio { @@ -26,6 +28,34 @@ void handlePODDataSIO(devT& device, PODData* data, size_t size) { device.data(dataPtr, count); } +/// Write anything that iterates like an std::map +template +void writeMapLike(sio::write_device& device, const MapLikeT& map) { + device.data((int)map.size()); + for (const auto& [key, value] : map) { + device.data(key); + device.data(value); + } +} + +/// Read anything that iterates like an std::map +template +void readMapLike(sio::read_device& device, MapLikeT& map) { + int size; + device.data(size); + while (size--) { + detail::GetKeyType key; + device.data(key); + detail::GetMappedType value; + device.data(value); + if constexpr (podio::detail::isVector) { + map.emplace_back(std::move(key), std::move(value)); + } else { + map.emplace(std::move(key), std::move(value)); + } + } +} + /// Base class for sio::block handlers used with PODIO class SIOBlock : public sio::block { @@ -141,6 +171,29 @@ class SIOEventMetaDataBlock : public sio::block { podio::GenericParameters* metadata{nullptr}; }; +/** + * A block to serialize anything that behaves similar in iterating as a + * map, e.g. vector>, which is what is used + * internally to represent the data to be written. + */ +template +struct SIOMapBlock : public sio::block { + SIOMapBlock() : sio::block("SIOMapBlock", sio::version::encode_version(0, 1)) { + } + + SIOMapBlock(const SIOMapBlock&) = delete; + SIOMapBlock& operator=(const SIOMapBlock&) = delete; + + void read(sio::read_device& device, sio::version_type) override { + readMapLike(device, mapData); + } + void write(sio::write_device& device) override { + writeMapLike(device, mapData); + } + + std::vector> mapData{}; +}; + /** * A block for handling the run and collection meta data */ @@ -219,6 +272,9 @@ namespace sio_helpers { /// The name of the TOCRecord static constexpr const char* SIOTocRecordName = "podio_SIO_TOC_Record"; + /// The name of the record containing the EDM definitions in json format + static constexpr const char* SIOEDMDefinitionName = "podio_SIO_EDMDefinitions"; + // should hopefully be enough for all practical purposes using position_type = uint32_t; } // namespace sio_helpers diff --git a/include/podio/SIOFrameReader.h b/include/podio/SIOFrameReader.h index 5b1adcd32..2511bae09 100644 --- a/include/podio/SIOFrameReader.h +++ b/include/podio/SIOFrameReader.h @@ -53,9 +53,7 @@ class SIOFrameReader { /// Get the names of all the availalable Frame categories in the current file(s) std::vector getAvailableCategories() const; - std::string_view getEDMDefinition(const std::string&) const { - return "{}"; - } + const std::string_view getEDMDefinition(const std::string& edmName) const; private: void readPodioHeader(); @@ -63,6 +61,8 @@ class SIOFrameReader { /// read the TOC record bool readFileTOCRecord(); + void readEDMDefinitions(); + sio::ifstream m_stream{}; ///< The stream from which we read /// Count how many times each an entry of this name has been read already @@ -72,6 +72,9 @@ class SIOFrameReader { SIOFileTOCRecord m_tocRecord{}; /// The podio version that has been used to write the file podio::version::Version m_fileVersion{0}; + + /// The available edm definitions in this file + std::vector> m_availEDMDefs{}; }; } // namespace podio diff --git a/include/podio/SIOFrameWriter.h b/include/podio/SIOFrameWriter.h index 1ccc7a2e8..fb42d576e 100644 --- a/include/podio/SIOFrameWriter.h +++ b/include/podio/SIOFrameWriter.h @@ -5,6 +5,7 @@ #include +#include #include #include #include @@ -33,8 +34,16 @@ class SIOFrameWriter { void finish(); private: + void registerEDMDef(const podio::CollectionBase* coll, const std::string& name); + sio::ofstream m_stream{}; ///< The output file stream SIOFileTOCRecord m_tocRecord{}; ///< The "table of contents" of the written file + + /** + * The indices in the EDM definition registry for all datamodels of which at + * least one collection was written. + */ + std::set m_edmDefRegistryIdcs{}; }; } // namespace podio diff --git a/include/podio/utilities/TypeHelpers.h b/include/podio/utilities/TypeHelpers.h index b351e2118..74d1a4d28 100644 --- a/include/podio/utilities/TypeHelpers.h +++ b/include/podio/utilities/TypeHelpers.h @@ -1,9 +1,11 @@ #ifndef PODIO_UTILITIES_TYPEHELPERS_H #define PODIO_UTILITIES_TYPEHELPERS_H +#include #include #include #include +#include #include namespace podio { @@ -100,6 +102,62 @@ namespace detail { template static constexpr bool isVector = IsVectorHelper::value; + /** + * Helper struct to detect whether a type is a std::map or std::unordered_map + */ + template + struct IsMapHelper : std::false_type {}; + + template + struct IsMapHelper> : std::true_type {}; + + template + struct IsMapHelper> : std::true_type {}; + + /** + * Alias template for deciding whether the passed type T is a map or + * unordered_map + */ + template + static constexpr bool isMap = IsMapHelper::value; + + /** + * Helper struct to homogenize the (type) access for things that behave like + * maps, e.g. vectors of pairs (and obviously maps). + * + * NOTE: This is not SFINAE friendly. + */ + template >, + typename IsVector = std::bool_constant && (std::tuple_size() == 2)>> + struct MapLikeTypeHelper {}; + + /** + * Specialization for actual maps + */ + template + struct MapLikeTypeHelper, std::bool_constant> { + using key_type = typename T::key_type; + using mapped_type = typename T::mapped_type; + }; + + /** + * Specialization for vector of pairs / tuples (of size 2) + */ + template + struct MapLikeTypeHelper, std::bool_constant> { + using key_type = typename std::tuple_element<0, typename T::value_type>::type; + using mapped_type = typename std::tuple_element<1, typename T::value_type>::type; + }; + + /** + * Type aliases for easier usage in actual code + */ + template + using GetKeyType = typename MapLikeTypeHelper::key_type; + + template + using GetMappedType = typename MapLikeTypeHelper::mapped_type; + } // namespace detail // forward declaration to be able to use it below diff --git a/src/SIOBlock.cc b/src/SIOBlock.cc index 6c7a95ba6..097e79224 100644 --- a/src/SIOBlock.cc +++ b/src/SIOBlock.cc @@ -49,41 +49,19 @@ void SIOCollectionIDTableBlock::write(sio::write_device& device) { device.data(_isSubsetColl); } -template -void writeParamMap(sio::write_device& device, const GenericParameters::MapType& map) { - device.data((int)map.size()); - for (const auto& [key, value] : map) { - device.data(key); - device.data(value); - } -} - -template -void readParamMap(sio::read_device& device, GenericParameters::MapType& map) { - int size; - device.data(size); - while (size--) { - std::string key; - device.data(key); - std::vector values; - device.data(values); - map.emplace(std::move(key), std::move(values)); - } -} - void writeGenericParameters(sio::write_device& device, const GenericParameters& params) { - writeParamMap(device, params.getIntMap()); - writeParamMap(device, params.getFloatMap()); - writeParamMap(device, params.getStringMap()); - writeParamMap(device, params.getDoubleMap()); + writeMapLike(device, params.getIntMap()); + writeMapLike(device, params.getFloatMap()); + writeMapLike(device, params.getStringMap()); + writeMapLike(device, params.getDoubleMap()); } void readGenericParameters(sio::read_device& device, GenericParameters& params, sio::version_type version) { - readParamMap(device, params.getIntMap()); - readParamMap(device, params.getFloatMap()); - readParamMap(device, params.getStringMap()); + readMapLike(device, params.getIntMap()); + readMapLike(device, params.getFloatMap()); + readMapLike(device, params.getStringMap()); if (version >= sio::version::encode_version(0, 2)) { - readParamMap(device, params.getDoubleMap()); + readMapLike(device, params.getDoubleMap()); } } diff --git a/src/SIOFrameReader.cc b/src/SIOFrameReader.cc index 47f5ec082..f494d1f9b 100644 --- a/src/SIOFrameReader.cc +++ b/src/SIOFrameReader.cc @@ -6,6 +6,7 @@ #include #include +#include #include namespace podio { @@ -23,6 +24,7 @@ void SIOFrameReader::openFile(const std::string& filename) { // NOTE: reading TOC record first because that jumps back to the start of the file! readFileTOCRecord(); readPodioHeader(); + readEDMDefinitions(); // Potentially could do this lazily } std::unique_ptr SIOFrameReader::readNextEntry(const std::string& name) { @@ -54,13 +56,29 @@ std::unique_ptr SIOFrameReader::readEntry(const std::string& name, } std::vector SIOFrameReader::getAvailableCategories() const { - return m_tocRecord.getRecordNames(); + // Filter the availalbe records from the TOC to remove records that are + // stored, but use reserved record names for podio meta data + auto recordNames = m_tocRecord.getRecordNames(); + recordNames.erase(std::remove_if(recordNames.begin(), recordNames.end(), + [](const auto& elem) { return elem == sio_helpers::SIOEDMDefinitionName; }), + recordNames.end()); + return recordNames; } unsigned SIOFrameReader::getEntries(const std::string& name) const { return m_tocRecord.getNRecords(name); } +const std::string_view SIOFrameReader::getEDMDefinition(const std::string& edmName) const { + const auto it = std::find_if(m_availEDMDefs.cbegin(), m_availEDMDefs.cend(), + [&edmName](const auto& entry) { return std::get<0>(entry) == edmName; }); + if (it != m_availEDMDefs.cend()) { + return std::get<1>(*it); + } + + return "{}"; +} + bool SIOFrameReader::readFileTOCRecord() { // Check if there is a dedicated marker at the end of the file that tells us // where the TOC actually starts @@ -101,4 +119,22 @@ void SIOFrameReader::readPodioHeader() { m_fileVersion = static_cast(blocks[0].get())->version; } +void SIOFrameReader::readEDMDefinitions() { + const auto recordPos = m_tocRecord.getPosition(sio_helpers::SIOEDMDefinitionName); + if (recordPos == 0) { + // No EDM definitions found + return; + } + m_stream.seekg(recordPos); + + const auto& [buffer, _] = sio_utils::readRecord(m_stream); + + sio::block_list blocks; + blocks.emplace_back(std::make_shared>()); + sio::api::read_blocks(buffer.span(), blocks); + + auto edmDefs = static_cast*>(blocks[0].get()); + m_availEDMDefs = std::move(edmDefs->mapData); +} + } // namespace podio diff --git a/src/SIOFrameWriter.cc b/src/SIOFrameWriter.cc index f33bdbccc..2ff156442 100644 --- a/src/SIOFrameWriter.cc +++ b/src/SIOFrameWriter.cc @@ -1,6 +1,7 @@ #include "podio/SIOFrameWriter.h" #include "podio/CollectionBase.h" #include "podio/CollectionIDTable.h" +#include "podio/EDMDefinitionRegistry.h" #include "podio/Frame.h" #include "podio/GenericParameters.h" #include "podio/SIOBlock.h" @@ -8,6 +9,7 @@ #include "sioUtils.h" #include +#include namespace podio { @@ -29,12 +31,24 @@ void SIOFrameWriter::writeFrame(const podio::Frame& frame, const std::string& ca writeFrame(frame, category, frame.getAvailableCollections()); } +void SIOFrameWriter::registerEDMDef(const podio::CollectionBase* coll, const std::string& name) { + const auto edmIndex = coll->getDefinitionRegistryIndex(); + if (edmIndex == EDMDefinitionRegistry::NoDefinitionAvailable) { + std::cerr << "No EDM definition available for collection " << name << std::endl; + } else { + if (edmIndex != EDMDefinitionRegistry::NoDefinitionNecessary) { + m_edmDefRegistryIdcs.insert(edmIndex); + } + } +} + void SIOFrameWriter::writeFrame(const podio::Frame& frame, const std::string& category, const std::vector& collsToWrite) { std::vector collections; collections.reserve(collsToWrite.size()); for (const auto& name : collsToWrite) { collections.emplace_back(name, frame.getCollectionForWrite(name)); + registerEDMDef(collections.back().second, name); } // Write necessary metadata and the actual data into two different records. @@ -49,7 +63,17 @@ void SIOFrameWriter::writeFrame(const podio::Frame& frame, const std::string& ca } void SIOFrameWriter::finish() { + auto edmDefMap = std::make_shared>(); + const auto& edmRegistry = podio::EDMDefinitionRegistry::instance(); + for (const auto& index : m_edmDefRegistryIdcs) { + edmDefMap->mapData.emplace_back(edmRegistry.getEDMName(index), edmRegistry.getDefinition(index)); + } + sio::block_list blocks; + blocks.push_back(edmDefMap); + m_tocRecord.addRecord(sio_helpers::SIOEDMDefinitionName, sio_utils::writeRecord(blocks, "EDMDefinitions", m_stream)); + + blocks.clear(); blocks.emplace_back(std::make_shared(&m_tocRecord)); auto tocStartPos = sio_utils::writeRecord(blocks, sio_helpers::SIOTocRecordName, m_stream); diff --git a/src/sioUtils.h b/src/sioUtils.h index 6e340d8fa..204867eaf 100644 --- a/src/sioUtils.h +++ b/src/sioUtils.h @@ -9,6 +9,7 @@ #include #include +#include #include namespace podio { diff --git a/tests/read_frame.h b/tests/read_frame.h index 0f8a724f2..76d4f0057 100644 --- a/tests/read_frame.h +++ b/tests/read_frame.h @@ -145,8 +145,6 @@ int read_frames(const std::string& filename) { } } - std::cout << reader.getEDMDefinition("datamodel") << std::endl; - return 0; } From 1f58965defe4e96f221cd41df4b3d706cbc90150 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Wed, 21 Dec 2022 16:48:14 +0100 Subject: [PATCH 07/19] Refactor EDM defintion I/O functionality for less code duplication --- include/podio/ROOTFrameReader.h | 7 +-- include/podio/ROOTFrameWriter.h | 13 +---- include/podio/SIOBlock.h | 3 ++ include/podio/SIOFrameReader.h | 8 +--- include/podio/SIOFrameWriter.h | 12 +---- .../podio/utilities/EDMRegistryIOHelpers.h | 48 +++++++++++++++++++ src/CMakeLists.txt | 2 + src/EDMRegistryIOHelpers.cc | 38 +++++++++++++++ src/ROOTFrameReader.cc | 11 ----- src/ROOTFrameWriter.cc | 19 +------- src/SIOFrameReader.cc | 10 ---- src/SIOFrameWriter.cc | 17 +------ 12 files changed, 101 insertions(+), 87 deletions(-) create mode 100644 include/podio/utilities/EDMRegistryIOHelpers.h create mode 100644 src/EDMRegistryIOHelpers.cc diff --git a/include/podio/ROOTFrameReader.h b/include/podio/ROOTFrameReader.h index b8710b064..0e8edb89c 100644 --- a/include/podio/ROOTFrameReader.h +++ b/include/podio/ROOTFrameReader.h @@ -4,6 +4,7 @@ #include "podio/CollectionBranches.h" #include "podio/ROOTFrameData.h" #include "podio/podioVersion.h" +#include "podio/utilities/EDMRegistryIOHelpers.h" #include "TChain.h" @@ -40,7 +41,7 @@ struct CollectionReadBuffers; * This class has the function to read available data from disk * and to prepare collections and buffers. **/ -class ROOTFrameReader { +class ROOTFrameReader : public EDMDefinitionHolder { public: ROOTFrameReader() = default; @@ -79,8 +80,6 @@ class ROOTFrameReader { /// Get the names of all the availalable Frame categories in the current file(s) std::vector getAvailableCategories() const; - const std::string_view getEDMDefinition(const std::string& edmName) const; - private: /** * Helper struct to group together all the necessary state to read / process a @@ -133,8 +132,6 @@ class ROOTFrameReader { std::unordered_map m_categories{}; ///< All categories std::vector m_availCategories{}; ///< All available categories from this file - std::vector> m_availEDMDefs{}; - podio::version::Version m_fileVersion{0, 0, 0}; }; diff --git a/include/podio/ROOTFrameWriter.h b/include/podio/ROOTFrameWriter.h index 5fa2a06cd..6ef1dc502 100644 --- a/include/podio/ROOTFrameWriter.h +++ b/include/podio/ROOTFrameWriter.h @@ -3,11 +3,11 @@ #include "podio/CollectionBranches.h" #include "podio/CollectionIDTable.h" +#include "podio/utilities/EDMRegistryIOHelpers.h" #include "TFile.h" #include -#include #include #include #include @@ -21,7 +21,7 @@ class Frame; class CollectionBase; class GenericParameters; -class ROOTFrameWriter { +class ROOTFrameWriter : EDMDefinitionCollector { public: ROOTFrameWriter(const std::string& filename); ~ROOTFrameWriter() = default; @@ -79,17 +79,8 @@ class ROOTFrameWriter { const std::vector& collections, /*const*/ podio::GenericParameters* parameters); - /// Register the EDM definition of this collection to be written to file - void registerEDMDef(const podio::CollectionBase* coll, const std::string& name); - std::unique_ptr m_file{nullptr}; ///< The storage file std::unordered_map m_categories{}; ///< All categories - - /** - * The indices in the EDM definition registry for all datamodels of which at - * least one collection was written. - */ - std::set m_edmDefRegistryIdcs{}; }; } // namespace podio diff --git a/include/podio/SIOBlock.h b/include/podio/SIOBlock.h index 36ee5b232..3e02561b8 100644 --- a/include/podio/SIOBlock.h +++ b/include/podio/SIOBlock.h @@ -180,6 +180,9 @@ template struct SIOMapBlock : public sio::block { SIOMapBlock() : sio::block("SIOMapBlock", sio::version::encode_version(0, 1)) { } + SIOMapBlock(std::vector>&& data) : + sio::block("SIOMapBlock", sio::version::encode_version(0, 1)), mapData(std::move(data)) { + } SIOMapBlock(const SIOMapBlock&) = delete; SIOMapBlock& operator=(const SIOMapBlock&) = delete; diff --git a/include/podio/SIOFrameReader.h b/include/podio/SIOFrameReader.h index 2511bae09..3892a5014 100644 --- a/include/podio/SIOFrameReader.h +++ b/include/podio/SIOFrameReader.h @@ -4,6 +4,7 @@ #include "podio/SIOBlock.h" #include "podio/SIOFrameData.h" #include "podio/podioVersion.h" +#include "podio/utilities/EDMRegistryIOHelpers.h" #include @@ -16,7 +17,7 @@ namespace podio { class CollectionIDTable; -class SIOFrameReader { +class SIOFrameReader : public EDMDefinitionHolder { public: SIOFrameReader(); @@ -53,8 +54,6 @@ class SIOFrameReader { /// Get the names of all the availalable Frame categories in the current file(s) std::vector getAvailableCategories() const; - const std::string_view getEDMDefinition(const std::string& edmName) const; - private: void readPodioHeader(); @@ -72,9 +71,6 @@ class SIOFrameReader { SIOFileTOCRecord m_tocRecord{}; /// The podio version that has been used to write the file podio::version::Version m_fileVersion{0}; - - /// The available edm definitions in this file - std::vector> m_availEDMDefs{}; }; } // namespace podio diff --git a/include/podio/SIOFrameWriter.h b/include/podio/SIOFrameWriter.h index fb42d576e..37fc3954b 100644 --- a/include/podio/SIOFrameWriter.h +++ b/include/podio/SIOFrameWriter.h @@ -2,10 +2,10 @@ #define PODIO_SIOFRAMEWRITER_H #include "podio/SIOBlock.h" +#include "podio/utilities/EDMRegistryIOHelpers.h" #include -#include #include #include #include @@ -14,7 +14,7 @@ namespace podio { class Frame; -class SIOFrameWriter { +class SIOFrameWriter : EDMDefinitionCollector { public: SIOFrameWriter(const std::string& filename); ~SIOFrameWriter() = default; @@ -34,16 +34,8 @@ class SIOFrameWriter { void finish(); private: - void registerEDMDef(const podio::CollectionBase* coll, const std::string& name); - sio::ofstream m_stream{}; ///< The output file stream SIOFileTOCRecord m_tocRecord{}; ///< The "table of contents" of the written file - - /** - * The indices in the EDM definition registry for all datamodels of which at - * least one collection was written. - */ - std::set m_edmDefRegistryIdcs{}; }; } // namespace podio diff --git a/include/podio/utilities/EDMRegistryIOHelpers.h b/include/podio/utilities/EDMRegistryIOHelpers.h new file mode 100644 index 000000000..f8f8a8f25 --- /dev/null +++ b/include/podio/utilities/EDMRegistryIOHelpers.h @@ -0,0 +1,48 @@ +#ifndef PODIO_UTILITIES_EDMREGISTRYIOHELPERS_H +#define PODIO_UTILITIES_EDMREGISTRYIOHELPERS_H + +#include "podio/CollectionBase.h" +#include "podio/EDMDefinitionRegistry.h" + +#include +#include +#include +#include + +namespace podio { + +/** + * Helper class (mixin) to collect the EDM (JSON) definitions that should be + * written. + */ +class EDMDefinitionCollector { +public: + /// Register the EDM where this collection is from to be written + void registerEDMDef(const podio::CollectionBase* coll, const std::string& name); + + /// Get all the names and JSON definitions that need to be written + std::vector> getEDMDefinitionsToWrite() const; + +private: + std::set m_edmDefRegistryIdcs{}; ///< The indices in the EDM definition registry that need to be written +}; + +/** + * Helper class (mixin) to hold and provide the EDM (JSON) definitions for + * reader classes. + */ +class EDMDefinitionHolder { +public: + /** + * Get the EDM definition for the given EDM name. Returns an empty model + * definition if no model is stored under the given name. + */ + const std::string_view getEDMDefinition(const std::string& edmName) const; + +protected: + std::vector> m_availEDMDefs{}; +}; + +} // namespace podio + +#endif // PODIO_UTILITIES_EDMREGISTRYIOHELPERS_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 956903316..e40ed9e61 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -50,6 +50,7 @@ SET(core_sources ASCIIWriter.cc EventStore.cc EDMDefinitionRegistry.cc + EDMRegistryIOHelpers.cc ) SET(core_headers @@ -62,6 +63,7 @@ SET(core_headers ${CMAKE_SOURCE_DIR}/include/podio/UserDataCollection.h ${CMAKE_SOURCE_DIR}/include/podio/podioVersion.h ${CMAKE_SOURCE_DIR}/include/podio/EDMDefinitionRegistry.h + ${CMAKE_SOURCE_DIR}/include/podio/utilities/EDMRegistryIOHelpers.h ) PODIO_ADD_LIB_AND_DICT(podio "${core_headers}" "${core_sources}" selection.xml) diff --git a/src/EDMRegistryIOHelpers.cc b/src/EDMRegistryIOHelpers.cc new file mode 100644 index 000000000..fc88c9204 --- /dev/null +++ b/src/EDMRegistryIOHelpers.cc @@ -0,0 +1,38 @@ +#include "podio/utilities/EDMRegistryIOHelpers.h" + +namespace podio { + +void EDMDefinitionCollector::registerEDMDef(const podio::CollectionBase* coll, const std::string& name) { + const auto edmIndex = coll->getDefinitionRegistryIndex(); + if (edmIndex == EDMDefinitionRegistry::NoDefinitionAvailable) { + std::cerr << "No EDM definition available for collection " << name << std::endl; + } else { + if (edmIndex != EDMDefinitionRegistry::NoDefinitionNecessary) { + m_edmDefRegistryIdcs.insert(edmIndex); + } + } +} + +std::vector> EDMDefinitionCollector::getEDMDefinitionsToWrite() const { + std::vector> edmDefinitions; + edmDefinitions.reserve(m_edmDefRegistryIdcs.size()); + for (const auto& index : m_edmDefRegistryIdcs) { + const auto& edmRegistry = podio::EDMDefinitionRegistry::instance(); + edmDefinitions.emplace_back(edmRegistry.getEDMName(index), edmRegistry.getDefinition(index)); + } + + return edmDefinitions; +} + +const std::string_view EDMDefinitionHolder::getEDMDefinition(const std::string& edmName) const { + const auto it = std::find_if(m_availEDMDefs.cbegin(), m_availEDMDefs.cend(), + [&edmName](const auto& entry) { return std::get<0>(entry) == edmName; }); + + if (it != m_availEDMDefs.cend()) { + return std::get<1>(*it); + } + + return "{}"; +} + +} // namespace podio diff --git a/src/ROOTFrameReader.cc b/src/ROOTFrameReader.cc index 421f13e56..ae124a46a 100644 --- a/src/ROOTFrameReader.cc +++ b/src/ROOTFrameReader.cc @@ -235,17 +235,6 @@ void ROOTFrameReader::openFiles(const std::vector& filenames) { } } -const std::string_view ROOTFrameReader::getEDMDefinition(const std::string& edmName) const { - const auto it = std::find_if(m_availEDMDefs.cbegin(), m_availEDMDefs.cend(), - [&edmName](const auto& entry) { return std::get<0>(entry) == edmName; }); - - if (it != m_availEDMDefs.cend()) { - return std::get<1>(*it); - } - - return "{}"; -} - unsigned ROOTFrameReader::getEntries(const std::string& name) const { if (auto it = m_categories.find(name); it != m_categories.end()) { return it->second.chain->GetEntries(); diff --git a/src/ROOTFrameWriter.cc b/src/ROOTFrameWriter.cc index b9438e5eb..710f98e6b 100644 --- a/src/ROOTFrameWriter.cc +++ b/src/ROOTFrameWriter.cc @@ -1,6 +1,5 @@ #include "podio/ROOTFrameWriter.h" #include "podio/CollectionBase.h" -#include "podio/EDMDefinitionRegistry.h" #include "podio/Frame.h" #include "podio/GenericParameters.h" #include "podio/podioVersion.h" @@ -61,17 +60,6 @@ ROOTFrameWriter::CategoryInfo& ROOTFrameWriter::getCategoryInfo(const std::strin return it->second; } -void ROOTFrameWriter::registerEDMDef(const podio::CollectionBase* coll, const std::string& name) { - const auto edmIndex = coll->getDefinitionRegistryIndex(); - if (edmIndex == EDMDefinitionRegistry::NoDefinitionAvailable) { - std::cerr << "No EDM definition available for collection " << name << std::endl; - } else { - if (edmIndex != EDMDefinitionRegistry::NoDefinitionNecessary) { - m_edmDefRegistryIdcs.insert(edmIndex); - } - } -} - void ROOTFrameWriter::initBranches(CategoryInfo& catInfo, const std::vector& collections, /*const*/ podio::GenericParameters& parameters) { catInfo.branches.reserve(collections.size() + 1); // collections + parameters @@ -143,12 +131,7 @@ void ROOTFrameWriter::finish() { auto podioVersion = podio::version::build_version; metaTree->Branch(root_utils::versionBranchName, &podioVersion); - std::vector> edmDefinitions; - edmDefinitions.reserve(m_edmDefRegistryIdcs.size()); - for (const auto& index : m_edmDefRegistryIdcs) { - const auto& edmRegistry = podio::EDMDefinitionRegistry::instance(); - edmDefinitions.emplace_back(edmRegistry.getEDMName(index), edmRegistry.getDefinition(index)); - } + auto edmDefinitions = getEDMDefinitionsToWrite(); metaTree->Branch(root_utils::edmDefBranchName, &edmDefinitions); metaTree->Fill(); diff --git a/src/SIOFrameReader.cc b/src/SIOFrameReader.cc index f494d1f9b..7bd3a93e0 100644 --- a/src/SIOFrameReader.cc +++ b/src/SIOFrameReader.cc @@ -69,16 +69,6 @@ unsigned SIOFrameReader::getEntries(const std::string& name) const { return m_tocRecord.getNRecords(name); } -const std::string_view SIOFrameReader::getEDMDefinition(const std::string& edmName) const { - const auto it = std::find_if(m_availEDMDefs.cbegin(), m_availEDMDefs.cend(), - [&edmName](const auto& entry) { return std::get<0>(entry) == edmName; }); - if (it != m_availEDMDefs.cend()) { - return std::get<1>(*it); - } - - return "{}"; -} - bool SIOFrameReader::readFileTOCRecord() { // Check if there is a dedicated marker at the end of the file that tells us // where the TOC actually starts diff --git a/src/SIOFrameWriter.cc b/src/SIOFrameWriter.cc index 2ff156442..4289aed05 100644 --- a/src/SIOFrameWriter.cc +++ b/src/SIOFrameWriter.cc @@ -31,17 +31,6 @@ void SIOFrameWriter::writeFrame(const podio::Frame& frame, const std::string& ca writeFrame(frame, category, frame.getAvailableCollections()); } -void SIOFrameWriter::registerEDMDef(const podio::CollectionBase* coll, const std::string& name) { - const auto edmIndex = coll->getDefinitionRegistryIndex(); - if (edmIndex == EDMDefinitionRegistry::NoDefinitionAvailable) { - std::cerr << "No EDM definition available for collection " << name << std::endl; - } else { - if (edmIndex != EDMDefinitionRegistry::NoDefinitionNecessary) { - m_edmDefRegistryIdcs.insert(edmIndex); - } - } -} - void SIOFrameWriter::writeFrame(const podio::Frame& frame, const std::string& category, const std::vector& collsToWrite) { std::vector collections; @@ -63,11 +52,7 @@ void SIOFrameWriter::writeFrame(const podio::Frame& frame, const std::string& ca } void SIOFrameWriter::finish() { - auto edmDefMap = std::make_shared>(); - const auto& edmRegistry = podio::EDMDefinitionRegistry::instance(); - for (const auto& index : m_edmDefRegistryIdcs) { - edmDefMap->mapData.emplace_back(edmRegistry.getEDMName(index), edmRegistry.getDefinition(index)); - } + auto edmDefMap = std::make_shared>(getEDMDefinitionsToWrite()); sio::block_list blocks; blocks.push_back(edmDefMap); From f888d9f2eb641c351e18eb424b34d2e53d78c587 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Wed, 21 Dec 2022 17:33:08 +0100 Subject: [PATCH 08/19] Add first version of model dumping --- .../podio/utilities/EDMRegistryIOHelpers.h | 2 ++ python/podio/base_reader.py | 25 ++++++++++++++++ src/EDMRegistryIOHelpers.cc | 10 +++++++ tools/podio-dump | 30 +++++++++++++++++-- 4 files changed, 65 insertions(+), 2 deletions(-) diff --git a/include/podio/utilities/EDMRegistryIOHelpers.h b/include/podio/utilities/EDMRegistryIOHelpers.h index f8f8a8f25..2e7773ebc 100644 --- a/include/podio/utilities/EDMRegistryIOHelpers.h +++ b/include/podio/utilities/EDMRegistryIOHelpers.h @@ -39,6 +39,8 @@ class EDMDefinitionHolder { */ const std::string_view getEDMDefinition(const std::string& edmName) const; + std::vector getAvailableEDMDefinitions() const; + protected: std::vector> m_availEDMDefs{}; }; diff --git a/python/podio/base_reader.py b/python/podio/base_reader.py index b45cfa3f1..c624696ee 100644 --- a/python/podio/base_reader.py +++ b/python/podio/base_reader.py @@ -55,3 +55,28 @@ def is_legacy(self): bool: True if this is a legacy file reader """ return self._is_legacy + + @property + def edm_definitions(self): + """Get the available EDM definitions from this reader. + + Returns: + tuple(str): The names of the available EDM definitions + """ + if self._is_legacy: + return () + return tuple(n.c_str() for n in self._reader.getAvailableEDMDefinitions()) + + def get_edm_definition(self, edm_name): + """Get the EDM definition of the passed EDM as JSON string. + + Args: + str: The name of the EDM + + Returns: + str: The complete model definition in JSON format. Use, e.g. json.loads + to convert it into a python dictionary. + """ + if self._is_legacy: + return "" + return self._reader.getEDMDefinition(edm_name).data() diff --git a/src/EDMRegistryIOHelpers.cc b/src/EDMRegistryIOHelpers.cc index fc88c9204..adda98ffe 100644 --- a/src/EDMRegistryIOHelpers.cc +++ b/src/EDMRegistryIOHelpers.cc @@ -1,4 +1,5 @@ #include "podio/utilities/EDMRegistryIOHelpers.h" +#include namespace podio { @@ -35,4 +36,13 @@ const std::string_view EDMDefinitionHolder::getEDMDefinition(const std::string& return "{}"; } +std::vector EDMDefinitionHolder::getAvailableEDMDefinitions() const { + std::vector defs{}; + defs.reserve(m_availEDMDefs.size()); + std::transform(m_availEDMDefs.cbegin(), m_availEDMDefs.cend(), std::back_inserter(defs), + [](const auto& elem) { return std::get<0>(elem); }); + + return defs; +} + } // namespace podio diff --git a/tools/podio-dump b/tools/podio-dump index efb5dfc18..d991f8427 100755 --- a/tools/podio-dump +++ b/tools/podio-dump @@ -2,6 +2,8 @@ """podio-dump tool to dump contents of podio files""" import sys +import json +import yaml from podio.reading import get_reader @@ -15,9 +17,11 @@ def print_general_info(reader, filename): Args: reader (root_io.Reader, sio_io.Reader): An initialized reader """ - print(f'input file: {filename}\n') legacy_text = ' (this is a legacy file!)' if reader.is_legacy else '' - print(f'Frame categories in this file{legacy_text}:') + print(f'input file: {filename}{legacy_text}\n') + print(f'EDM model definitions stored in this file: {", ".join(reader.edm_definitions)}') + print() + print('Frame categories in this file:') print(f'{"Name":<20} {"Entries":<10}') print('-' * 31) for category in reader.categories: @@ -68,6 +72,18 @@ def print_frame(frame, cat_name, ientry, detailed): print('\n', flush=True) +def dump_model(reader, model_name): + """Dump the model in yaml format""" + if model_name not in reader.edm_definitions: + print(f'ERROR: Cannot dump model \'{model_name}\' (not present in file)') + return False + + model_def = json.loads(reader.get_edm_definition(model_name)) + print(yaml.dump(model_def)) + + return True + + def main(args): """Main""" try: @@ -76,6 +92,12 @@ def main(args): print(f'ERROR: Cannot open file \'{args.inputfile}\': {err}') sys.exit(1) + if args.dump_edm is not None: + if dump_model(reader, args.dump_edm): + sys.exit(0) + else: + sys.exit(1) + print_general_info(reader, args.inputfile) if args.category not in reader.categories: print(f'ERROR: Cannot print category \'{args.category}\' (not present in file)') @@ -120,6 +142,10 @@ if __name__ == '__main__': type=parse_entry_range, default=[0]) parser.add_argument('-d', '--detailed', help='Dump the full contents not just the collection info', action='store_true', default=False) + parser.add_argument('--dump-edm', + help='Dump the specified EDM definition from the file in yaml format', + type=str, default=None) + clargs = parser.parse_args() main(clargs) From 10cdac911343ae0c817df6db142eb6a69bda791b Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 22 Dec 2022 16:11:11 +0100 Subject: [PATCH 09/19] Make dumped models look more like inputs - Change order of main keys (options, components, datatypes) - Slightly tweak formatting (as far as possible with PyYAML) --- python/podio/generator_utils.py | 5 +++-- tests/CMakeLists.txt | 2 +- tests/{ => scripts}/get_test_inputs.sh | 0 tools/podio-dump | 3 +-- 4 files changed, 5 insertions(+), 5 deletions(-) rename tests/{ => scripts}/get_test_inputs.sh (100%) diff --git a/python/podio/generator_utils.py b/python/podio/generator_utils.py index e267772f8..e50b139e7 100644 --- a/python/podio/generator_utils.py +++ b/python/podio/generator_utils.py @@ -195,9 +195,8 @@ def _to_json(self): class DataModel: # pylint: disable=too-few-public-methods """A class for holding a complete datamodel read from a configuration file""" + def __init__(self, datatypes=None, components=None, options=None): - self.datatypes = datatypes or {} - self.components = components or {} self.options = options or { # should getters / setters be prefixed with get / set? "getSyntax": False, @@ -206,6 +205,8 @@ def __init__(self, datatypes=None, components=None, options=None): # use subfolder when including package header files "includeSubfolder": False, } + self.components = components or {} + self.datatypes = datatypes or {} def _to_json(self): """Return the dictionary, so that we can easily hook this into the pythons diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ad24cbf31..f490b6a32 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -73,7 +73,7 @@ endforeach() if (NOT DEFINED CACHE{PODIO_TEST_INPUT_DATA_DIR} OR NOT EXISTS ${PODIO_TEST_INPUT_DATA_DIR}/example.root) message("Getting test input files") execute_process( - COMMAND bash ${CMAKE_CURRENT_LIST_DIR}/get_test_inputs.sh + COMMAND bash ${CMAKE_CURRENT_LIST_DIR}/scripts/get_test_inputs.sh OUTPUT_VARIABLE podio_test_input_data_dir RESULT_VARIABLE test_inputs_available ) diff --git a/tests/get_test_inputs.sh b/tests/scripts/get_test_inputs.sh similarity index 100% rename from tests/get_test_inputs.sh rename to tests/scripts/get_test_inputs.sh diff --git a/tools/podio-dump b/tools/podio-dump index d991f8427..d7e951c4b 100755 --- a/tools/podio-dump +++ b/tools/podio-dump @@ -79,7 +79,7 @@ def dump_model(reader, model_name): return False model_def = json.loads(reader.get_edm_definition(model_name)) - print(yaml.dump(model_def)) + print(yaml.dump(model_def, sort_keys=False, default_flow_style=False)) return True @@ -146,6 +146,5 @@ if __name__ == '__main__': help='Dump the specified EDM definition from the file in yaml format', type=str, default=None) - clargs = parser.parse_args() main(clargs) From 53098cc63346b1b828d55e6d80c68906d4476dea Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 22 Dec 2022 17:03:06 +0100 Subject: [PATCH 10/19] Add roundtrip tests for stored EDM definitions --- src/SIOBlock.cc | 2 +- tests/CMakeLists.txt | 44 +++++++++++++++++++++++++++++ tests/CTestCustom.cmake | 5 ++++ tests/scripts/dumpModelRoundTrip.sh | 36 +++++++++++++++++++++++ 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100755 tests/scripts/dumpModelRoundTrip.sh diff --git a/src/SIOBlock.cc b/src/SIOBlock.cc index 097e79224..c0a514e6a 100644 --- a/src/SIOBlock.cc +++ b/src/SIOBlock.cc @@ -126,7 +126,7 @@ SIOBlockLibraryLoader::SIOBlockLibraryLoader() { const auto status = loadLib(lib); switch (status) { case LoadStatus::Success: - std::cout << "Loaded SIOBlocks library \'" << lib << "\' (from " << dir << ")" << std::endl; + std::cerr << "Loaded SIOBlocks library \'" << lib << "\' (from " << dir << ")" << std::endl; break; case LoadStatus::AlreadyLoaded: std::cerr << "SIOBlocks library \'" << lib << "\' already loaded. Not loading again from " << dir << std::endl; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f490b6a32..cac189bec 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -245,3 +245,47 @@ else() LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/src:$:$ENV{LD_LIBRARY_PATH} ) endif() + +# Add tests for storing and retrieving the EDM definitions into the produced +# files +add_test(edm_def_store_roundtrip_root ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.root datamodel) +add_test(edm_def_store_roundtrip_root_extension ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.root datamodel extension_datamodel) + + +# Need the input files that are produced by other tests +set_tests_properties( + edm_def_store_roundtrip_root + edm_def_store_roundtrip_root_extension + PROPERTIES + DEPENDS write_frame_root + ) + +set(sio_roundtrip_tests "") +if (TARGET read_sio) + add_test(edm_def_store_roundtrip_sio ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.sio datamodel) + add_test(edm_def_store_roundtrip_sio_extension ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.sio datamodel extension_datamodel) + + set(sio_roundtrip_tests + edm_def_store_roundtrip_sio + edm_def_store_roundtrip_sio_extension + ) + + set_tests_properties( + ${sio_roundtrip_tests} + PROPERTIES + DEPENDS write_frame_sio + ) +endif() + +# We need to convert this into a list of arguments that can be used as environment variable +list(JOIN PODIO_IO_HANDLERS " " IO_HANDLERS) + +set_tests_properties( + edm_def_store_roundtrip_root + edm_def_store_roundtrip_root_extension + ${sio_roundtrip_tests} + PROPERTIES + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ENVIRONMENT + "PODIO_BASE=${CMAKE_SOURCE_DIR};IO_HANDLERS=${IO_HANDLERS};LD_LIBRARY_PATH=${CMAKE_CURRENT_BINARY_DIR}:${CMAKE_BINARY_DIR}/src:$ENV{LD_LIBRARY_PATH};PYTHONPATH=${CMAKE_SOURCE_DIR}/python:$ENV{PYTHONPATH};ROOT_INCLUDE_PATH=${CMAKE_SOURCE_DIR}/tests/datamodel:${CMAKE_SOURCE_DIR}/include:$ENV{ROOT_INCLUDE_PATH}" + ) diff --git a/tests/CTestCustom.cmake b/tests/CTestCustom.cmake index 6d1e4e165..214fbc88f 100644 --- a/tests/CTestCustom.cmake +++ b/tests/CTestCustom.cmake @@ -52,6 +52,11 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ podio-dump-sio podio-dump-detailed-sio podio-dump-detailed-sio-legacy + + edm_def_store_roundtrip_root + edm_def_store_roundtrip_root_extension + edm_def_store_roundtrip_sio + edm_def_store_roundtrip_sio_extension ) # ostream_operator is working with Memory sanitizer (at least locally) diff --git a/tests/scripts/dumpModelRoundTrip.sh b/tests/scripts/dumpModelRoundTrip.sh new file mode 100755 index 000000000..9f9bc2148 --- /dev/null +++ b/tests/scripts/dumpModelRoundTrip.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Script to check that an EDM definition dumped from a file is "equivalent" to +# the original definition. Essentially does not check that the YAML file is the +# same, but rather that the generated code is the same + +set -eu + +INPUT_FILE=${1} # the datafile +EDM_NAME=${2} # the name of the EDM +COMP_BASE_FOLDER="" # where the source to compare against is +if [ -$# -gt 2 ]; then + COMP_BASE_FOLDER=${3} +fi + +# Create a few temporary but unique files and directories to store output +DUMPED_MODEL=${INPUT_FILE}.dumped_${EDM_NAME}.yaml +OUTPUT_FOLDER=${INPUT_FILE}.dumped_${EDM_NAME} +mkdir -p ${OUTPUT_FOLDER} + +# Dump the model to a yaml file +${PODIO_BASE}/tools/podio-dump --dump-edm ${EDM_NAME} ${INPUT_FILE} > ${DUMPED_MODEL} + +# Regenerate the code via the class generator and the freshly dumped modl +${PODIO_BASE}/python/podio_class_generator.py \ + --clangformat \ + ${DUMPED_MODEL} \ + ${OUTPUT_FOLDER} \ + ${EDM_NAME} \ + ${IO_HANDLERS} + +# Compare to the originally generated code, that has been used to write the data +# file. Need to diff subfolders explitly here because $PODIO_BASE/tests contains +# more stuff +diff -ru ${OUTPUT_FOLDER}/${EDM_NAME} ${PODIO_BASE}/tests/${COMP_BASE_FOLDER}/${EDM_NAME} +diff -ru ${OUTPUT_FOLDER}/src ${PODIO_BASE}/tests/${COMP_BASE_FOLDER}/src +diff -u ${OUTPUT_FOLDER}/podio_generated_files.cmake ${PODIO_BASE}/tests/podio_generated_files.cmake From 0f9ec169adc2ff372b7c376319c8f48cf6854d50 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Fri, 23 Dec 2022 12:45:46 +0100 Subject: [PATCH 11/19] Add documentation for EDM definition embedding --- doc/advanced_topics.md | 115 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/doc/advanced_topics.md b/doc/advanced_topics.md index 1ebdcad28..bfd504cbc 100644 --- a/doc/advanced_topics.md +++ b/doc/advanced_topics.md @@ -110,3 +110,118 @@ To implement your own transient event store, the only requirement is to set the - Run pre-commit manually `$ pre-commit run --all-files` + +## Retrieving the EDM definition from a data file +It is possible to get the EDM definition(s) that was used to generate the +datatypes that are stored in a data file. This makes it possible to re-generate +the necessary code and build all libraries again in case they are not easily +available otherwise. To see which EDM definitions are available in a data file +use the `podio-dump` utility + +```bash +podio-dump +``` +which will give an (exemplary) output like this +``` +input file: + +EDM model definitions stored in this file: edm4hep + +[...] +``` + +To actually dump the model definition to stdout use the `--dump-edm` option +and the name of the datamodel you want to dump: + +```bash +podio-dump --dump-edm edm4hep > dumped_edm4hep.yaml +``` + +Here we directly redirected the output to a yaml file that can then again be +used by the `podio_class_generator.py` to generate the corresponding c++ code +(or be passed to the cmake macros). + +**Note that the dumped EDM definition is equivalent but not necessarily exactly +the same as the original EDM definition.** E.g. all the datatypes will have all +their fields (`Members`, `OneToOneRelations`, `OneToManyRelations`, +`VectorMembers`) defined, and defaulted to empty lists in case they were not +present in the original EDM definition. The reason for this is that the embedded +EDM definition is the pre-processed and validated one [as described +below](#technical-details-on-edm-definition-embedding) + +### Accessing the EDM definition programmatically +The EDM definition can also be accessed programmatically via the +`[ROOT|SIO]FrameReader::getEDMDefinition` method. It takes an EDM name as its +single argument and returns the EDM definition as a JSON string. Most likely +this has to be decoded into an actual JSON structure in order to be usable (e.g. +via `json.loads` in python to get a `dict`). + +### Technical details on EDM definition embedding +The EDM definition is embedded into the core EDM library as a raw string literal +in JSON format. This string is generated into the `DatamodelDefinition.h` file as + +```cpp +namespace ::meta { +static constexpr auto __JSONDefinition = R"EDMDEFINITION()EDMDEFINITION"; +} +``` + +where `` is the name of the EDM as passed to the +`podio_class_generator.py` (or the cmake macro). The `` +is obtained from the pre-processed EDM definition that is read from the yaml +file. During this pre-processing the EDM definition is validated, and optional +fields are filled with empty defaults. Additionally, the `includeSubfolder` +option will be populated with the actual include subfolder, in case it has been +set to `True` in the yaml file. Since the json encoded definition is generated +right before the pre-processed model is passed to the class generator, this +definition is equivalent, but not necessarily equal to the original definition. + +#### The `EDMDefinitionRegistry` +To make access to the embedded datamodel definitions a bit easier the +`EDMDefinitionRegistry` (singleton) keeps a map of all loaded EDMs and makes +access to this string slightly easier, providing two access methods: + +```cpp +const std::string_view getEDMDefinition(const std::string& edmName) const; + +const std::string_view getEDMDefinition(size_t index) const; +``` + +where `index` can be obtained from each collection via +`getDefinitionRegistryIndex`. That in turn simply calls +`::meta::RegistryIndex::value()`, another singleton like object +that takes care of registering an EDM definition to the `EDMDefinitionRegistry` +during its static initialization. It is also defined in the +`DatamodelDefinition.h` header. + +Since the EDM definition is embedded as a raw string literal into the core +datamodel shared library, it is in principle also relatively straight forward to +retrieve it from this library by inspecting the binary, e.g. via +```bash +readelf -p .rodata libedm4hep.so | grep options +``` + +which will result in something like + +``` + [ 300] {"options": {"getSyntax": true, "exposePODMembers": false, "includeSubfolder": "edm4hep/"}, "components": {<...>}, "datatypes": {<...>}} +``` + +#### I/O helpers for EDM definition storing +The `podio/utilities/EDMRegistryIOHelpers.h` header defines two utility (mixin) +classes, that help with instrumenting readers and writers with functionality to +read and write all the necessary EDM definitions. + +- The `EDMDefinitionCollector` is intended to be inherited from by writer. It + essentially collects the EDM definitions of all the collections it encounters. + The `registerEDMDef` method it provides should be called with every collection + that is written. The `getEDMDefinitionsToWrite` method returns a vector of all + EDM names and their definition that were encountered during writing. **It is + then the writers responsibility to actually store this information into the + file**. +- The `EDMDefinitionHolder` is intended to be inherited from by readers (needs + to be `public` inheritance if the functionality should be directly exposed to + users of the reader). It provides the `getEDMDefinition` and + `getAvailableEDMDefinitions` methods. In order for it to work properly **its + `m_availEDMDefs` member has to be populated by the reader** before a possible + call to either of these two methods. From a02c19134199815f0736349afb0a4395babb641a Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Tue, 14 Feb 2023 10:21:33 +0100 Subject: [PATCH 12/19] Fix documentation --- doc/advanced_topics.md | 2 +- include/podio/CollectionBase.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/advanced_topics.md b/doc/advanced_topics.md index bfd504cbc..afa9e2468 100644 --- a/doc/advanced_topics.md +++ b/doc/advanced_topics.md @@ -212,7 +212,7 @@ The `podio/utilities/EDMRegistryIOHelpers.h` header defines two utility (mixin) classes, that help with instrumenting readers and writers with functionality to read and write all the necessary EDM definitions. -- The `EDMDefinitionCollector` is intended to be inherited from by writer. It +- The `EDMDefinitionCollector` is intended to be inherited from by writers. It essentially collects the EDM definitions of all the collections it encounters. The `registerEDMDef` method it provides should be called with every collection that is written. The `getEDMDefinitionsToWrite` method returns a vector of all diff --git a/include/podio/CollectionBase.h b/include/podio/CollectionBase.h index 16efd8aa0..5a032ddd5 100644 --- a/include/podio/CollectionBase.h +++ b/include/podio/CollectionBase.h @@ -77,6 +77,7 @@ class CollectionBase { /// print this collection to the passed stream virtual void print(std::ostream& os = std::cout, bool flush = true) const = 0; + /// Get the index in the EDMDefinitionRegistry of the EDM this collection belongs to virtual size_t getDefinitionRegistryIndex() const = 0; }; From e88c7884032c72b7f6f822b2e6e54b899b4a933d Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Tue, 14 Feb 2023 10:31:03 +0100 Subject: [PATCH 13/19] Add warnings output when trying to retrieve non existant EDMs --- include/podio/EDMDefinitionRegistry.h | 11 +++++++--- python/templates/Collection.cc.jinja2 | 2 +- python/templates/DatamodelDefinition.h.jinja2 | 6 +++--- src/EDMDefinitionRegistry.cc | 20 ++++++++++++++----- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/include/podio/EDMDefinitionRegistry.h b/include/podio/EDMDefinitionRegistry.h index b1e377085..7a2296738 100644 --- a/include/podio/EDMDefinitionRegistry.h +++ b/include/podio/EDMDefinitionRegistry.h @@ -9,7 +9,12 @@ namespace podio { class EDMDefinitionRegistry { public: - static EDMDefinitionRegistry& instance(); + /// Get the registry + static const EDMDefinitionRegistry& instance(); + + // mutable instance only used for the initial registration! + static EDMDefinitionRegistry& mutInstance(); + ~EDMDefinitionRegistry() = default; EDMDefinitionRegistry(const EDMDefinitionRegistry&) = delete; EDMDefinitionRegistry& operator=(const EDMDefinitionRegistry&) = delete; @@ -22,11 +27,11 @@ class EDMDefinitionRegistry { static constexpr size_t NoDefinitionAvailable = -2; /** - * Get the definition (in JSON format) of the EDM with the given edm_name. If + * Get the definition (in JSON format) of the EDM with the given edmName. If * no EDM under the given name can be found, an empty model definition is * returned */ - const std::string_view getDefinition(std::string_view edm_name) const; + const std::string_view getDefinition(std::string_view edmName) const; /** * Get the defintion (in JSON format) of the EDM wth the given index. If no diff --git a/python/templates/Collection.cc.jinja2 b/python/templates/Collection.cc.jinja2 index 2db4f6005..45ee2cb37 100644 --- a/python/templates/Collection.cc.jinja2 +++ b/python/templates/Collection.cc.jinja2 @@ -180,7 +180,7 @@ podio::CollectionReadBuffers {{ collection_type }}::createBuffers() /*const*/ { {% endfor %} size_t {{ collection_type }}::getDefinitionRegistryIndex() const { - return {{ package_name }}::meta::RegistryIndex::value(); + return {{ package_name }}::meta::DefinitionRegistryIndex::value(); } #ifdef PODIO_JSON_OUTPUT diff --git a/python/templates/DatamodelDefinition.h.jinja2 b/python/templates/DatamodelDefinition.h.jinja2 index 3ef6694aa..23ff21b18 100644 --- a/python/templates/DatamodelDefinition.h.jinja2 +++ b/python/templates/DatamodelDefinition.h.jinja2 @@ -15,15 +15,15 @@ static constexpr auto {{ package_name }}__JSONDefinition = R"EDMDEFINITION({{ ed * Implemented as a singleton mainly to ensure only a single registration of * each EDM, during the constructor */ -class RegistryIndex { +class DefinitionRegistryIndex { public: static size_t value() { - static auto index = RegistryIndex(podio::EDMDefinitionRegistry::instance().registerEDM("{{ package_name }}", {{ package_name }}__JSONDefinition)); + static auto index = DefinitionRegistryIndex(podio::EDMDefinitionRegistry::mutInstance().registerEDM("{{ package_name }}", {{ package_name }}__JSONDefinition)); return index.m_value; } private: - RegistryIndex(size_t v) : m_value(v) {} + DefinitionRegistryIndex(size_t v) : m_value(v) {} size_t m_value{podio::EDMDefinitionRegistry::NoDefinitionAvailable}; }; diff --git a/src/EDMDefinitionRegistry.cc b/src/EDMDefinitionRegistry.cc index f111b0644..7ec98e864 100644 --- a/src/EDMDefinitionRegistry.cc +++ b/src/EDMDefinitionRegistry.cc @@ -6,11 +6,15 @@ #include namespace podio { -EDMDefinitionRegistry& EDMDefinitionRegistry::instance() { +const EDMDefinitionRegistry& EDMDefinitionRegistry::instance() { static EDMDefinitionRegistry registryInstance; return registryInstance; } +EDMDefinitionRegistry& EDMDefinitionRegistry::mutInstance() { + return const_cast(instance()); +} + size_t EDMDefinitionRegistry::registerEDM(std::string name, std::string_view definition) { const auto it = std::find_if(m_definitions.cbegin(), m_definitions.cend(), [&name](const auto& kvPair) { return kvPair.first == name; }); @@ -25,16 +29,21 @@ size_t EDMDefinitionRegistry::registerEDM(std::string name, std::string_view def return std::distance(m_definitions.cbegin(), it); } -const std::string_view EDMDefinitionRegistry::getDefinition(std::string_view edm_name) const { +const std::string_view EDMDefinitionRegistry::getDefinition(std::string_view edmName) const { const auto it = std::find_if(m_definitions.cbegin(), m_definitions.cend(), - [&edm_name](const auto& kvPair) { return kvPair.first == edm_name; }); + [&edmName](const auto& kvPair) { return kvPair.first == edmName; }); + if (it == m_definitions.cend()) { + std::cerr << "PODIO WARNING: Cannot find the definition for the EDM with the name " << edmName << std::endl; + static constexpr std::string_view emptyDef = "{}"; // valid empty JSON + return emptyDef; + } - // TODO: Output when not found - return getDefinition(std::distance(m_definitions.cbegin(), it)); + return it->second; } const std::string_view EDMDefinitionRegistry::getDefinition(size_t index) const { if (index >= m_definitions.size()) { + std::cerr << "PODIO WARNING: Cannot find the definition for the EDM with the index " << index << std::endl; static constexpr std::string_view emptyDef = "{}"; // valid empty JSON return emptyDef; } @@ -44,6 +53,7 @@ const std::string_view EDMDefinitionRegistry::getDefinition(size_t index) const const std::string& EDMDefinitionRegistry::getEDMName(size_t index) const { if (index >= m_definitions.size()) { + std::cout << "PODIO WARNING: Cannot find the name of the EDM with the index " << index << std::endl; static const std::string emptyName = ""; return emptyName; } From 27ab1bb76d1956e4401dfb292657c60491e7e16b Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 2 Mar 2023 16:46:45 +0100 Subject: [PATCH 14/19] Rename EDMDefinitionRegistry and clarify documentation --- include/podio/CollectionBase.h | 4 +- include/podio/DatamodelRegistry.h | 99 +++++++++++++++++++ include/podio/EDMDefinitionRegistry.h | 62 ------------ include/podio/ROOTFrameReader.h | 2 +- include/podio/ROOTFrameWriter.h | 2 +- include/podio/SIOFrameReader.h | 2 +- include/podio/SIOFrameWriter.h | 2 +- include/podio/UserDataCollection.h | 6 +- .../podio/utilities/EDMRegistryIOHelpers.h | 37 ++++--- python/templates/Collection.cc.jinja2 | 4 +- python/templates/Collection.h.jinja2 | 2 +- python/templates/DatamodelDefinition.h.jinja2 | 18 ++-- src/CMakeLists.txt | 4 +- ...nitionRegistry.cc => DatamodelRegistry.cc} | 24 ++--- src/EDMRegistryIOHelpers.cc | 21 ++-- src/ROOTFrameWriter.cc | 4 +- src/SIOFrameWriter.cc | 6 +- 17 files changed, 175 insertions(+), 124 deletions(-) create mode 100644 include/podio/DatamodelRegistry.h delete mode 100644 include/podio/EDMDefinitionRegistry.h rename src/{EDMDefinitionRegistry.cc => DatamodelRegistry.cc} (64%) diff --git a/include/podio/CollectionBase.h b/include/podio/CollectionBase.h index 5a032ddd5..fcb81401a 100644 --- a/include/podio/CollectionBase.h +++ b/include/podio/CollectionBase.h @@ -77,8 +77,8 @@ class CollectionBase { /// print this collection to the passed stream virtual void print(std::ostream& os = std::cout, bool flush = true) const = 0; - /// Get the index in the EDMDefinitionRegistry of the EDM this collection belongs to - virtual size_t getDefinitionRegistryIndex() const = 0; + /// Get the index in the DatatypeRegistry of the EDM this collection belongs to + virtual size_t getDatamodelRegistryIndex() const = 0; }; } // namespace podio diff --git a/include/podio/DatamodelRegistry.h b/include/podio/DatamodelRegistry.h new file mode 100644 index 000000000..a32aa8218 --- /dev/null +++ b/include/podio/DatamodelRegistry.h @@ -0,0 +1,99 @@ +#ifndef PODIO_DATAMODELREGISTRY_H +#define PODIO_DATAMODELREGISTRY_H + +#include +#include +#include +#include + +namespace podio { + +/** + * Global registry holding information about datamodels and datatypes defined + * therein that are currently known by podio (i.e. which have been dynamically + * loaded). + * + * This is a singleton which is (statically) populated during dynamic loading of + * generated EDMs. In this context an **EDM refers to the shared library** that + * is compiled from the generated code from a datamodel definition in YAML + * format. When we refer to a **datamodel** in this context we talk about the + * entity as a whole, i.e. its definition in a YAML file, but also the concrete + * implementation as an EDM, as well as all other information that is related to + * it. In the API of this registry this will be used, unless we want to + * highlight that we are referring to a specific part of a datamodel. + */ +class DatamodelRegistry { +public: + /// Get the registry + static const DatamodelRegistry& instance(); + + // Mutable instance only used for the initial registration! + static DatamodelRegistry& mutInstance(); + + ~DatamodelRegistry() = default; + DatamodelRegistry(const DatamodelRegistry&) = delete; + DatamodelRegistry& operator=(const DatamodelRegistry&) = delete; + DatamodelRegistry(DatamodelRegistry&&) = delete; + DatamodelRegistry& operator=(const DatamodelRegistry&&) = delete; + + /// Dedicated index value for collections that don't have a datamodel + /// definition (e.g. UserDataCollection) + static constexpr size_t NoDefinitionNecessary = -1; + /// Dedicated index value for error checking, used to default init the generated RegistryIndex + static constexpr size_t NoDefinitionAvailable = -2; + + /** + * Get the definition (in JSON format) of the datamodel with the given + * edmName. + * + * If no datamodel with the given name can be found, an empty datamodel + * definition, i.e. an empty JSON object ("{}"), is returned. + * + * @param name The name of the datamodel + */ + const std::string_view getDatamodelDefinition(std::string_view name) const; + + /** + * Get the defintion (in JSON format) of the datamodel wth the given index. + * + * If no datamodel is found under the given index, an empty datamodel + * definition, i.e. an empty JSON object ("{}"), is returned. + * + * @param index The datamodel definition index that can be obtained from each + * collection + */ + const std::string_view getDatamodelDefinition(size_t index) const; + + /** + * Get the name of the datamodel that is stored under the given index. + * + * If no datamodel is found under the given index, an empty string is returned + * + * @param index The datamodel definition index that can be obtained from each + * collection + */ + const std::string& getDatamodelName(size_t index) const; + + /** + * Register a datamodel return the index in the registry. + * + * This is the hook that is called during dynamic loading of an EDM to + * register information for this EDM. If an EDM has already been registered + * under this name, than the index to the existing EDM in the registry will be + * returned. + * + * @param name The name of the EDM that should be registered + * @param definition The datamodel definition from which this EDM has been + * generated in JSON format + * + */ + size_t registerDatamodel(std::string name, std::string_view definition); + +private: + DatamodelRegistry() = default; + /// The stored definitions + std::vector> m_definitions{}; +}; +} // namespace podio + +#endif // PODIO_DATAMODELREGISTRY_H diff --git a/include/podio/EDMDefinitionRegistry.h b/include/podio/EDMDefinitionRegistry.h deleted file mode 100644 index 7a2296738..000000000 --- a/include/podio/EDMDefinitionRegistry.h +++ /dev/null @@ -1,62 +0,0 @@ -#ifndef PODIO_EDMDEFINITIONREGISTRY_H -#define PODIO_EDMDEFINITIONREGISTRY_H - -#include -#include -#include -#include - -namespace podio { -class EDMDefinitionRegistry { -public: - /// Get the registry - static const EDMDefinitionRegistry& instance(); - - // mutable instance only used for the initial registration! - static EDMDefinitionRegistry& mutInstance(); - - ~EDMDefinitionRegistry() = default; - EDMDefinitionRegistry(const EDMDefinitionRegistry&) = delete; - EDMDefinitionRegistry& operator=(const EDMDefinitionRegistry&) = delete; - EDMDefinitionRegistry(EDMDefinitionRegistry&&) = delete; - EDMDefinitionRegistry& operator=(const EDMDefinitionRegistry&&) = delete; - - /// Dedicated index value for collections that don't need a definition (e.g. UserDataCollection) - static constexpr size_t NoDefinitionNecessary = -1; - /// Dedicated index value for error checking, used to default init the generated RegistryIndex - static constexpr size_t NoDefinitionAvailable = -2; - - /** - * Get the definition (in JSON format) of the EDM with the given edmName. If - * no EDM under the given name can be found, an empty model definition is - * returned - */ - const std::string_view getDefinition(std::string_view edmName) const; - - /** - * Get the defintion (in JSON format) of the EDM wth the given index. If no - * EDM is found under the given index, an empty model definition is returned. - */ - const std::string_view getDefinition(size_t index) const; - - /** - * Get the name of the EDM that is stored under the given index. If no EDM is - * found under the given index, an empty string is returned - */ - const std::string& getEDMName(size_t index) const; - - /** - * Register a definition and return the index in the registry. If a definition - * already exists under the given name, then the index of the existing - * definition is returned - */ - size_t registerEDM(std::string name, std::string_view definition); - -private: - EDMDefinitionRegistry() = default; - /// The stored definitions - std::vector> m_definitions{}; -}; -} // namespace podio - -#endif // PODIO_EDMDEFINITIONREGISTRY_H diff --git a/include/podio/ROOTFrameReader.h b/include/podio/ROOTFrameReader.h index 0e8edb89c..7df493096 100644 --- a/include/podio/ROOTFrameReader.h +++ b/include/podio/ROOTFrameReader.h @@ -41,7 +41,7 @@ struct CollectionReadBuffers; * This class has the function to read available data from disk * and to prepare collections and buffers. **/ -class ROOTFrameReader : public EDMDefinitionHolder { +class ROOTFrameReader : public DatamodelDefinitionHolder { public: ROOTFrameReader() = default; diff --git a/include/podio/ROOTFrameWriter.h b/include/podio/ROOTFrameWriter.h index 6ef1dc502..cae0e7bba 100644 --- a/include/podio/ROOTFrameWriter.h +++ b/include/podio/ROOTFrameWriter.h @@ -21,7 +21,7 @@ class Frame; class CollectionBase; class GenericParameters; -class ROOTFrameWriter : EDMDefinitionCollector { +class ROOTFrameWriter : DatamodelDefinitionCollector { public: ROOTFrameWriter(const std::string& filename); ~ROOTFrameWriter() = default; diff --git a/include/podio/SIOFrameReader.h b/include/podio/SIOFrameReader.h index 3892a5014..51aeae8b3 100644 --- a/include/podio/SIOFrameReader.h +++ b/include/podio/SIOFrameReader.h @@ -17,7 +17,7 @@ namespace podio { class CollectionIDTable; -class SIOFrameReader : public EDMDefinitionHolder { +class SIOFrameReader : public DatamodelDefinitionHolder { public: SIOFrameReader(); diff --git a/include/podio/SIOFrameWriter.h b/include/podio/SIOFrameWriter.h index 37fc3954b..5e4b95328 100644 --- a/include/podio/SIOFrameWriter.h +++ b/include/podio/SIOFrameWriter.h @@ -14,7 +14,7 @@ namespace podio { class Frame; -class SIOFrameWriter : EDMDefinitionCollector { +class SIOFrameWriter : DatamodelDefinitionCollector { public: SIOFrameWriter(const std::string& filename); ~SIOFrameWriter() = default; diff --git a/include/podio/UserDataCollection.h b/include/podio/UserDataCollection.h index 81970c41f..7d28e2c99 100644 --- a/include/podio/UserDataCollection.h +++ b/include/podio/UserDataCollection.h @@ -3,7 +3,7 @@ #include "podio/CollectionBase.h" #include "podio/CollectionBuffers.h" -#include "podio/EDMDefinitionRegistry.h" +#include "podio/DatamodelRegistry.h" #include "podio/utilities/TypeHelpers.h" #include @@ -173,8 +173,8 @@ class UserDataCollection : public CollectionBase { } } - size_t getDefinitionRegistryIndex() const override { - return EDMDefinitionRegistry::NoDefinitionNecessary; + size_t getDatamodelRegistryIndex() const override { + return DatamodelRegistry::NoDefinitionNecessary; } // ----- some wrapers for std::vector and access to the complete std::vector (if really needed) diff --git a/include/podio/utilities/EDMRegistryIOHelpers.h b/include/podio/utilities/EDMRegistryIOHelpers.h index 2e7773ebc..fe8881ea6 100644 --- a/include/podio/utilities/EDMRegistryIOHelpers.h +++ b/include/podio/utilities/EDMRegistryIOHelpers.h @@ -2,7 +2,7 @@ #define PODIO_UTILITIES_EDMREGISTRYIOHELPERS_H #include "podio/CollectionBase.h" -#include "podio/EDMDefinitionRegistry.h" +#include "podio/DatamodelRegistry.h" #include #include @@ -12,34 +12,47 @@ namespace podio { /** - * Helper class (mixin) to collect the EDM (JSON) definitions that should be + * Helper class (mixin) to collect the datamodel (JSON) definitions that should be * written. */ -class EDMDefinitionCollector { +class DatamodelDefinitionCollector { public: - /// Register the EDM where this collection is from to be written - void registerEDMDef(const podio::CollectionBase* coll, const std::string& name); + /** + * Register the datamodel definition of the EDM this collection is from to be + * written. + * + * @param coll A collection of an EDM + * @param name The name under which this collection is stored on file + */ + void registerDatamodelDefinition(const podio::CollectionBase* coll, const std::string& name); /// Get all the names and JSON definitions that need to be written - std::vector> getEDMDefinitionsToWrite() const; + std::vector> getDatamodelDefinitionsToWrite() const; private: std::set m_edmDefRegistryIdcs{}; ///< The indices in the EDM definition registry that need to be written }; /** - * Helper class (mixin) to hold and provide the EDM (JSON) definitions for + * Helper class (mixin) to hold and provide the datamodel (JSON) definitions for * reader classes. */ -class EDMDefinitionHolder { +class DatamodelDefinitionHolder { public: /** - * Get the EDM definition for the given EDM name. Returns an empty model - * definition if no model is stored under the given name. + * Get the datamodel definition for the given datamodel name. + * + * Returns an empty model definition if no model is stored under the given + * name. + * + * @param name The name of the datamodel */ - const std::string_view getEDMDefinition(const std::string& edmName) const; + const std::string_view getDatamodelDefinition(const std::string& name) const; - std::vector getAvailableEDMDefinitions() const; + /** + * Get all names of the datamodels that have been read from file + */ + std::vector getAvailableDatamodels() const; protected: std::vector> m_availEDMDefs{}; diff --git a/python/templates/Collection.cc.jinja2 b/python/templates/Collection.cc.jinja2 index 45ee2cb37..8c121de20 100644 --- a/python/templates/Collection.cc.jinja2 +++ b/python/templates/Collection.cc.jinja2 @@ -179,8 +179,8 @@ podio::CollectionReadBuffers {{ collection_type }}::createBuffers() /*const*/ { {{ macros.vectorized_access(class, member) }} {% endfor %} -size_t {{ collection_type }}::getDefinitionRegistryIndex() const { - return {{ package_name }}::meta::DefinitionRegistryIndex::value(); +size_t {{ collection_type }}::getDatamodelRegistryIndex() const { + return {{ package_name }}::meta::DatamodelRegistryIndex::value(); } #ifdef PODIO_JSON_OUTPUT diff --git a/python/templates/Collection.h.jinja2 b/python/templates/Collection.h.jinja2 index 6e42ff76e..2c1a80e3b 100644 --- a/python/templates/Collection.h.jinja2 +++ b/python/templates/Collection.h.jinja2 @@ -130,7 +130,7 @@ public: return m_isValid; } - size_t getDefinitionRegistryIndex() const final; + size_t getDatamodelRegistryIndex() const final; // support for the iterator protocol iterator begin() { diff --git a/python/templates/DatamodelDefinition.h.jinja2 b/python/templates/DatamodelDefinition.h.jinja2 index 23ff21b18..17a300cb9 100644 --- a/python/templates/DatamodelDefinition.h.jinja2 +++ b/python/templates/DatamodelDefinition.h.jinja2 @@ -1,30 +1,30 @@ // AUTOMATICALLY GENERATED FILE - DO NOT EDIT -#include "podio/EDMDefinitionRegistry.h" +#include "podio/DatamodelRegistry.h" namespace {{ package_name }}::meta { /** * The complete definition of the datamodel at generation time in JSON format. */ -static constexpr auto {{ package_name }}__JSONDefinition = R"EDMDEFINITION({{ edm_definition }})EDMDEFINITION"; +static constexpr auto {{ package_name }}__JSONDefinition = R"DATAMODELDEF({{ edm_definition }})DATAMODELDEF"; /** - * The helper class that takes care of registering the EDM definition to the - * EDMDefinitionRegistry and to provide the index in that registry. + * The helper class that takes care of registering the datamodel definition to + * the DatamodelRegistry and to provide the index in that registry. * * Implemented as a singleton mainly to ensure only a single registration of - * each EDM, during the constructor + * each datamodel, during the constructor */ -class DefinitionRegistryIndex { +class DatamodelRegistryIndex { public: static size_t value() { - static auto index = DefinitionRegistryIndex(podio::EDMDefinitionRegistry::mutInstance().registerEDM("{{ package_name }}", {{ package_name }}__JSONDefinition)); + static auto index = DatamodelRegistryIndex(podio::DatamodelRegistry::mutInstance().registerDatamodel("{{ package_name }}", {{ package_name }}__JSONDefinition)); return index.m_value; } private: - DefinitionRegistryIndex(size_t v) : m_value(v) {} - size_t m_value{podio::EDMDefinitionRegistry::NoDefinitionAvailable}; + DatamodelRegistryIndex(size_t v) : m_value(v) {} + size_t m_value{podio::DatamodelRegistry::NoDefinitionAvailable}; }; } // namespace {{ package_name }}::meta diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e40ed9e61..4525922d4 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -49,7 +49,7 @@ SET(core_sources GenericParameters.cc ASCIIWriter.cc EventStore.cc - EDMDefinitionRegistry.cc + DatamodelRegistry.cc EDMRegistryIOHelpers.cc ) @@ -62,7 +62,7 @@ SET(core_headers ${CMAKE_SOURCE_DIR}/include/podio/ObjectID.h ${CMAKE_SOURCE_DIR}/include/podio/UserDataCollection.h ${CMAKE_SOURCE_DIR}/include/podio/podioVersion.h - ${CMAKE_SOURCE_DIR}/include/podio/EDMDefinitionRegistry.h + ${CMAKE_SOURCE_DIR}/include/podio/DatamodelRegistry.h ${CMAKE_SOURCE_DIR}/include/podio/utilities/EDMRegistryIOHelpers.h ) diff --git a/src/EDMDefinitionRegistry.cc b/src/DatamodelRegistry.cc similarity index 64% rename from src/EDMDefinitionRegistry.cc rename to src/DatamodelRegistry.cc index 7ec98e864..d5a96e364 100644 --- a/src/EDMDefinitionRegistry.cc +++ b/src/DatamodelRegistry.cc @@ -1,4 +1,4 @@ -#include "podio/EDMDefinitionRegistry.h" +#include "podio/DatamodelRegistry.h" #include #include @@ -6,16 +6,16 @@ #include namespace podio { -const EDMDefinitionRegistry& EDMDefinitionRegistry::instance() { - static EDMDefinitionRegistry registryInstance; - return registryInstance; +const DatamodelRegistry& DatamodelRegistry::instance() { + return mutInstance(); } -EDMDefinitionRegistry& EDMDefinitionRegistry::mutInstance() { - return const_cast(instance()); +DatamodelRegistry& DatamodelRegistry::mutInstance() { + static DatamodelRegistry registryInstance; + return registryInstance; } -size_t EDMDefinitionRegistry::registerEDM(std::string name, std::string_view definition) { +size_t DatamodelRegistry::registerDatamodel(std::string name, std::string_view definition) { const auto it = std::find_if(m_definitions.cbegin(), m_definitions.cend(), [&name](const auto& kvPair) { return kvPair.first == name; }); @@ -29,11 +29,11 @@ size_t EDMDefinitionRegistry::registerEDM(std::string name, std::string_view def return std::distance(m_definitions.cbegin(), it); } -const std::string_view EDMDefinitionRegistry::getDefinition(std::string_view edmName) const { +const std::string_view DatamodelRegistry::getDatamodelDefinition(std::string_view name) const { const auto it = std::find_if(m_definitions.cbegin(), m_definitions.cend(), - [&edmName](const auto& kvPair) { return kvPair.first == edmName; }); + [&name](const auto& kvPair) { return kvPair.first == name; }); if (it == m_definitions.cend()) { - std::cerr << "PODIO WARNING: Cannot find the definition for the EDM with the name " << edmName << std::endl; + std::cerr << "PODIO WARNING: Cannot find the definition for the EDM with the name " << name << std::endl; static constexpr std::string_view emptyDef = "{}"; // valid empty JSON return emptyDef; } @@ -41,7 +41,7 @@ const std::string_view EDMDefinitionRegistry::getDefinition(std::string_view edm return it->second; } -const std::string_view EDMDefinitionRegistry::getDefinition(size_t index) const { +const std::string_view DatamodelRegistry::getDatamodelDefinition(size_t index) const { if (index >= m_definitions.size()) { std::cerr << "PODIO WARNING: Cannot find the definition for the EDM with the index " << index << std::endl; static constexpr std::string_view emptyDef = "{}"; // valid empty JSON @@ -51,7 +51,7 @@ const std::string_view EDMDefinitionRegistry::getDefinition(size_t index) const return m_definitions[index].second; } -const std::string& EDMDefinitionRegistry::getEDMName(size_t index) const { +const std::string& DatamodelRegistry::getDatamodelName(size_t index) const { if (index >= m_definitions.size()) { std::cout << "PODIO WARNING: Cannot find the name of the EDM with the index " << index << std::endl; static const std::string emptyName = ""; diff --git a/src/EDMRegistryIOHelpers.cc b/src/EDMRegistryIOHelpers.cc index adda98ffe..7875d17e2 100644 --- a/src/EDMRegistryIOHelpers.cc +++ b/src/EDMRegistryIOHelpers.cc @@ -3,31 +3,32 @@ namespace podio { -void EDMDefinitionCollector::registerEDMDef(const podio::CollectionBase* coll, const std::string& name) { - const auto edmIndex = coll->getDefinitionRegistryIndex(); - if (edmIndex == EDMDefinitionRegistry::NoDefinitionAvailable) { +void DatamodelDefinitionCollector::registerDatamodelDefinition(const podio::CollectionBase* coll, + const std::string& name) { + const auto edmIndex = coll->getDatamodelRegistryIndex(); + if (edmIndex == DatamodelRegistry::NoDefinitionAvailable) { std::cerr << "No EDM definition available for collection " << name << std::endl; } else { - if (edmIndex != EDMDefinitionRegistry::NoDefinitionNecessary) { + if (edmIndex != DatamodelRegistry::NoDefinitionNecessary) { m_edmDefRegistryIdcs.insert(edmIndex); } } } -std::vector> EDMDefinitionCollector::getEDMDefinitionsToWrite() const { +std::vector> DatamodelDefinitionCollector::getDatamodelDefinitionsToWrite() const { std::vector> edmDefinitions; edmDefinitions.reserve(m_edmDefRegistryIdcs.size()); for (const auto& index : m_edmDefRegistryIdcs) { - const auto& edmRegistry = podio::EDMDefinitionRegistry::instance(); - edmDefinitions.emplace_back(edmRegistry.getEDMName(index), edmRegistry.getDefinition(index)); + const auto& edmRegistry = podio::DatamodelRegistry::instance(); + edmDefinitions.emplace_back(edmRegistry.getDatamodelName(index), edmRegistry.getDatamodelDefinition(index)); } return edmDefinitions; } -const std::string_view EDMDefinitionHolder::getEDMDefinition(const std::string& edmName) const { +const std::string_view DatamodelDefinitionHolder::getDatamodelDefinition(const std::string& name) const { const auto it = std::find_if(m_availEDMDefs.cbegin(), m_availEDMDefs.cend(), - [&edmName](const auto& entry) { return std::get<0>(entry) == edmName; }); + [&name](const auto& entry) { return std::get<0>(entry) == name; }); if (it != m_availEDMDefs.cend()) { return std::get<1>(*it); @@ -36,7 +37,7 @@ const std::string_view EDMDefinitionHolder::getEDMDefinition(const std::string& return "{}"; } -std::vector EDMDefinitionHolder::getAvailableEDMDefinitions() const { +std::vector DatamodelDefinitionHolder::getAvailableDatamodels() const { std::vector defs{}; defs.reserve(m_availEDMDefs.size()); std::transform(m_availEDMDefs.cbegin(), m_availEDMDefs.cend(), std::back_inserter(defs), diff --git a/src/ROOTFrameWriter.cc b/src/ROOTFrameWriter.cc index 710f98e6b..ace7e27e8 100644 --- a/src/ROOTFrameWriter.cc +++ b/src/ROOTFrameWriter.cc @@ -36,7 +36,7 @@ void ROOTFrameWriter::writeFrame(const podio::Frame& frame, const std::string& c auto* coll = frame.getCollectionForWrite(name); collections.emplace_back(name, const_cast(coll)); - registerEDMDef(coll, name); + registerDatamodelDefinition(coll, name); } // We will at least have a parameters branch, even if there are no @@ -131,7 +131,7 @@ void ROOTFrameWriter::finish() { auto podioVersion = podio::version::build_version; metaTree->Branch(root_utils::versionBranchName, &podioVersion); - auto edmDefinitions = getEDMDefinitionsToWrite(); + auto edmDefinitions = getDatamodelDefinitionsToWrite(); metaTree->Branch(root_utils::edmDefBranchName, &edmDefinitions); metaTree->Fill(); diff --git a/src/SIOFrameWriter.cc b/src/SIOFrameWriter.cc index 4289aed05..d65eca675 100644 --- a/src/SIOFrameWriter.cc +++ b/src/SIOFrameWriter.cc @@ -1,7 +1,7 @@ #include "podio/SIOFrameWriter.h" #include "podio/CollectionBase.h" #include "podio/CollectionIDTable.h" -#include "podio/EDMDefinitionRegistry.h" +// #include "podio/DatatypeRegistry.h" #include "podio/Frame.h" #include "podio/GenericParameters.h" #include "podio/SIOBlock.h" @@ -37,7 +37,7 @@ void SIOFrameWriter::writeFrame(const podio::Frame& frame, const std::string& ca collections.reserve(collsToWrite.size()); for (const auto& name : collsToWrite) { collections.emplace_back(name, frame.getCollectionForWrite(name)); - registerEDMDef(collections.back().second, name); + registerDatamodelDefinition(collections.back().second, name); } // Write necessary metadata and the actual data into two different records. @@ -52,7 +52,7 @@ void SIOFrameWriter::writeFrame(const podio::Frame& frame, const std::string& ca } void SIOFrameWriter::finish() { - auto edmDefMap = std::make_shared>(getEDMDefinitionsToWrite()); + auto edmDefMap = std::make_shared>(getDatamodelDefinitionsToWrite()); sio::block_list blocks; blocks.push_back(edmDefMap); From ca90a1fd457435fb199826f3516b7d14da409ab6 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 2 Mar 2023 18:24:10 +0100 Subject: [PATCH 15/19] Fix python bindings and rename tests --- python/podio/base_reader.py | 16 ++++++++-------- tests/CMakeLists.txt | 20 ++++++++++---------- tools/podio-dump | 6 +++--- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/python/podio/base_reader.py b/python/podio/base_reader.py index c624696ee..88d3acc3e 100644 --- a/python/podio/base_reader.py +++ b/python/podio/base_reader.py @@ -57,21 +57,21 @@ def is_legacy(self): return self._is_legacy @property - def edm_definitions(self): - """Get the available EDM definitions from this reader. + def datamodel_definitions(self): + """Get the available datamodel definitions from this reader. Returns: - tuple(str): The names of the available EDM definitions + tuple(str): The names of the available datamodel definitions """ if self._is_legacy: return () - return tuple(n.c_str() for n in self._reader.getAvailableEDMDefinitions()) + return tuple(n.c_str() for n in self._reader.getAvailableDatamodels()) - def get_edm_definition(self, edm_name): - """Get the EDM definition of the passed EDM as JSON string. + def get_datamodel_definition(self, edm_name): + """Get the datamodel definition as JSON string. Args: - str: The name of the EDM + str: The name of the datamodel Returns: str: The complete model definition in JSON format. Use, e.g. json.loads @@ -79,4 +79,4 @@ def get_edm_definition(self, edm_name): """ if self._is_legacy: return "" - return self._reader.getEDMDefinition(edm_name).data() + return self._reader.getDatamodelDefinition(edm_name).data() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cac189bec..2b056bc74 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -248,26 +248,26 @@ endif() # Add tests for storing and retrieving the EDM definitions into the produced # files -add_test(edm_def_store_roundtrip_root ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.root datamodel) -add_test(edm_def_store_roundtrip_root_extension ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.root datamodel extension_datamodel) +add_test(datamodel_def_store_roundtrip_root ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.root datamodel) +add_test(datamodel_def_store_roundtrip_root_extension ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.root datamodel extension_datamodel) # Need the input files that are produced by other tests set_tests_properties( - edm_def_store_roundtrip_root - edm_def_store_roundtrip_root_extension + datamodel_def_store_roundtrip_root + datamodel_def_store_roundtrip_root_extension PROPERTIES DEPENDS write_frame_root ) set(sio_roundtrip_tests "") if (TARGET read_sio) - add_test(edm_def_store_roundtrip_sio ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.sio datamodel) - add_test(edm_def_store_roundtrip_sio_extension ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.sio datamodel extension_datamodel) + add_test(datamodel_def_store_roundtrip_sio ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.sio datamodel) + add_test(datamodel_def_store_roundtrip_sio_extension ${CMAKE_CURRENT_LIST_DIR}/scripts/dumpModelRoundTrip.sh ${CMAKE_CURRENT_BINARY_DIR}/example_frame.sio datamodel extension_datamodel) set(sio_roundtrip_tests - edm_def_store_roundtrip_sio - edm_def_store_roundtrip_sio_extension + datamodel_def_store_roundtrip_sio + datamodel_def_store_roundtrip_sio_extension ) set_tests_properties( @@ -281,8 +281,8 @@ endif() list(JOIN PODIO_IO_HANDLERS " " IO_HANDLERS) set_tests_properties( - edm_def_store_roundtrip_root - edm_def_store_roundtrip_root_extension + datamodel_def_store_roundtrip_root + datamodel_def_store_roundtrip_root_extension ${sio_roundtrip_tests} PROPERTIES WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} diff --git a/tools/podio-dump b/tools/podio-dump index d7e951c4b..8685aa19e 100755 --- a/tools/podio-dump +++ b/tools/podio-dump @@ -19,7 +19,7 @@ def print_general_info(reader, filename): """ legacy_text = ' (this is a legacy file!)' if reader.is_legacy else '' print(f'input file: {filename}{legacy_text}\n') - print(f'EDM model definitions stored in this file: {", ".join(reader.edm_definitions)}') + print(f'datamodel model definitions stored in this file: {", ".join(reader.datamodel_definitions)}') print() print('Frame categories in this file:') print(f'{"Name":<20} {"Entries":<10}') @@ -74,11 +74,11 @@ def print_frame(frame, cat_name, ientry, detailed): def dump_model(reader, model_name): """Dump the model in yaml format""" - if model_name not in reader.edm_definitions: + if model_name not in reader.datamodel_definitions: print(f'ERROR: Cannot dump model \'{model_name}\' (not present in file)') return False - model_def = json.loads(reader.get_edm_definition(model_name)) + model_def = json.loads(reader.get_datamodel_definition(model_name)) print(yaml.dump(model_def, sort_keys=False, default_flow_style=False)) return True From 07f506ad064b61706e755b36a20489e38e134114 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 2 Mar 2023 18:37:48 +0100 Subject: [PATCH 16/19] Make utility classes members instead of using them as mixin --- include/podio/ROOTFrameReader.h | 13 +++++++++++- include/podio/ROOTFrameWriter.h | 4 +++- include/podio/SIOFrameReader.h | 14 ++++++++++++- include/podio/SIOFrameWriter.h | 3 ++- .../podio/utilities/EDMRegistryIOHelpers.h | 21 +++++++++++++++---- src/ROOTFrameReader.cc | 7 +++++-- src/ROOTFrameWriter.cc | 4 ++-- src/SIOFrameReader.cc | 5 +++-- src/SIOFrameWriter.cc | 5 +++-- 9 files changed, 60 insertions(+), 16 deletions(-) diff --git a/include/podio/ROOTFrameReader.h b/include/podio/ROOTFrameReader.h index 7df493096..d3040d5b9 100644 --- a/include/podio/ROOTFrameReader.h +++ b/include/podio/ROOTFrameReader.h @@ -41,7 +41,7 @@ struct CollectionReadBuffers; * This class has the function to read available data from disk * and to prepare collections and buffers. **/ -class ROOTFrameReader : public DatamodelDefinitionHolder { +class ROOTFrameReader { public: ROOTFrameReader() = default; @@ -80,6 +80,16 @@ class ROOTFrameReader : public DatamodelDefinitionHolder { /// Get the names of all the availalable Frame categories in the current file(s) std::vector getAvailableCategories() const; + /// Get the datamodel definition for the given name + const std::string_view getDatamodelDefinition(const std::string& name) const { + return m_datamodelHolder.getDatamodelDefinition(name); + } + + /// Get all names of the datamodels that ara available from this reader + std::vector getAvailableDatamodels() const { + return m_datamodelHolder.getAvailableDatamodels(); + } + private: /** * Helper struct to group together all the necessary state to read / process a @@ -133,6 +143,7 @@ class ROOTFrameReader : public DatamodelDefinitionHolder { std::vector m_availCategories{}; ///< All available categories from this file podio::version::Version m_fileVersion{0, 0, 0}; + DatamodelDefinitionHolder m_datamodelHolder{}; }; } // namespace podio diff --git a/include/podio/ROOTFrameWriter.h b/include/podio/ROOTFrameWriter.h index cae0e7bba..5680baf1d 100644 --- a/include/podio/ROOTFrameWriter.h +++ b/include/podio/ROOTFrameWriter.h @@ -21,7 +21,7 @@ class Frame; class CollectionBase; class GenericParameters; -class ROOTFrameWriter : DatamodelDefinitionCollector { +class ROOTFrameWriter { public: ROOTFrameWriter(const std::string& filename); ~ROOTFrameWriter() = default; @@ -81,6 +81,8 @@ class ROOTFrameWriter : DatamodelDefinitionCollector { std::unique_ptr m_file{nullptr}; ///< The storage file std::unordered_map m_categories{}; ///< All categories + + DatamodelDefinitionCollector m_datamodelCollector{}; }; } // namespace podio diff --git a/include/podio/SIOFrameReader.h b/include/podio/SIOFrameReader.h index 51aeae8b3..f89d77280 100644 --- a/include/podio/SIOFrameReader.h +++ b/include/podio/SIOFrameReader.h @@ -17,7 +17,7 @@ namespace podio { class CollectionIDTable; -class SIOFrameReader : public DatamodelDefinitionHolder { +class SIOFrameReader { public: SIOFrameReader(); @@ -54,6 +54,16 @@ class SIOFrameReader : public DatamodelDefinitionHolder { /// Get the names of all the availalable Frame categories in the current file(s) std::vector getAvailableCategories() const; + /// Get the datamodel definition for the given name + const std::string_view getDatamodelDefinition(const std::string& name) const { + return m_datamodelHolder.getDatamodelDefinition(name); + } + + /// Get all names of the datamodels that ara available from this reader + std::vector getAvailableDatamodels() const { + return m_datamodelHolder.getAvailableDatamodels(); + } + private: void readPodioHeader(); @@ -71,6 +81,8 @@ class SIOFrameReader : public DatamodelDefinitionHolder { SIOFileTOCRecord m_tocRecord{}; /// The podio version that has been used to write the file podio::version::Version m_fileVersion{0}; + + DatamodelDefinitionHolder m_datamodelHolder{}; }; } // namespace podio diff --git a/include/podio/SIOFrameWriter.h b/include/podio/SIOFrameWriter.h index 5e4b95328..988640790 100644 --- a/include/podio/SIOFrameWriter.h +++ b/include/podio/SIOFrameWriter.h @@ -14,7 +14,7 @@ namespace podio { class Frame; -class SIOFrameWriter : DatamodelDefinitionCollector { +class SIOFrameWriter { public: SIOFrameWriter(const std::string& filename); ~SIOFrameWriter() = default; @@ -36,6 +36,7 @@ class SIOFrameWriter : DatamodelDefinitionCollector { private: sio::ofstream m_stream{}; ///< The output file stream SIOFileTOCRecord m_tocRecord{}; ///< The "table of contents" of the written file + DatamodelDefinitionCollector m_datamodelCollector{}; }; } // namespace podio diff --git a/include/podio/utilities/EDMRegistryIOHelpers.h b/include/podio/utilities/EDMRegistryIOHelpers.h index fe8881ea6..89f314678 100644 --- a/include/podio/utilities/EDMRegistryIOHelpers.h +++ b/include/podio/utilities/EDMRegistryIOHelpers.h @@ -12,7 +12,7 @@ namespace podio { /** - * Helper class (mixin) to collect the datamodel (JSON) definitions that should be + * Helper class to collect the datamodel (JSON) definitions that should be * written. */ class DatamodelDefinitionCollector { @@ -34,11 +34,24 @@ class DatamodelDefinitionCollector { }; /** - * Helper class (mixin) to hold and provide the datamodel (JSON) definitions for - * reader classes. + * Helper class to hold and provide the datamodel (JSON) definitions for reader + * classes. */ class DatamodelDefinitionHolder { public: + /// The "map" type that is used internally + using MapType = std::vector>; + /// Constructor from an existing collection of names and datamodel definitions + DatamodelDefinitionHolder(MapType&& definitions) : m_availEDMDefs(std::move(definitions)) { + } + + DatamodelDefinitionHolder() = default; + ~DatamodelDefinitionHolder() = default; + DatamodelDefinitionHolder(const DatamodelDefinitionHolder&) = delete; + DatamodelDefinitionHolder& operator=(const DatamodelDefinitionHolder&) = delete; + DatamodelDefinitionHolder(DatamodelDefinitionHolder&&) = default; + DatamodelDefinitionHolder& operator=(DatamodelDefinitionHolder&&) = default; + /** * Get the datamodel definition for the given datamodel name. * @@ -55,7 +68,7 @@ class DatamodelDefinitionHolder { std::vector getAvailableDatamodels() const; protected: - std::vector> m_availEDMDefs{}; + MapType m_availEDMDefs{}; }; } // namespace podio diff --git a/src/ROOTFrameReader.cc b/src/ROOTFrameReader.cc index ae124a46a..419051696 100644 --- a/src/ROOTFrameReader.cc +++ b/src/ROOTFrameReader.cc @@ -3,6 +3,7 @@ #include "podio/CollectionBuffers.h" #include "podio/CollectionIDTable.h" #include "podio/GenericParameters.h" +#include "podio/utilities/EDMRegistryIOHelpers.h" #include "rootUtils.h" // ROOT specific includes @@ -217,10 +218,12 @@ void ROOTFrameReader::openFiles(const std::vector& filenames) { m_fileVersion = versionPtr ? *versionPtr : podio::version::Version{0, 0, 0}; delete versionPtr; - auto* edmDefs = &m_availEDMDefs; if (auto* edmDefBranch = root_utils::getBranch(m_metaChain.get(), root_utils::edmDefBranchName)) { - edmDefBranch->SetAddress(&edmDefs); + auto* datamodelDefs = new DatamodelDefinitionHolder::MapType{}; + edmDefBranch->SetAddress(&datamodelDefs); edmDefBranch->GetEntry(0); + m_datamodelHolder = DatamodelDefinitionHolder(std::move(*datamodelDefs)); + delete datamodelDefs; } // Do some work up front for setting up categories and setup all the chains diff --git a/src/ROOTFrameWriter.cc b/src/ROOTFrameWriter.cc index ace7e27e8..3f552d69f 100644 --- a/src/ROOTFrameWriter.cc +++ b/src/ROOTFrameWriter.cc @@ -36,7 +36,7 @@ void ROOTFrameWriter::writeFrame(const podio::Frame& frame, const std::string& c auto* coll = frame.getCollectionForWrite(name); collections.emplace_back(name, const_cast(coll)); - registerDatamodelDefinition(coll, name); + m_datamodelCollector.registerDatamodelDefinition(coll, name); } // We will at least have a parameters branch, even if there are no @@ -131,7 +131,7 @@ void ROOTFrameWriter::finish() { auto podioVersion = podio::version::build_version; metaTree->Branch(root_utils::versionBranchName, &podioVersion); - auto edmDefinitions = getDatamodelDefinitionsToWrite(); + auto edmDefinitions = m_datamodelCollector.getDatamodelDefinitionsToWrite(); metaTree->Branch(root_utils::edmDefBranchName, &edmDefinitions); metaTree->Fill(); diff --git a/src/SIOFrameReader.cc b/src/SIOFrameReader.cc index 7bd3a93e0..985c16a8b 100644 --- a/src/SIOFrameReader.cc +++ b/src/SIOFrameReader.cc @@ -1,6 +1,7 @@ #include "podio/SIOFrameReader.h" #include "podio/SIOBlock.h" +#include "podio/utilities/EDMRegistryIOHelpers.h" #include "sioUtils.h" #include @@ -123,8 +124,8 @@ void SIOFrameReader::readEDMDefinitions() { blocks.emplace_back(std::make_shared>()); sio::api::read_blocks(buffer.span(), blocks); - auto edmDefs = static_cast*>(blocks[0].get()); - m_availEDMDefs = std::move(edmDefs->mapData); + auto datamodelDefs = static_cast*>(blocks[0].get()); + m_datamodelHolder = DatamodelDefinitionHolder(std::move(datamodelDefs->mapData)); } } // namespace podio diff --git a/src/SIOFrameWriter.cc b/src/SIOFrameWriter.cc index d65eca675..4d4cf00bf 100644 --- a/src/SIOFrameWriter.cc +++ b/src/SIOFrameWriter.cc @@ -37,7 +37,7 @@ void SIOFrameWriter::writeFrame(const podio::Frame& frame, const std::string& ca collections.reserve(collsToWrite.size()); for (const auto& name : collsToWrite) { collections.emplace_back(name, frame.getCollectionForWrite(name)); - registerDatamodelDefinition(collections.back().second, name); + m_datamodelCollector.registerDatamodelDefinition(collections.back().second, name); } // Write necessary metadata and the actual data into two different records. @@ -52,7 +52,8 @@ void SIOFrameWriter::writeFrame(const podio::Frame& frame, const std::string& ca } void SIOFrameWriter::finish() { - auto edmDefMap = std::make_shared>(getDatamodelDefinitionsToWrite()); + auto edmDefMap = std::make_shared>( + m_datamodelCollector.getDatamodelDefinitionsToWrite()); sio::block_list blocks; blocks.push_back(edmDefMap); From 5f1a3c4dd4637175d8987a7b06f9ab084326b66d Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 2 Mar 2023 21:07:19 +0100 Subject: [PATCH 17/19] Rename header file to match new terminology --- include/podio/ROOTFrameReader.h | 2 +- include/podio/ROOTFrameWriter.h | 2 +- include/podio/SIOFrameReader.h | 2 +- include/podio/SIOFrameWriter.h | 2 +- ...{EDMRegistryIOHelpers.h => DatamodelRegistryIOHelpers.h} | 6 +++--- src/CMakeLists.txt | 4 ++-- ...DMRegistryIOHelpers.cc => DatamodelRegistryIOHelpers.cc} | 2 +- src/ROOTFrameReader.cc | 1 - src/SIOFrameReader.cc | 1 - src/SIOFrameWriter.cc | 1 - 10 files changed, 10 insertions(+), 13 deletions(-) rename include/podio/utilities/{EDMRegistryIOHelpers.h => DatamodelRegistryIOHelpers.h} (93%) rename src/{EDMRegistryIOHelpers.cc => DatamodelRegistryIOHelpers.cc} (96%) diff --git a/include/podio/ROOTFrameReader.h b/include/podio/ROOTFrameReader.h index d3040d5b9..1a2f48a4d 100644 --- a/include/podio/ROOTFrameReader.h +++ b/include/podio/ROOTFrameReader.h @@ -4,7 +4,7 @@ #include "podio/CollectionBranches.h" #include "podio/ROOTFrameData.h" #include "podio/podioVersion.h" -#include "podio/utilities/EDMRegistryIOHelpers.h" +#include "podio/utilities/DatamodelRegistryIOHelpers.h" #include "TChain.h" diff --git a/include/podio/ROOTFrameWriter.h b/include/podio/ROOTFrameWriter.h index 5680baf1d..2546613d8 100644 --- a/include/podio/ROOTFrameWriter.h +++ b/include/podio/ROOTFrameWriter.h @@ -3,7 +3,7 @@ #include "podio/CollectionBranches.h" #include "podio/CollectionIDTable.h" -#include "podio/utilities/EDMRegistryIOHelpers.h" +#include "podio/utilities/DatamodelRegistryIOHelpers.h" #include "TFile.h" diff --git a/include/podio/SIOFrameReader.h b/include/podio/SIOFrameReader.h index f89d77280..5fefdab75 100644 --- a/include/podio/SIOFrameReader.h +++ b/include/podio/SIOFrameReader.h @@ -4,7 +4,7 @@ #include "podio/SIOBlock.h" #include "podio/SIOFrameData.h" #include "podio/podioVersion.h" -#include "podio/utilities/EDMRegistryIOHelpers.h" +#include "podio/utilities/DatamodelRegistryIOHelpers.h" #include diff --git a/include/podio/SIOFrameWriter.h b/include/podio/SIOFrameWriter.h index 988640790..a8a7d084f 100644 --- a/include/podio/SIOFrameWriter.h +++ b/include/podio/SIOFrameWriter.h @@ -2,7 +2,7 @@ #define PODIO_SIOFRAMEWRITER_H #include "podio/SIOBlock.h" -#include "podio/utilities/EDMRegistryIOHelpers.h" +#include "podio/utilities/DatamodelRegistryIOHelpers.h" #include diff --git a/include/podio/utilities/EDMRegistryIOHelpers.h b/include/podio/utilities/DatamodelRegistryIOHelpers.h similarity index 93% rename from include/podio/utilities/EDMRegistryIOHelpers.h rename to include/podio/utilities/DatamodelRegistryIOHelpers.h index 89f314678..4ca996ae6 100644 --- a/include/podio/utilities/EDMRegistryIOHelpers.h +++ b/include/podio/utilities/DatamodelRegistryIOHelpers.h @@ -1,5 +1,5 @@ -#ifndef PODIO_UTILITIES_EDMREGISTRYIOHELPERS_H -#define PODIO_UTILITIES_EDMREGISTRYIOHELPERS_H +#ifndef PODIO_UTILITIES_DATAMODELREGISTRYIOHELPERS_H +#define PODIO_UTILITIES_DATAMODELREGISTRYIOHELPERS_H #include "podio/CollectionBase.h" #include "podio/DatamodelRegistry.h" @@ -73,4 +73,4 @@ class DatamodelDefinitionHolder { } // namespace podio -#endif // PODIO_UTILITIES_EDMREGISTRYIOHELPERS_H +#endif // PODIO_UTILITIES_DATAMODELREGISTRYIOHELPERS_H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4525922d4..589c073a6 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -50,7 +50,7 @@ SET(core_sources ASCIIWriter.cc EventStore.cc DatamodelRegistry.cc - EDMRegistryIOHelpers.cc + DatamodelRegistryIOHelpers.cc ) SET(core_headers @@ -63,7 +63,7 @@ SET(core_headers ${CMAKE_SOURCE_DIR}/include/podio/UserDataCollection.h ${CMAKE_SOURCE_DIR}/include/podio/podioVersion.h ${CMAKE_SOURCE_DIR}/include/podio/DatamodelRegistry.h - ${CMAKE_SOURCE_DIR}/include/podio/utilities/EDMRegistryIOHelpers.h + ${CMAKE_SOURCE_DIR}/include/podio/utilities/DatamodelRegistryIOHelpers.h ) PODIO_ADD_LIB_AND_DICT(podio "${core_headers}" "${core_sources}" selection.xml) diff --git a/src/EDMRegistryIOHelpers.cc b/src/DatamodelRegistryIOHelpers.cc similarity index 96% rename from src/EDMRegistryIOHelpers.cc rename to src/DatamodelRegistryIOHelpers.cc index 7875d17e2..901dbb113 100644 --- a/src/EDMRegistryIOHelpers.cc +++ b/src/DatamodelRegistryIOHelpers.cc @@ -1,4 +1,4 @@ -#include "podio/utilities/EDMRegistryIOHelpers.h" +#include "podio/utilities/DatamodelRegistryIOHelpers.h" #include namespace podio { diff --git a/src/ROOTFrameReader.cc b/src/ROOTFrameReader.cc index 419051696..f8880133c 100644 --- a/src/ROOTFrameReader.cc +++ b/src/ROOTFrameReader.cc @@ -3,7 +3,6 @@ #include "podio/CollectionBuffers.h" #include "podio/CollectionIDTable.h" #include "podio/GenericParameters.h" -#include "podio/utilities/EDMRegistryIOHelpers.h" #include "rootUtils.h" // ROOT specific includes diff --git a/src/SIOFrameReader.cc b/src/SIOFrameReader.cc index 985c16a8b..0997ae8dc 100644 --- a/src/SIOFrameReader.cc +++ b/src/SIOFrameReader.cc @@ -1,7 +1,6 @@ #include "podio/SIOFrameReader.h" #include "podio/SIOBlock.h" -#include "podio/utilities/EDMRegistryIOHelpers.h" #include "sioUtils.h" #include diff --git a/src/SIOFrameWriter.cc b/src/SIOFrameWriter.cc index 4d4cf00bf..360c948d2 100644 --- a/src/SIOFrameWriter.cc +++ b/src/SIOFrameWriter.cc @@ -1,7 +1,6 @@ #include "podio/SIOFrameWriter.h" #include "podio/CollectionBase.h" #include "podio/CollectionIDTable.h" -// #include "podio/DatatypeRegistry.h" #include "podio/Frame.h" #include "podio/GenericParameters.h" #include "podio/SIOBlock.h" From 74ef2e698c1df3d2bbef3f736750920021475de2 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Thu, 2 Mar 2023 21:08:59 +0100 Subject: [PATCH 18/19] Update documentation to match implementation again --- doc/advanced_topics.md | 56 ++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/doc/advanced_topics.md b/doc/advanced_topics.md index afa9e2468..c2235f565 100644 --- a/doc/advanced_topics.md +++ b/doc/advanced_topics.md @@ -176,26 +176,34 @@ set to `True` in the yaml file. Since the json encoded definition is generated right before the pre-processed model is passed to the class generator, this definition is equivalent, but not necessarily equal to the original definition. -#### The `EDMDefinitionRegistry` -To make access to the embedded datamodel definitions a bit easier the -`EDMDefinitionRegistry` (singleton) keeps a map of all loaded EDMs and makes -access to this string slightly easier, providing two access methods: - +#### The `DatamodelRegistry` +To make access to information about currently loaded and available datamodels a +bit easier the `DatamodelRegistry` (singleton) keeps a map of all loaded +datamodels and provides access to this information possible. In this context we +refer to an *EDM* as the shared library (and the corresponding public headers) +that have been compiled from code that has been generated from a *datamodel +definition* in the original YAML file. In general whenever we refer to a +*datamodel* in this context we mean the enitity as a whole, i.e. its definition +in a YAML file, the concrete implementation as an EDM, as well as other related +information that is related to it. + +Currently the `DatamodelRegistry` provides mainly access to the original +definition of available datamodels via two methods: ```cpp -const std::string_view getEDMDefinition(const std::string& edmName) const; +const std::string_view getDatamodelDefinition(const std::string& edmName) const; -const std::string_view getEDMDefinition(size_t index) const; +const std::string_view getDatamodelDefinition(size_t index) const; ``` where `index` can be obtained from each collection via -`getDefinitionRegistryIndex`. That in turn simply calls -`::meta::RegistryIndex::value()`, another singleton like object -that takes care of registering an EDM definition to the `EDMDefinitionRegistry` +`getDatamodelRegistryIndex`. That in turn simply calls +`::meta::DatamodelRegistryIndex::value()`, another singleton like object +that takes care of registering an EDM definition to the `DatamodelRegistry` during its static initialization. It is also defined in the `DatamodelDefinition.h` header. -Since the EDM definition is embedded as a raw string literal into the core -datamodel shared library, it is in principle also relatively straight forward to +Since the datamodel definition is embedded as a raw string literal into the core +EDM shared library, it is in principle also relatively straight forward to retrieve it from this library by inspecting the binary, e.g. via ```bash readelf -p .rodata libedm4hep.so | grep options @@ -208,20 +216,20 @@ which will result in something like ``` #### I/O helpers for EDM definition storing -The `podio/utilities/EDMRegistryIOHelpers.h` header defines two utility (mixin) +The `podio/utilities/DatamodelRegistryIOHelpers.h` header defines two utility classes, that help with instrumenting readers and writers with functionality to read and write all the necessary EDM definitions. -- The `EDMDefinitionCollector` is intended to be inherited from by writers. It - essentially collects the EDM definitions of all the collections it encounters. - The `registerEDMDef` method it provides should be called with every collection - that is written. The `getEDMDefinitionsToWrite` method returns a vector of all - EDM names and their definition that were encountered during writing. **It is +- The `DatamodelDefinitionCollector` is intended for usage in writers. It + essentially collects the datamodel definitions of all the collections it encounters. + The `registerDatamodelDefinition` method it provides should be called with every collection + that is written. The `getDatamodelDefinitionsToWrite` method returns a vector of all + datamodel names and their definition that were encountered during writing. **It is then the writers responsibility to actually store this information into the file**. -- The `EDMDefinitionHolder` is intended to be inherited from by readers (needs - to be `public` inheritance if the functionality should be directly exposed to - users of the reader). It provides the `getEDMDefinition` and - `getAvailableEDMDefinitions` methods. In order for it to work properly **its - `m_availEDMDefs` member has to be populated by the reader** before a possible - call to either of these two methods. +- The `DatamodelDefinitionHolder` is intended to be used by readers. It + provides the `getDatamodelDefinition` and `getAvailableDatamodels` methods. + **It is again the readers property to correctly populate it with the data it + has read from file.** Currently the `SIOFrameReader` and the `ROOTFrameReader` + use it and also offer the same functionality as public methods with the help + of it. From 465416d4a7408d9e61e3e6fab5bcbe4b3167a28d Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Fri, 3 Mar 2023 09:11:16 +0100 Subject: [PATCH 19/19] Update test names also in ignored tests list --- tests/CTestCustom.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/CTestCustom.cmake b/tests/CTestCustom.cmake index 214fbc88f..71812378a 100644 --- a/tests/CTestCustom.cmake +++ b/tests/CTestCustom.cmake @@ -53,10 +53,10 @@ if ((NOT "@FORCE_RUN_ALL_TESTS@" STREQUAL "ON") AND (NOT "@USE_SANITIZER@" STREQ podio-dump-detailed-sio podio-dump-detailed-sio-legacy - edm_def_store_roundtrip_root - edm_def_store_roundtrip_root_extension - edm_def_store_roundtrip_sio - edm_def_store_roundtrip_sio_extension + datamodel_def_store_roundtrip_root + datamodel_def_store_roundtrip_root_extension + datamodel_def_store_roundtrip_sio + datamodel_def_store_roundtrip_sio_extension ) # ostream_operator is working with Memory sanitizer (at least locally)