Add a generic reference histogram comparator (#2317)

* Add a generic reference data comparator

The code introduces some tools that allow to compare the plots produced by given task(s) with that of a reference run:
- `ReferenceComparatorTask` draws histograms with their reference superimposed, as well as the ratio between histograms and reference
- `ReferenceComparatorCheck` compares the histograms with their corresponding reference, and assesses their compatibility. The comparison is performed by a dynamically loadable module that provides the actual comparison algorithm
- `ReferenceComparatorPlot` is an utility class to draw the current and reference histograms, and their ratio
- `ObjectComparator*` are dynamically loadable modules than implement different histogram comparison methods
  Three examples are provided:
    - `ObjectComparatorDeviation` evaluates the average relative difference between the histogram bins
    - `ObjectComparatorChi2` compares the histograms using a chi2-based test
    - `ObjectComparatorKolmogorov` compares the histograms using a Kolmogorov test

* [Common] implement review comments

Modules/Common/CMakeLists.txt

@@ -22,6 +22,14 @@ target_sources(O2QcCommon
  src/CcdbInspectorTask.cxx
  src/CcdbInspectorTaskConfig.cxx
  src/CcdbInspectorCheck.cxx
+ src/ObjectComparatorInterface.cxx
+ src/ObjectComparatorDeviation.cxx
+ src/ObjectComparatorChi2.cxx
+ src/ObjectComparatorKolmogorov.cxx
+ src/ReferenceComparatorPlot.cxx
+ src/ReferenceComparatorTask.cxx
+ src/ReferenceComparatorTaskConfig.cxx
+ src/ReferenceComparatorCheck.cxx
  src/NonEmpty.cxx
  src/MeanIsAbove.cxx
  src/TH1Reductor.cxx
@@ -54,6 +62,12 @@ add_root_dictionary(O2QcCommon
  include/Common/BigScreen.h
  include/Common/CcdbInspectorTask.h
  include/Common/CcdbInspectorCheck.h
+ include/Common/ObjectComparatorInterface.h
+ include/Common/ObjectComparatorDeviation.h
+ include/Common/ObjectComparatorChi2.h
+ include/Common/ObjectComparatorKolmogorov.h
+ include/Common/ReferenceComparatorTask.h
+ include/Common/ReferenceComparatorCheck.h
  include/Common/MeanIsAbove.h
  include/Common/TH1Ratio.h
  include/Common/TH2Ratio.h

Modules/Common/etc/reference-comparator-example.json

@@ -0,0 +1,98 @@
+{
+ "qc": {
+ "config": {
+ "database": {
+ "implementation": "CCDB",
+ "host": "ccdb-test.cern.ch:8080",
+ "username": "not_applicable",
+ "password": "not_applicable",
+ "name": "not_applicable"
+ },
+ "Activity": {
+ },
+ "monitoring": {
+ "url": "infologger:///debug?qc"
+ },
+ "consul": {
+ "url": ""
+ },
+ "conditionDB": {
+ "url": "ccdb-test.cern.ch:8080"
+ }
+ },
+ "postprocessing": {
+ "ExampleRefComp": {
+ "active": "true",
+ "className": "o2::quality_control_modules::common::ReferenceComparatorTask",
+ "moduleName": "QualityControl",
+ "detectorName": "MCH",
+ "extendedTaskParameters": {
+ "default": {
+ "default": {
+ "notOlderThan" : "300",
+ "referenceRun" : "551875"
+ }
+ },
+ "PHYSICS": {
+ "PROTON-PROTON": {
+ "referenceRun" : "551890"
+ }
+ }
+ },
+ "dataGroups": [
+ {
+ "name": "Tracks",
+ "inputPath": "MCH/MO/Tracks/WithCuts",
+ "referencePath": "MCH/MO/Tracks",
+ "outputPath": "Tracks/WithCuts",
+ "normalizeReference": "true",
+ "drawRatioOnly": "false",
+ "drawOption1D": "E",
+ "drawOption2D": "COL",
+ "inputObjects": [
+ "TrackEta",
+ "TrackEtaPhi"
+ ]
+ }
+ ],
+ "initTrigger": [
+ "userorcontrol"
+ ],
+ "updateTrigger": [
+ "60 seconds"
+ ],
+ "stopTrigger": [
+ "userorcontrol"
+ ]
+ }
+ },
+ "checks": {
+ "ExampleRefCheck": {
+ "active": "true",
+ "className": "o2::quality_control_modules::common::ReferenceComparatorCheck",
+ "moduleName": "QualityControl",
+ "detectorName": "MCH",
+ "policy": "OnAny",
+ "extendedCheckParameters": {
+ "default": { 
+ "default": { 
+ "moduleName" : "QualityControl",
+ "comparatorName" : "o2::quality_control_modules::common::ObjectComparatorChi2",
+ "threshold" : "0.5"
+ }
+ }
+ },
+ "dataSource": [
+ {
+ "type": "PostProcessing",
+ "name": "ExampleRefComp",
+ "MOs" : [
+ "Tracks/WithCuts/TrackEta",
+ "Tracks/WithCuts/TrackEtaPhi"
+ ]
+ }
+ ]
+ }
+ }
+ }
+}

Modules/Common/include/Common/LinkDef.h

@@ -19,6 +19,11 @@
 #pragma link C++ class o2::quality_control_modules::common::BigScreen + ;
 #pragma link C++ class o2::quality_control_modules::common::CcdbInspectorTask + ;
 #pragma link C++ class o2::quality_control_modules::common::CcdbInspectorCheck + ;
+#pragma link C++ class o2::quality_control_modules::common::ObjectComparatorDeviation + ;
+#pragma link C++ class o2::quality_control_modules::common::ObjectComparatorChi2 + ;
+#pragma link C++ class o2::quality_control_modules::common::ObjectComparatorKolmogorov + ;
+#pragma link C++ class o2::quality_control_modules::common::ReferenceComparatorTask + ;
+#pragma link C++ class o2::quality_control_modules::common::ReferenceComparatorCheck + ;
 #pragma link C++ class o2::quality_control_modules::common::WorstOfAllAggregator + ;
 #pragma link C++ class o2::quality_control_modules::common::IncreasingEntries + ;
 #pragma link C++ class o2::quality_control_modules::common::TH1SliceReductor + ;

Modules/Common/include/Common/ObjectComparatorChi2.h

@@ -0,0 +1,42 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+///
+/// \file ObjectComparatorChi2.h
+/// \author Andrea Ferrero
+/// \brief A class for comparing two histogram objects based on a chi2 test
+///
+
+#ifndef QUALITYCONTROL_ObjectComparatorChi2_H
+#define QUALITYCONTROL_ObjectComparatorChi2_H
+
+#include "Common/ObjectComparatorInterface.h"
+
+namespace o2::quality_control_modules::common
+{
+
+/// \brief A class for comparing two histogram objects based on a chi2 test
+class ObjectComparatorChi2 : public ObjectComparatorInterface
+{
+ public:
+ /// \brief Constructor
+ ObjectComparatorChi2() = default;
+ /// \brief Destructor
+ virtual ~ObjectComparatorChi2() = default;
+
+ /// \brief objects comparison function
+ /// \return the quality resulting from the object comparison
+ o2::quality_control::core::Quality compare(TObject* object, TObject* referenceObject, std::string& message) override;
+};
+
+} // namespace o2::quality_control_modules::common
+
+#endif // QUALITYCONTROL_ObjectComparatorChi2_H

Modules/Common/include/Common/ObjectComparatorDeviation.h

@@ -0,0 +1,42 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+///
+/// \file ObjectComparatorDeviation.h
+/// \author Andrea Ferrero
+/// \brief A class for comparing two histogram objects based on the average of the relative deviation between the bins
+///
+
+#ifndef QUALITYCONTROL_ObjectComparatorDeviation_H
+#define QUALITYCONTROL_ObjectComparatorDeviation_H
+
+#include "Common/ObjectComparatorInterface.h"
+
+namespace o2::quality_control_modules::common
+{
+
+/// \brief A class for comparing two histogram objects based on the average of the relative deviation between the bins
+class ObjectComparatorDeviation : public ObjectComparatorInterface
+{
+ public:
+ /// \brief Constructor
+ ObjectComparatorDeviation() = default;
+ /// \brief Destructor
+ virtual ~ObjectComparatorDeviation() = default;
+
+ /// \brief objects comparison function
+ /// \return the quality resulting from the object comparison
+ o2::quality_control::core::Quality compare(TObject* object, TObject* referenceObject, std::string& message) override;
+};
+
+} // namespace o2::quality_control_modules::common
+
+#endif // QUALITYCONTROL_ObjectComparatorDeviation_H

Modules/Common/include/Common/ObjectComparatorInterface.h

@@ -0,0 +1,64 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+///
+/// \file ObjectComparatorInterface.h
+/// \author Andrea Ferrero
+/// \brief An interface for comparing two TObject
+///
+
+#ifndef QUALITYCONTROL_ObjectComparatorInterface_H
+#define QUALITYCONTROL_ObjectComparatorInterface_H
+
+#include "QualityControl/Quality.h"
+#include "QualityControl/CustomParameters.h"
+#include "QualityControl/Activity.h"
+
+#include <tuple>
+
+class TObject;
+class TH1;
+
+namespace o2::quality_control_modules::common
+{
+
+/// \brief An interface for comparing two TObject
+class ObjectComparatorInterface
+{
+ public:
+ /// \brief Constructor
+ ObjectComparatorInterface() = default;
+ /// \brief Destructor
+ virtual ~ObjectComparatorInterface() = default;
+
+ /// \brief comparator configuration via CustomParameters
+ // virtual void configure(const o2::quality_control::core::CustomParameters& customParameters, const o2::quality_control::core::Activity activity = {}){};
+
+ /// setter/getter methods for the threshold to define the goodness of the comparison
+ void setThreshold(double threshold) { mThreshold = threshold; }
+ double getThreshold() { return mThreshold; }
+
+ /// perform a number of sanity checks on the input objects
+ /// \return a tuple containing pointers to the histogram, the reference histogram, and a boolean indicating the success of the checks
+ std::tuple<TH1*, TH1*, bool> checkInputObjects(TObject* object, TObject* referenceObject, std::string& message);
+
+ /// \brief objects comparison function
+ /// \return the quality resulting from the object comparison
+ virtual o2::quality_control::core::Quality compare(TObject* object, TObject* referenceObject, std::string& message) = 0;
+
+ private:
+ /// the threshold to define the goodness of the comparison
+ double mThreshold{ 0 };
+};
+
+} // namespace o2::quality_control_modules::common
+
+#endif // QUALITYCONTROL_ObjectComparatorInterface_H

Modules/Common/include/Common/ObjectComparatorKolmogorov.h

@@ -0,0 +1,42 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+///
+/// \file ObjectComparatorKolmogorov.h
+/// \author Andrea Ferrero
+/// \brief A class for comparing two histogram objects based on a Kolmogorov test
+///
+
+#ifndef QUALITYCONTROL_ObjectComparatorKolmogorov_H
+#define QUALITYCONTROL_ObjectComparatorKolmogorov_H
+
+#include "Common/ObjectComparatorInterface.h"
+
+namespace o2::quality_control_modules::common
+{
+
+/// \brief A class for comparing two histogram objects based on a Kolmogorov test
+class ObjectComparatorKolmogorov : public ObjectComparatorInterface
+{
+ public:
+ /// \brief Constructor
+ ObjectComparatorKolmogorov() = default;
+ /// \brief Destructor
+ virtual ~ObjectComparatorKolmogorov() = default;
+
+ /// \brief objects comparison function
+ /// \return the quality resulting from the object comparison
+ o2::quality_control::core::Quality compare(TObject* object, TObject* referenceObject, std::string& message) override;
+};
+
+} // namespace o2::quality_control_modules::common
+
+#endif // QUALITYCONTROL_ObjectComparatorKolmogorov_H

Modules/Common/include/Common/ReferenceComparatorCheck.h

@@ -0,0 +1,59 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+///
+/// \file ReferenceComparatorCheck.h
+/// \author Andrea Ferrero
+/// \brief A generic QC check that compares a given set of histograms with their corresponding references
+///
+
+#ifndef QUALITYCONTROL_ReferenceComparatorCheck_H
+#define QUALITYCONTROL_ReferenceComparatorCheck_H
+
+#include "QualityControl/CheckInterface.h"
+#include "Common/ObjectComparatorInterface.h"
+
+#include <sstream>
+
+class TPaveText;
+
+namespace o2::quality_control_modules::common
+{
+
+/// \brief A generic QC check that compares a given set of histograms with their corresponding references
+/// \author Andrea Ferrero
+class ReferenceComparatorCheck : public o2::quality_control::checker::CheckInterface
+{
+ public:
+ /// Default constructor
+ ReferenceComparatorCheck() = default;
+ /// Destructor
+ ~ReferenceComparatorCheck() override = default;
+
+ // Override interface
+ void configure() override;
+ Quality check(std::map<std::string, std::shared_ptr<MonitorObject>>* moMap) override;
+ void reset() override;
+ void beautify(std::shared_ptr<MonitorObject> mo, Quality checkResult = Quality::Null) override;
+ std::string getAcceptedType() override;
+
+ void startOfActivity(const Activity& activity) override;
+ void endOfActivity(const Activity& activity) override;
+
+ private:
+ std::unique_ptr<ObjectComparatorInterface> mComparator;
+ std::map<std::string, Quality> mQualityFlags;
+ std::map<std::string, std::shared_ptr<TPaveText>> mQualityLabels;
+};
+
+} // namespace o2::quality_control_modules::common
+
+#endif // QC_MODULE_SKELETON_ReferenceComparatorCheck_H