From ff60ee906067e38eb70fe622308898962933696c Mon Sep 17 00:00:00 2001 From: Laura Serksnyte Date: Mon, 23 Sep 2024 18:59:41 +0200 Subject: [PATCH] Update utility for timestamp extraction to optimize it + switches for IDC task what to plot --- Modules/TPC/include/TPC/IDCs.h | 2 + Modules/TPC/include/TPC/Utility.h | 6 ++ Modules/TPC/src/IDCs.cxx | 141 ++++++++++++++++++------------ Modules/TPC/src/Utility.cxx | 34 ++++++- 4 files changed, 126 insertions(+), 57 deletions(-) diff --git a/Modules/TPC/include/TPC/IDCs.h b/Modules/TPC/include/TPC/IDCs.h index f571416027..7dcc6e442a 100644 --- a/Modules/TPC/include/TPC/IDCs.h +++ b/Modules/TPC/include/TPC/IDCs.h @@ -73,6 +73,8 @@ class IDCs : public quality_control::postprocessing::PostProcessingInterface o2::ccdb::CcdbApi mCdbApi; std::string mHost; bool mDoIDCDelta = false; + bool mDoIDC1 = false; + bool mDoFourier = false; std::unique_ptr mIDCZeroScale; std::unique_ptr mIDCZerOverview; std::unique_ptr mIDCZeroSides; diff --git a/Modules/TPC/include/TPC/Utility.h b/Modules/TPC/include/TPC/Utility.h index 18f505200f..59474b5c51 100644 --- a/Modules/TPC/include/TPC/Utility.h +++ b/Modules/TPC/include/TPC/Utility.h @@ -30,6 +30,12 @@ namespace o2::quality_control_modules::tpc { +/// \brief Get boolean configurable from json file +/// This function checks if a configurable is available in the json file and makes sure that different versions of it are accepted (true, TRUE, 1, etc) +/// \param config ConfigurationInterface with prefix set to "" +/// \param id Task id +/// \param property Property name to be looked for in json +bool getPropertyBool(const boost::property_tree::ptree& config, const std::string& id, const std::string property); /// \brief Prepare canvases to publish DalPad data /// This function creates canvases for CalPad data and registers them to be published on the QCG. diff --git a/Modules/TPC/src/IDCs.cxx b/Modules/TPC/src/IDCs.cxx index 128226efc0..4c7715e32a 100644 --- a/Modules/TPC/src/IDCs.cxx +++ b/Modules/TPC/src/IDCs.cxx @@ -111,21 +111,9 @@ void IDCs::configure(const boost::property_tree::ptree& config) mHost = config.get("qc.postprocessing." + id + ".dataSourceURL"); - boost::optional doDeltaExists = config.get_child_optional("qc.postprocessing." + id + ".doIDCDelta"); - if (doDeltaExists) { - auto doDelta = config.get("qc.postprocessing." + id + ".doIDCDelta"); - if (doDelta == "1" || doDelta == "true" || doDelta == "True" || doDelta == "TRUE" || doDelta == "yes") { - mDoIDCDelta = true; - } else if (doDelta == "0" || doDelta == "false" || doDelta == "False" || doDelta == "FALSE" || doDelta == "no") { - mDoIDCDelta = false; - } else { - mDoIDCDelta = false; - ILOG(Warning, Support) << "No valid input for 'doIDCDelta'. Using default value 'false'." << ENDM; - } - } else { - mDoIDCDelta = false; - ILOG(Warning, Support) << "Option 'doIDCDelta' is missing. Using default value 'false'." << ENDM; - } + mDoIDCDelta = getPropertyBool(config, id, "doIDCDelta"); + mDoIDC1 = getPropertyBool(config, id, "doIDC1"); + mDoFourier = getPropertyBool(config, id, "doFourier"); } void IDCs::initialize(Trigger, framework::ServiceRegistryRef) @@ -137,9 +125,6 @@ void IDCs::initialize(Trigger, framework::ServiceRegistryRef) mIDCZeroRadialProf.reset(); mIDCZeroStacksA.reset(); mIDCZeroStacksC.reset(); - mIDCOneSides1D.reset(); - mFourierCoeffsA.reset(); - mFourierCoeffsC.reset(); mIDCZeroScale = std::make_unique("c_sides_IDC0_scale"); mIDCZerOverview = std::make_unique("c_sides_IDC0_overview"); @@ -147,10 +132,11 @@ void IDCs::initialize(Trigger, framework::ServiceRegistryRef) mIDCZeroStacksA = std::make_unique("c_GEMStacks_IDC0_1D_ASide"); mIDCZeroStacksC = std::make_unique("c_GEMStacks_IDC0_1D_CSide"); - mIDCOneSides1D = std::make_unique("c_sides_IDC1_1D"); - - mFourierCoeffsA = std::make_unique("c_FourierCoefficients_1D_ASide"); - mFourierCoeffsC = std::make_unique("c_FourierCoefficients_1D_CSide"); + getObjectsManager()->startPublishing(mIDCZeroScale.get()); + getObjectsManager()->startPublishing(mIDCZerOverview.get()); + getObjectsManager()->startPublishing(mIDCZeroRadialProf.get()); + getObjectsManager()->startPublishing(mIDCZeroStacksA.get()); + getObjectsManager()->startPublishing(mIDCZeroStacksC.get()); if (mDoIDCDelta) { mIDCDeltaStacksA.reset(); @@ -161,32 +147,75 @@ void IDCs::initialize(Trigger, framework::ServiceRegistryRef) getObjectsManager()->startPublishing(mIDCDeltaStacksC.get()); } - getObjectsManager()->startPublishing(mIDCZeroScale.get()); - getObjectsManager()->startPublishing(mIDCZerOverview.get()); - getObjectsManager()->startPublishing(mIDCZeroRadialProf.get()); - getObjectsManager()->startPublishing(mIDCZeroStacksA.get()); - getObjectsManager()->startPublishing(mIDCZeroStacksC.get()); - - getObjectsManager()->startPublishing(mIDCOneSides1D.get()); + if (mDoIDC1) { + mIDCOneSides1D.reset(); + mIDCOneSides1D = std::make_unique("c_sides_IDC1_1D"); + getObjectsManager()->startPublishing(mIDCOneSides1D.get()); + } - getObjectsManager()->startPublishing(mFourierCoeffsA.get()); - getObjectsManager()->startPublishing(mFourierCoeffsC.get()); + if (mDoFourier) { + mFourierCoeffsA.reset(); + mFourierCoeffsC.reset(); + mFourierCoeffsA = std::make_unique("c_FourierCoefficients_1D_ASide"); + mFourierCoeffsC = std::make_unique("c_FourierCoefficients_1D_CSide"); + getObjectsManager()->startPublishing(mFourierCoeffsA.get()); + getObjectsManager()->startPublishing(mFourierCoeffsC.get()); + } } void IDCs::update(Trigger, framework::ServiceRegistryRef) { std::vector availableTimestampsIDCZeroA = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDC0A), 1, mTimestamps["IDCZero"]); std::vector availableTimestampsIDCZeroC = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDC0C), 1, mTimestamps["IDCZero"]); - std::vector availableTimestampsIDCOneA = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDC1A), 1, mTimestamps["IDCOne"]); - std::vector availableTimestampsIDCOneC = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDC1C), 1, mTimestamps["IDCOne"]); - std::vector availableTimestampsFFTA = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDCFourierA), 1, mTimestamps["FourierCoeffs"]); - std::vector availableTimestampsFFTC = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDCFourierC), 1, mTimestamps["FourierCoeffs"]); std::vector availableTimestampsIDCDeltaA{ 0 }; std::vector availableTimestampsIDCDeltaC{ 0 }; + std::vector availableTimestampsIDCOneA{ 0 }; + std::vector availableTimestampsIDCOneC{ 0 }; + std::vector availableTimestampsFFTA{ 0 }; + std::vector availableTimestampsFFTC{ 0 }; + bool timestampFoundForIDCZero = false; + if (availableTimestampsIDCZeroA.size() == 0 || availableTimestampsIDCZeroC.size() == 0) { + ILOG(Warning, Support) << fmt::format("No timstemp found for '{}' produced in the last day.", "IDCZero") << ENDM; + } else { + timestampFoundForIDCZero = true; + } + bool timestampFoundForIDCDelta = false; if (mDoIDCDelta) { availableTimestampsIDCDeltaA = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDCDeltaA), 1, mTimestamps["IDCDelta"]); availableTimestampsIDCDeltaC = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDCDeltaC), 1, mTimestamps["IDCDelta"]); + if (availableTimestampsIDCDeltaA.size() == 0 || availableTimestampsIDCDeltaC.size() == 0) { + ILOG(Warning, Support) << fmt::format("No timstemp found for '{}' produced in the last day.", "IDCDelta") << ENDM; + } else { + timestampFoundForIDCDelta = true; + } + mIDCDeltaStacksA.get()->Clear(); + mIDCDeltaStacksC.get()->Clear(); + } + + bool timestampFoundForIDCOne = false; + if (mDoIDC1) { + availableTimestampsIDCOneA = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDC1A), 1, mTimestamps["IDCOne"]); + availableTimestampsIDCOneC = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDC1C), 1, mTimestamps["IDCOne"]); + if (availableTimestampsIDCOneA.size() == 0 || availableTimestampsIDCOneC.size() == 0) { + ILOG(Warning, Support) << fmt::format("No timstemp found for '{}' produced in the last day.", "IDCOne") << ENDM; + } else { + timestampFoundForIDCOne = true; + } + mIDCOneSides1D.get()->Clear(); + } + + bool timestampFoundForFourier = false; + if (mDoFourier) { + availableTimestampsFFTA = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDCFourierA), 1, mTimestamps["FourierCoeffs"]); + availableTimestampsFFTC = getDataTimestamps(mCdbApi, CDBTypeMap.at(CDBType::CalIDCFourierC), 1, mTimestamps["FourierCoeffs"]); + if (availableTimestampsFFTA.size() == 0 || availableTimestampsFFTC.size() == 0) { + ILOG(Warning, Support) << fmt::format("No timstemp found for '{}' produced in the last day.", "FourierCoeffs") << ENDM; + } else { + timestampFoundForFourier = true; + } + mFourierCoeffsA.get()->Clear(); + mFourierCoeffsC.get()->Clear(); } mIDCZeroScale.get()->Clear(); @@ -194,14 +223,6 @@ void IDCs::update(Trigger, framework::ServiceRegistryRef) mIDCZeroRadialProf.get()->Clear(); mIDCZeroStacksA.get()->Clear(); mIDCZeroStacksC.get()->Clear(); - mIDCOneSides1D.get()->Clear(); - mFourierCoeffsA.get()->Clear(); - mFourierCoeffsC.get()->Clear(); - - if (mDoIDCDelta) { - mIDCDeltaStacksA.get()->Clear(); - mIDCDeltaStacksC.get()->Clear(); - } o2::tpc::IDCZero* idcZeroA = nullptr; o2::tpc::IDCZero* idcZeroC = nullptr; @@ -212,16 +233,24 @@ void IDCs::update(Trigger, framework::ServiceRegistryRef) o2::tpc::FourierCoeff* idcFFTA = nullptr; o2::tpc::FourierCoeff* idcFFTC = nullptr; - idcZeroA = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDC0A), std::map{}, availableTimestampsIDCZeroA[0]); - idcZeroC = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDC0C), std::map{}, availableTimestampsIDCZeroC[0]); - if (mDoIDCDelta) { + if (timestampFoundForIDCZero) { + idcZeroA = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDC0A), std::map{}, availableTimestampsIDCZeroA[0]); + idcZeroC = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDC0C), std::map{}, availableTimestampsIDCZeroC[0]); + } + + if (mDoIDCDelta && timestampFoundForIDCDelta) { idcDeltaA = mCdbApi.retrieveFromTFileAny>(CDBTypeMap.at(CDBType::CalIDCDeltaA), std::map{}, availableTimestampsIDCDeltaA[0]); idcDeltaC = mCdbApi.retrieveFromTFileAny>(CDBTypeMap.at(CDBType::CalIDCDeltaC), std::map{}, availableTimestampsIDCDeltaC[0]); } - idcOneA = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDC1A), std::map{}, availableTimestampsIDCOneA[0]); - idcOneC = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDC1C), std::map{}, availableTimestampsIDCOneC[0]); - idcFFTA = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDCFourierA), std::map{}, availableTimestampsFFTA[0]); - idcFFTC = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDCFourierC), std::map{}, availableTimestampsFFTC[0]); + if (mDoIDC1 && timestampFoundForIDCOne) { + idcOneA = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDC1A), std::map{}, availableTimestampsIDCOneA[0]); + idcOneC = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDC1C), std::map{}, availableTimestampsIDCOneC[0]); + } + + if (mDoFourier && timestampFoundForFourier) { + idcFFTA = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDCFourierA), std::map{}, availableTimestampsFFTA[0]); + idcFFTC = mCdbApi.retrieveFromTFileAny(CDBTypeMap.at(CDBType::CalIDCFourierC), std::map{}, availableTimestampsFFTC[0]); + } if (idcZeroA && idcZeroC) { mCCDBHelper.setIDCZero(idcZeroA, Side::A); @@ -290,10 +319,14 @@ void IDCs::finalize(Trigger, framework::ServiceRegistryRef) getObjectsManager()->stopPublishing(mIDCZeroStacksA.get()); getObjectsManager()->stopPublishing(mIDCZeroStacksC.get()); - getObjectsManager()->stopPublishing(mIDCOneSides1D.get()); + if (mDoIDC1) { + getObjectsManager()->stopPublishing(mIDCOneSides1D.get()); + } - getObjectsManager()->stopPublishing(mFourierCoeffsA.get()); - getObjectsManager()->stopPublishing(mFourierCoeffsC.get()); + if (mDoFourier) { + getObjectsManager()->stopPublishing(mFourierCoeffsA.get()); + getObjectsManager()->stopPublishing(mFourierCoeffsC.get()); + } if (mDoIDCDelta) { getObjectsManager()->stopPublishing(mIDCDeltaStacksA.get()); @@ -301,4 +334,4 @@ void IDCs::finalize(Trigger, framework::ServiceRegistryRef) } } -} // namespace o2::quality_control_modules::tpc +} // namespace o2::quality_control_modules::tpc \ No newline at end of file diff --git a/Modules/TPC/src/Utility.cxx b/Modules/TPC/src/Utility.cxx index d538581e0e..dffb2b4c75 100644 --- a/Modules/TPC/src/Utility.cxx +++ b/Modules/TPC/src/Utility.cxx @@ -31,10 +31,30 @@ #include #include #include +#include namespace o2::quality_control_modules::tpc { +bool getPropertyBool(const boost::property_tree::ptree& config, const std::string& id, const std::string property) +{ + const auto propertyFullName = fmt::format("qc.postprocessing.{}.{}", id, property); + const boost::optional propertyExists = config.get_child_optional(propertyFullName); + if (propertyExists) { + const auto doProperty = config.get(propertyFullName); + if (doProperty == "1" || doProperty == "true" || doProperty == "True" || doProperty == "TRUE" || doProperty == "yes") { + return true; + } else if (doProperty == "0" || doProperty == "false" || doProperty == "False" || doProperty == "FALSE" || doProperty == "no") { + return false; + } else { + ILOG(Warning, Support) << fmt::format("No valid input for '{}'. Using default value 'false'.", property) << ENDM; + } + } else { + ILOG(Warning, Support) << fmt::format("Option '{}' is missing. Using default value 'false'.", property) << ENDM; + } + return false; +} + void addAndPublish(std::shared_ptr objectsManager, std::vector>& canVec, std::vector canvNames, const std::map& metaData) { for (const auto& canvName : canvNames) { @@ -174,10 +194,15 @@ std::vector getDataTimestamps(const o2::ccdb::CcdbApi& cdbApi, const std:: { std::vector outVec{}; std::vector tmpVec{}; - - std::vector fileList = o2::utils::Str::tokenize(cdbApi.list(path.data()), '\n'); + std::vector tmpVec2{}; if (limit == -1) { + + // get the list of files for the latest timestamps up to 1 day ago from the moment the code is running + // added some seconds to upper timestamp limit as the ccdbapi uses creation timestamp, not validity! + const auto to = std::chrono::duration_cast((std::chrono::system_clock::now() + std::chrono::minutes(1)).time_since_epoch()).count(); + const auto from = std::chrono::duration_cast((std::chrono::system_clock::now() + std::chrono::weeks(-2)).time_since_epoch()).count(); + std::vector fileList = o2::utils::Str::tokenize(cdbApi.list(path.data(), false, "text/plain", to, from), '\n'); for (const auto& metaData : fileList) { getTimestamp(metaData, outVec); if (outVec.size() == nFiles) { @@ -185,6 +210,9 @@ std::vector getDataTimestamps(const o2::ccdb::CcdbApi& cdbApi, const std:: } } } else { + // get file list for requested timestamp minus three days + // added some seconds to upper timestamp limit as the ccdbapi uses creation timestamp, not validity! + std::vector fileList = o2::utils::Str::tokenize(cdbApi.list(path.data(), false, "text/plain", limit + 600000, limit - 86400000), '\n'); for (const auto& metaData : fileList) { if (outVec.size() < nFiles) { getTimestamp(metaData, tmpVec); @@ -313,4 +341,4 @@ void retrieveStatistics(std::vector& values, std::vector& errors } } -} // namespace o2::quality_control_modules::tpc +} // namespace o2::quality_control_modules::tpc \ No newline at end of file