From 3615f5fb8fb55a0d3c70df2a3ee143fef4eaa95f Mon Sep 17 00:00:00 2001 From: pnorbert Date: Tue, 23 Jul 2024 19:12:58 -0400 Subject: [PATCH] Fix CampaignReader when a campaign has multiple hosts. Directory indices are global, not per-host indices (#4259) --- source/adios2/engine/campaign/CampaignData.cpp | 5 +++-- source/adios2/engine/campaign/CampaignData.h | 3 ++- .../adios2/engine/campaign/CampaignReader.cpp | 18 +++++++----------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index 62b2825919..5176364a9f 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -61,7 +61,8 @@ static int sqlcb_directory(void *p, int argc, char **argv, char **azColName) CampaignData *cdp = reinterpret_cast(p); size_t hostid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); size_t hostidx = hostid - 1; // SQL rows start from 1, vector idx start from 0 - cdp->hosts[hostidx].directory.push_back(argv[1]); + cdp->directory.push_back(argv[1]); + cdp->hosts[hostidx].dirIdx.push_back(cdp->directory.size() - 1); return 0; }; @@ -73,7 +74,7 @@ static int sqlcb_bpdataset(void *p, int argc, char **argv, char **azColName) size_t hostid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int"); size_t dirid = helper::StringToSizeT(std::string(argv[2]), "SQL callback convert text to int"); cds.hostIdx = hostid - 1; // SQL rows start from 1, vector idx start from 0 - cds.dirIdx = dirid - 1; + cds.dirIdx = dirid - 1; // SQL rows start from 1, vector idx start from 0 cds.name = argv[3]; cdp->bpdatasets[dsid] = cds; return 0; diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h index fa316c1bcb..8cf4df2bc3 100644 --- a/source/adios2/engine/campaign/CampaignData.h +++ b/source/adios2/engine/campaign/CampaignData.h @@ -31,7 +31,7 @@ struct CampaignHost { std::string hostname; std::string longhostname; - std::vector directory; + std::vector dirIdx; // index in CampaignData.directory global list of dirs }; struct CampaignBPFile @@ -56,6 +56,7 @@ struct CampaignData { std::string version; std::vector hosts; + std::vector directory; std::map bpdatasets; }; diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 9c512c7c66..bd56f12b75 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -198,9 +198,9 @@ void CampaignReader::InitTransports() CampaignHost &h = m_CampaignData.hosts[hostidx]; std::cout << " host =" << h.hostname << " long name = " << h.longhostname << " directories: \n"; - for (size_t diridx = 0; diridx < h.directory.size(); ++diridx) + for (size_t diridx = 0; diridx < h.dirIdx.size(); ++diridx) { - std::cout << " dir = " << h.directory[diridx] << "\n"; + std::cout << " dir = " << m_CampaignData.directory[h.dirIdx[diridx]] << "\n"; } } std::cout << " datasets:\n"; @@ -208,8 +208,7 @@ void CampaignReader::InitTransports() { CampaignBPDataset &ds = it.second; std::cout << " " << m_CampaignData.hosts[ds.hostIdx].hostname << ":" - << m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] << PathSeparator - << ds.name << "\n"; + << m_CampaignData.directory[ds.dirIdx] << PathSeparator << ds.name << "\n"; for (auto &bpf : ds.files) { std::cout << " file: " << bpf.name << "\n"; @@ -237,8 +236,7 @@ void CampaignReader::InitTransports() if (ho.protocol == HostAccessProtocol::S3) { const std::string endpointURL = ho.endpoint; - const std::string objPath = - m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + "/" + ds.name; + const std::string objPath = m_CampaignData.directory[ds.dirIdx] + "/" + ds.name; Params p; p.emplace("Library", "awssdk"); p.emplace("endpoint", endpointURL); @@ -249,8 +247,7 @@ void CampaignReader::InitTransports() p.emplace("recheck_metadata", (ho.recheckMetadata ? "true" : "false")); io.AddTransport("File", p); io.SetEngine("BP5"); - localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + - PathSeparator + ds.name; + localPath = m_CampaignData.directory[ds.dirIdx] + PathSeparator + ds.name; if (ho.isAWS_EC2) { adios2sys::SystemTools::PutEnv("AWS_EC2_METADATA_DISABLED=false"); @@ -277,7 +274,7 @@ void CampaignReader::InitTransports() if (!done) { const std::string remotePath = - m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; + m_CampaignData.directory[ds.dirIdx] + PathSeparator + ds.name; const std::string remoteURL = m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath; localPath = m_Options.cachepath + PathSeparator + @@ -302,8 +299,7 @@ void CampaignReader::InitTransports() } else { - localPath = - m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; + localPath = m_CampaignData.directory[ds.dirIdx] + PathSeparator + ds.name; if (m_Options.verbose > 0) { std::cout << "Open local file " << localPath << "\n";