Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

added blosc and bzip2 to Table Engine #2756

Merged
merged 1 commit into from
Jun 8, 2021
Merged

added blosc and bzip2 to Table Engine #2756

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions source/adios2/engine/table/TableWriter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
*/

#include "TableWriter.tcc"
#include "adios2/helper/adiosString.h"

namespace adios2
{
Expand All @@ -25,17 +26,17 @@ TableWriter::TableWriter(IO &io, const std::string &name, const Mode mode,
{
m_MpiRank = m_Comm.Rank();
m_MpiSize = m_Comm.Size();

helper::GetParameter(m_IO.m_Parameters, "Compressor", m_UseCompressor);
m_SubEngine = &m_SubIO.Open(m_Name, adios2::Mode::Write);
}

TableWriter::~TableWriter()
{
if (m_SzOperator)
if (m_Compressor)
{
delete m_SzOperator;
delete m_Compressor;
}
m_SzOperator = nullptr;
m_Compressor = nullptr;
}

StepStatus TableWriter::BeginStep(StepMode mode, const float timeoutSeconds)
Expand Down
10 changes: 3 additions & 7 deletions source/adios2/engine/table/TableWriter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,6 @@
#include "adios2/core/Engine.h"
#include "adios2/core/IO.h"
#include "adios2/core/Variable.h"
#include "adios2/helper/adiosComm.h"

#ifdef ADIOS2_HAVE_SZ
#include "adios2/operator/compress/CompressSZ.h"
#endif

namespace adios2
{
Expand Down Expand Up @@ -50,8 +45,9 @@ class TableWriter : public Engine
ADIOS m_SubAdios;
IO &m_SubIO;
Engine *m_SubEngine = nullptr;
Operator *m_SzOperator = nullptr;
std::unordered_map<std::string, bool> m_Indexing;
Operator *m_Compressor = nullptr;
std::unordered_map<std::string, bool> m_IndexerMap;
std::string m_UseCompressor;

void PutSubEngine(bool finalPut = false);

Expand Down
55 changes: 30 additions & 25 deletions source/adios2/engine/table/TableWriter.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@

#include "TableWriter.h"

#ifdef ADIOS2_HAVE_BLOSC
#include "adios2/operator/compress/CompressBlosc.h"
#endif

#ifdef ADIOS2_HAVE_BZIP2
#include "adios2/operator/compress/CompressBZIP2.h"
#endif

namespace adios2
{
namespace core
Expand Down Expand Up @@ -60,36 +68,33 @@ void TableWriter::PutDeferredCommon(Variable<T> &variable, const T *data)
if (!var)
{
var = &m_SubIO.DefineVariable<T>(variable.m_Name, variable.m_Shape);
for (const auto &i : m_IO.m_TransportsParameters)
if (m_UseCompressor == "blosc")
{
auto itVar = i.find("Variable");
if (itVar != i.end() && itVar->second == variable.m_Name)
{
auto itAccuracy = i.find("Accuracy");
if (itAccuracy != i.end())
{
#ifdef ADIOS2_HAVE_SZ
m_SzOperator = new compress::CompressSZ(Params());
var->AddOperation(
*m_SzOperator,
{{adios2::ops::sz::key::accuracy, itAccuracy->second}});
#ifdef ADIOS2_HAVE_BLOSC
m_Compressor = new compress::CompressBlosc({});
var->AddOperation(*m_Compressor, {});
#else
std::cerr << "ADIOS2 is not compiled with SZ" << std::endl;
std::cerr
<< "ADIOS2 is not compiled with c-blosc "
"(https://github.com/Blosc/c-blosc), compressor not added"
<< std::endl;
#endif
}
else if (m_UseCompressor == "bzip2")
{
#ifdef ADIOS2_HAVE_BZIP2
m_Compressor = new compress::CompressBZIP2({});
var->AddOperation(*m_Compressor, {});
#else
std::cerr << "ADIOS2 is not compiled with Bzip2 "
"(https://gitlab.com/federicomenaquintero/bzip2), "
"compressor not added"
<< std::endl;
#endif
}
auto itIndexing = i.find("Index");
if (itIndexing != i.end())
{
m_Indexing[variable.m_Name] = true;
}
else
{
m_Indexing[variable.m_Name] = false;
}
}
}
}
if (m_Indexing[variable.m_Name])

if (m_IndexerMap[variable.m_Name])
{
// TODO: implement indexing
}
Expand Down