Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement BP5 (and BP4) reader-side memory selection, do testing #3591

Merged
merged 5 commits into from
Apr 28, 2023
Merged

Implement BP5 (and BP4) reader-side memory selection, do testing #3591

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
321efe4
Implement BP5 reader-side memory selection
eisenhauer Apr 24, 2023
aeed4f6
Don't run selection tests on Windows (where we default to BP4).
eisenhauer Apr 24, 2023
b5e92fa
Implement memory selection in BP4 reader
eisenhauer Apr 25, 2023
cac818f
kill commented-out SetEngine calls
eisenhauer Apr 28, 2023
70f330c
BP3->BP4
eisenhauer Apr 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions source/adios2/helper/adiosType.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,17 @@ class CoreDims
typedef size_t *iterator;
iterator begin() { return &DimensSpan[0]; }
iterator end() { return &DimensSpan[DimCount]; }
friend std::ostream &operator<<(std::ostream &os, const CoreDims &m)
{
os << "{";
for (size_t i = 0; i < m.size(); i++)
{
os << m[i];
if (i < m.size() - 1)
os << ", ";
}
return os << "}";
}
};

class DimsArray : public CoreDims
Expand Down
54 changes: 49 additions & 5 deletions source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -598,11 +598,55 @@ void BP4Deserializer::PostDataRead(
? Dims(blockInfo.Count.size(), 0)
: blockInfo.Start;

helper::ClipContiguousMemory(
blockInfo.Data, blockInfoStart, blockInfo.Count,
m_ThreadBuffers[threadID][0].data(), subStreamBoxInfo.BlockBox,
subStreamBoxInfo.IntersectionBox, m_IsRowMajor, m_ReverseDimensions,
endianReverse, blockInfo.MemSpace);
if (!blockInfo.MemoryStart.empty())
{
if (endianReverse)
{
helper::Throw<std::invalid_argument>(
"Toolkit", "format::bp::BP4Deserializer", "PostDataRead",
"endianReverse "
"not supported with MemorySelection");
}

if (m_ReverseDimensions)
{
helper::Throw<std::invalid_argument>(
"Toolkit", "format::bp::BP4Deserializer", "PostDataRead",
"ReverseDimensions not supported with "
"MemorySelection");
}

helper::DimsArray intersectStart(
subStreamBoxInfo.IntersectionBox.first);
helper::DimsArray intersectCount(
subStreamBoxInfo.IntersectionBox.second);
helper::DimsArray blockStart(subStreamBoxInfo.BlockBox.first);
helper::DimsArray blockCount(subStreamBoxInfo.BlockBox.second);
helper::DimsArray memoryStart(blockInfoStart);
for (size_t d = 0; d < intersectStart.size(); d++)
{
// change {intersect,block}Count from [start, end] to {start, count}
intersectCount[d] -= (intersectStart[d] - 1);
blockCount[d] -= (blockStart[d] - 1);
// shift everything by MemoryStart
intersectStart[d] += blockInfo.MemoryStart[d];
blockStart[d] += blockInfo.MemoryStart[d];
}
helper::NdCopy(m_ThreadBuffers[threadID][0].data(), intersectStart,
intersectCount, true, true,
reinterpret_cast<char *>(blockInfo.Data), intersectStart,
intersectCount, true, true, sizeof(T), intersectStart,
blockCount, memoryStart,
helper::DimsArray(blockInfo.MemoryCount), false);
}
else
{
helper::ClipContiguousMemory(
blockInfo.Data, blockInfoStart, blockInfo.Count,
m_ThreadBuffers[threadID][0].data(), subStreamBoxInfo.BlockBox,
subStreamBoxInfo.IntersectionBox, m_IsRowMajor, m_ReverseDimensions,
endianReverse, blockInfo.MemSpace);
}
}

void BP4Deserializer::BackCompatDecompress(
Expand Down
67 changes: 63 additions & 4 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1473,7 +1473,9 @@ static bool IntersectionStartCount(const size_t dimensionsSize,
outstart[d] = intersectionStart;
outcount[d] = intersectionEnd - intersectionStart + 1;
if (outcount[d] == 0)
{
return false;
}
}
return true;
}
Expand Down Expand Up @@ -1822,22 +1824,79 @@ void BP5Deserializer::FinalizeGet(const ReadRequest &Read, const bool freeAddr)
}
}

VariableBase *VB = static_cast<VariableBase *>(Req.VarRec->Variable);
DimsArray inStart(DimCount, RankOffset);
DimsArray inCount(DimCount, RankSize);
DimsArray outStart(DimCount, SelOffset);
DimsArray outCount(DimCount, SelSize);
DimsArray outMemStart(VB->m_MemoryStart);
DimsArray outMemCount(VB->m_MemoryCount);
if (!m_ReaderIsRowMajor)
{
std::reverse(inStart.begin(), inStart.end());
std::reverse(inCount.begin(), inCount.end());
std::reverse(outStart.begin(), outStart.end());
std::reverse(outCount.begin(), outCount.end());
std::reverse(outMemStart.begin(), outMemStart.end());
std::reverse(outMemCount.begin(), outMemCount.end());
}

helper::NdCopy(VirtualIncomingData, inStart, inCount, true, true,
(char *)Req.Data, outStart, outCount, true, true,
ElementSize, CoreDims(), CoreDims(), CoreDims(), CoreDims(),
false, Req.MemSpace);
if (VB->m_MemoryStart.size() > 0)
{
#ifdef NOTDEF // haven't done endinness for BP5
if (endianReverse)
{
helper::Throw<std::invalid_argument>(
"Toolkit", "format::bp::BP5Deserializer", "PostDataRead",
"endianReverse "
"not supported with MemorySelection");
}
if (m_ReaderIsRowMajor)
{
helper::Throw<std::invalid_argument>(
"Toolkit", "format::bp::BP5Deserializer", "PostDataRead",
"ReverseDimensions not supported with "
"MemorySelection");
}
#endif

const Box<Dims> selectionBox =
helper::StartEndBox(Dims(outStart.begin(), outStart.end()),
Dims(outCount.begin(), outCount.end()));
const Box<Dims> BlockBox =
helper::StartEndBox(Dims(inStart.begin(), inStart.end()),
Dims(inCount.begin(), inCount.end()));
const Box<Dims> IntersectionBox =
helper::IntersectionBox(selectionBox, BlockBox);
VirtualIncomingData =
Read.DestinationAddr; // Don't do the fake offset thing
helper::DimsArray intersectStart(IntersectionBox.first);
helper::DimsArray intersectCount(IntersectionBox.second);
helper::DimsArray blockStart(BlockBox.first);
helper::DimsArray blockCount(BlockBox.second);
helper::DimsArray memoryStart(DimCount, SelOffset);
helper::DimsArray memoryCount(VB->m_MemoryCount);
for (size_t d = 0; d < intersectStart.size(); d++)
{
// change {intersect,block}Count from [start, end] to {start, count}
intersectCount[d] -= (intersectStart[d] - 1);
blockCount[d] -= (blockStart[d] - 1);
// shift everything by MemoryStart
intersectStart[d] += VB->m_MemoryStart[d];
blockStart[d] += VB->m_MemoryStart[d];
}
helper::NdCopy(VirtualIncomingData, intersectStart, intersectCount,
true, true, (char *)Req.Data, intersectStart,
intersectCount, true, true, ElementSize, intersectStart,
blockCount, memoryStart, memoryCount, false);
}
else
{
helper::NdCopy(VirtualIncomingData, inStart, inCount, true, true,
(char *)Req.Data, outStart, outCount, true, true,
ElementSize, CoreDims(), CoreDims(), CoreDims(),
CoreDims(), false, Req.MemSpace);
}
if (freeAddr)
{
free((char *)Read.DestinationAddr);
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/format/bp5/BP5Serializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -894,7 +894,7 @@ void BP5Serializer::Marshal(void *Variable, const char *Name,
lf_QueueSpanMinMax(*Span, ElemCount, (DataType)Rec->Type,
MemSpace, Rec->MetaOffset,
Rec->MinMaxOffset,
MetaEntry->BlockCount /*BlockNum*/);
MetaEntry->BlockCount - 1 /*BlockNum*/);
}
}

Expand Down
21 changes: 19 additions & 2 deletions testing/adios2/interface/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,26 @@
# These tests should be *very* fast
set(CTEST_TEST_TIMEOUT 10)

gtest_add_tests_helper(Interface MPI_ALLOW ADIOS Interface. "")
set(BP3_DIR ${CMAKE_CURRENT_BINARY_DIR}/bp3)
set(BP4_DIR ${CMAKE_CURRENT_BINARY_DIR}/bp4)
set(BP5_DIR ${CMAKE_CURRENT_BINARY_DIR}/bp5)
set(BPfile_DIR ${CMAKE_CURRENT_BINARY_DIR}/bpfile)
set(FS_DIR ${CMAKE_CURRENT_BINARY_DIR}/filestream)
file(MAKE_DIRECTORY ${BP3_DIR})
file(MAKE_DIRECTORY ${BP4_DIR})
file(MAKE_DIRECTORY ${BP5_DIR})
file(MAKE_DIRECTORY ${BPfile_DIR})
file(MAKE_DIRECTORY ${FS_DIR})

gtest_add_tests_helper(Interface MPI_ALLOW ADIOS Interface. .BP3
WORKING_DIRECTORY ${BP3_DIR} EXTRA_ARGS "BP3")
gtest_add_tests_helper(Interface MPI_ALLOW ADIOS Interface. .BPfile
WORKING_DIRECTORY ${BPfile_DIR} EXTRA_ARGS "BPfile")
gtest_add_tests_helper(Write MPI_ALLOW ADIOSInterface Interface. "")
gtest_add_tests_helper(DefineVariable MPI_ALLOW ADIOS Interface. "")
gtest_add_tests_helper(DefineAttribute MPI_ALLOW ADIOS Interface. "")
gtest_add_tests_helper(Selection MPI_NONE ADIOS Interface. "")
gtest_add_tests_helper(Selection MPI_NONE ADIOS Interface. .BP3
WORKING_DIRECTORY ${BP3_DIR} EXTRA_ARGS "BP3")
gtest_add_tests_helper(Selection MPI_NONE ADIOS Interface. .BPfile
WORKING_DIRECTORY ${BPfile_DIR} EXTRA_ARGS "BPfile")
gtest_add_tests_helper(NoMpi MPI_NONE ADIOS Interface. "")
Loading