Skip to content

Commit

Permalink
PR requested changes
Browse files Browse the repository at this point in the history 
* Added out-of-core array checks. Fixed ConvertColorToGrayScale parallelization code for out-of-core datastores.
  • Loading branch information
@mmarineBlueQuartz @imikejackson
mmarineBlueQuartz authored and imikejackson committed Nov 8, 2023
1 parent 6753b7c commit 4ab6dc0
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 50 deletions.
13 changes: 8 additions & 5 deletions src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/ConvertColorToGrayScale.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

#include "complex/Common/Array.hpp"
#include "complex/Common/Range.hpp"
#include "complex/Core/Application.hpp"
#include "complex/Core/Preferences.hpp"
#include "complex/DataStructure/DataArray.hpp"
#include "complex/DataStructure/DataGroup.hpp"
#include "complex/Utilities/ParallelDataAlgorithm.hpp"
Expand Down Expand Up @@ -35,8 +37,8 @@ class LuminosityImpl
for(size_t i = start; i < end; i++)
{
auto temp = static_cast<int32>(
roundf((m_ImageData[m_NumComp * i] * m_ColorWeights.getX()) + (m_ImageData[m_NumComp * i + 1] * m_ColorWeights.getY()) + (m_ImageData[m_NumComp * i + 2] * m_ColorWeights.getZ())));
m_FlatImageData[i] = static_cast<uint8>(temp);
roundf((m_ImageData.at(m_NumComp * i) * m_ColorWeights.getX()) + (m_ImageData.at(m_NumComp * i + 1) * m_ColorWeights.getY()) + (m_ImageData.at(m_NumComp * i + 2) * m_ColorWeights.getZ())));
m_FlatImageData.setValue(i, static_cast<uint8>(temp));
}
}

Expand Down Expand Up @@ -72,7 +74,7 @@ class LightnessImpl
for(size_t i = start; i < end; i++)
{
auto minmax = std::minmax_element(m_ImageData.begin() + (i * m_NumComp), m_ImageData.begin() + (i * m_NumComp + 3));
m_FlatImageData[i] = static_cast<uint8_t>(roundf(static_cast<float>(static_cast<int16_t>(*(minmax.first)) + static_cast<int16_t>(*(minmax.second))) / 2.0f));
m_FlatImageData.setValue(i, static_cast<uint8_t>(roundf(static_cast<float>(static_cast<int16_t>(*(minmax.first)) + static_cast<int16_t>(*(minmax.second))) / 2.0f)));
}
}

Expand Down Expand Up @@ -108,7 +110,7 @@ class SingleChannelImpl
{
for(size_t i = start; i < end; i++)
{
m_FlatImageData[i] = static_cast<uint8_t>((m_ImageData[m_NumComp * i + static_cast<size_t>(m_Channel)]));
m_FlatImageData.setValue(i, static_cast<uint8_t>((m_ImageData.at(m_NumComp * i + static_cast<size_t>(m_Channel)))));
}
}

Expand Down Expand Up @@ -198,9 +200,10 @@ Result<> ConvertColorToGrayScale::operator()()
case ConversionType::Average:
ParallelWrapper::Run<LuminosityImpl>(LuminosityImpl(inputColorData, outputGrayData, {0.3333f, 0.3333f, 0.3333f}, comp), totalPoints, algArrays);
break;
case ConversionType::Lightness:
case ConversionType::Lightness: {
ParallelWrapper::Run<LightnessImpl>(LightnessImpl(inputColorData, outputGrayData, comp), totalPoints, algArrays);
break;
}
case ConversionType::SingleChannel:
ParallelWrapper::Run<SingleChannelImpl>(SingleChannelImpl(inputColorData, outputGrayData, comp, m_InputValues->ColorChannel), totalPoints, algArrays);
break;
Expand Down
65 changes: 44 additions & 21 deletions src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/FindArrayStatistics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -660,26 +660,42 @@ void FindStatistics(const DataArray<T>& source, const Int32Array* featureIds, co
auto* modalBinsArrayPtr = dynamic_cast<NeighborList<float32>*>(arrays[11]);
auto* featureHasDataPtr = dynamic_cast<BoolArray*>(arrays[12]);

// #ifdef COMPLEX_ENABLE_MULTICORE
#if 0
const tbb::simple_partitioner simplePartitioner;
const size_t grainSize = 500;
const tbb::blocked_range<size_t> tbbRange(0, numFeatures, grainSize);
tbb::parallel_for(tbbRange,
FindArrayStatisticsByIndexImpl<T>(inputValues->FindLength, inputValues->FindMin, inputValues->FindMax, inputValues->FindMean, inputValues->FindMode,
inputValues->FindStdDeviation, inputValues->FindSummation, inputValues->FindHistogram, inputValues->MinRange, inputValues->MaxRange,
inputValues->UseFullRange, inputValues->NumBins, inputValues->FindModalBinRanges, mask, featureIds, source, featureHasDataPtr, lengthArrayPtr,
minArrayPtr, maxArrayPtr, meanArrayPtr, modeArrayPtr, stdDevArrayPtr, summationArrayPtr, histArrayPtr, mostPopulatedBinPtr, modalBinsArrayPtr,
filter),
simplePartitioner);
#else
ParallelDataAlgorithm indexAlg;
indexAlg.setRange(0, numFeatures);
indexAlg.execute(FindArrayStatisticsByIndexImpl<T>(inputValues->FindLength, inputValues->FindMin, inputValues->FindMax, inputValues->FindMean, inputValues->FindMode, inputValues->FindStdDeviation,
inputValues->FindSummation, inputValues->FindHistogram, inputValues->MinRange, inputValues->MaxRange, inputValues->UseFullRange,
inputValues->NumBins, inputValues->FindModalBinRanges, mask, featureIds, source, featureHasDataPtr, lengthArrayPtr, minArrayPtr, maxArrayPtr,
meanArrayPtr, modeArrayPtr, stdDevArrayPtr, summationArrayPtr, histArrayPtr, mostPopulatedBinPtr, modalBinsArrayPtr, filter));
indexAlg.setParallelizationEnabled(false);
IParallelAlgorithm::AlgorithmArrays indexAlgArrays;
indexAlgArrays.push_back(&source);
indexAlgArrays.push_back(featureHasDataPtr);
indexAlgArrays.push_back(lengthArrayPtr);
indexAlgArrays.push_back(minArrayPtr);
indexAlgArrays.push_back(maxArrayPtr);
indexAlgArrays.push_back(meanArrayPtr);
indexAlgArrays.push_back(stdDevArrayPtr);
indexAlgArrays.push_back(summationArrayPtr);
indexAlgArrays.push_back(histArrayPtr);
indexAlgArrays.push_back(mostPopulatedBinPtr);

#ifdef COMPLEX_ENABLE_MULTICORE
if(IParallelAlgorithm::CheckArraysInMemory(indexAlgArrays))
{
const tbb::simple_partitioner simplePartitioner;
const size_t grainSize = 500;
const tbb::blocked_range<size_t> tbbRange(0, numFeatures, grainSize);
tbb::parallel_for(tbbRange,
FindArrayStatisticsByIndexImpl<T>(inputValues->FindLength, inputValues->FindMin, inputValues->FindMax, inputValues->FindMean, inputValues->FindMode,
inputValues->FindStdDeviation, inputValues->FindSummation, inputValues->FindHistogram, inputValues->MinRange, inputValues->MaxRange,
inputValues->UseFullRange, inputValues->NumBins, inputValues->FindModalBinRanges, mask, featureIds, source, featureHasDataPtr, lengthArrayPtr,
minArrayPtr, maxArrayPtr, meanArrayPtr, modeArrayPtr, stdDevArrayPtr, summationArrayPtr, histArrayPtr, mostPopulatedBinPtr, modalBinsArrayPtr,
filter),
simplePartitioner);
}
else
{
ParallelDataAlgorithm indexAlg;
indexAlg.setRange(0, numFeatures);
indexAlg.requireArraysInMemory(indexAlgArrays);
indexAlg.execute(FindArrayStatisticsByIndexImpl<T>(
inputValues->FindLength, inputValues->FindMin, inputValues->FindMax, inputValues->FindMean, inputValues->FindMode, inputValues->FindStdDeviation, inputValues->FindSummation,
inputValues->FindHistogram, inputValues->MinRange, inputValues->MaxRange, inputValues->UseFullRange, inputValues->NumBins, inputValues->FindModalBinRanges, mask, featureIds, source,
featureHasDataPtr, lengthArrayPtr, minArrayPtr, maxArrayPtr, meanArrayPtr, modeArrayPtr, stdDevArrayPtr, summationArrayPtr, histArrayPtr, mostPopulatedBinPtr, modalBinsArrayPtr, filter));
}
#endif

if(inputValues->FindMedian || inputValues->FindNumUniqueValues)
Expand All @@ -689,8 +705,15 @@ void FindStatistics(const DataArray<T>& source, const Int32Array* featureIds, co
auto* medianArrayPtr = dynamic_cast<Float32Array*>(arrays[4]);
auto* numUniqueValuesArrayPtr = dynamic_cast<Int32Array*>(arrays[9]);

IParallelAlgorithm::AlgorithmArrays medianAlgArrays;
medianAlgArrays.push_back(featureIds);
medianAlgArrays.push_back(&source);
medianAlgArrays.push_back(medianArrayPtr);
medianAlgArrays.push_back(numUniqueValuesArrayPtr);
medianAlgArrays.push_back(lengthArrayPtr);

ParallelDataAlgorithm medianDataAlg;
medianDataAlg.setParallelizationEnabled(false);
medianDataAlg.requireArraysInMemory(medianAlgArrays);
medianDataAlg.setRange(0, numFeatures);
medianDataAlg.execute(
FindArrayMedianUniqueByIndexImpl<T>(mask, featureIds, source, inputValues->FindMedian, inputValues->FindNumUniqueValues, medianArrayPtr, numUniqueValuesArrayPtr, lengthArrayPtr, filter));
Expand Down
22 changes: 17 additions & 5 deletions src/Plugins/ComplexCore/src/ComplexCore/Filters/Algorithms/PartitionGeometry.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,13 @@ class PartitionCellBasedGeometryImpl
auto partitionIndexResult = m_PSImageGeom.getIndex(coord[0], coord[1], coord[2]);
if(partitionIndexResult.has_value())
{
// partitionIdsStore.setValue(index, static_cast<int32>(*partitionIndexResult) + m_StartingPartitionId);
partitionIdsStore[index] = static_cast<int32>(*partitionIndexResult) + m_StartingPartitionId;
partitionIdsStore.setValue(index, static_cast<int32>(*partitionIndexResult) + m_StartingPartitionId);
// partitionIdsStore[index] = static_cast<int32>(*partitionIndexResult) + m_StartingPartitionId;
}
else
{
// partitionIdsStore.setValue(index, m_OutOfBoundsValue);
partitionIdsStore[index] = m_OutOfBoundsValue;
partitionIdsStore.setValue(index, m_OutOfBoundsValue);
// partitionIdsStore[index] = m_OutOfBoundsValue;
}
}
}
Expand Down Expand Up @@ -244,9 +244,12 @@ Result<> PartitionGeometry::partitionCellBasedGeometry(const IGridGeometry& inpu
{
SizeVec3 dims = inputGeometry.getDimensions();

IParallelAlgorithm::AlgorithmArrays algArrays;
algArrays.push_back(&partitionIds);

ParallelData3DAlgorithm dataAlg;
dataAlg.setRange(dims[0], dims[1], dims[2]);
dataAlg.setParallelizationEnabled(false);
dataAlg.requireArraysInMemory(algArrays);
dataAlg.execute(PartitionCellBasedGeometryImpl(inputGeometry, partitionIds, psImageGeom, m_InputValues->StartingFeatureID, outOfBoundsValue, m_ShouldCancel));

return {};
Expand All @@ -258,9 +261,18 @@ Result<> PartitionGeometry::partitionCellBasedGeometry(const IGridGeometry& inpu
Result<> PartitionGeometry::partitionNodeBasedGeometry(const IGeometry::SharedVertexList& vertexList, Int32Array& partitionIds, const ImageGeom& psImageGeom, int outOfBoundsValue,
const std::optional<const BoolArray>& maskArrayOpt)
{
IParallelAlgorithm::AlgorithmArrays algArrays;
algArrays.push_back(&vertexList);
algArrays.push_back(&partitionIds);
if(maskArrayOpt.has_value())
{
algArrays.push_back(&(maskArrayOpt.value()));
}

// Allow data-based parallelization
ParallelDataAlgorithm dataAlg;
dataAlg.setRange(0, vertexList.getNumberOfTuples());
dataAlg.requireArraysInMemory(algArrays);
dataAlg.execute(PartitionNodeBasedGeometryImpl(vertexList, partitionIds, psImageGeom, m_InputValues->StartingFeatureID, outOfBoundsValue, maskArrayOpt, m_ShouldCancel));

return {};
Expand Down
50 changes: 31 additions & 19 deletions src/complex/Utilities/IParallelAlgorithm.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,40 @@

namespace complex
{
// -----------------------------------------------------------------------------
bool IParallelAlgorithm::CheckArraysInMemory(const AlgorithmArrays& arrays)
{
if(arrays.size() == 0)
{
true;

Check warning on line 13 in src/complex/Utilities/IParallelAlgorithm.cpp

View workflow job for this annotation

GitHub Actions / build (macos-11) 

expression result unused [-Wunused-value]

Check warning on line 13 in src/complex/Utilities/IParallelAlgorithm.cpp

View workflow job for this annotation

GitHub Actions / build (ubuntu-20.04, clang++-10) 

expression result unused [-Wunused-value]
}

for(const auto* array : arrays)
{
if(array == nullptr)
{
continue;
}

if(array->getIDataStoreRef().getDataFormat().empty() == false)
{
return false;
}
}

return true;
}

// -----------------------------------------------------------------------------
IParallelAlgorithm::IParallelAlgorithm()
{
#ifdef COMPLEX_ENABLE_MULTICORE
// Do not run OOC data in parallel by default.
m_RunParallel = !Application::GetOrCreateInstance()->getPreferences()->useOocData();
#endif
}

// -----------------------------------------------------------------------------
IParallelAlgorithm::~IParallelAlgorithm() = default;

// -----------------------------------------------------------------------------
Expand All @@ -23,29 +50,14 @@ bool IParallelAlgorithm::getParallelizationEnabled() const
// -----------------------------------------------------------------------------
void IParallelAlgorithm::setParallelizationEnabled(bool doParallel)
{
#ifdef COMPLEX_ENABLE_MULTICORE
m_RunParallel = doParallel;
#endif
}
// -----------------------------------------------------------------------------
void IParallelAlgorithm::requireArraysInMemory(const AlgorithmArrays& arrays)
{
if(arrays.size() == 0)
{
return;
}

for(const auto* array : arrays)
{
if(array == nullptr)
{
continue;
}

if(array->getIDataStoreRef().getDataFormat().empty() == false)
{
setParallelizationEnabled(false);
return;
}
}
setParallelizationEnabled(true);
bool inMemory = CheckArraysInMemory(arrays);
setParallelizationEnabled(!inMemory);
}
} // namespace complex
2 changes: 2 additions & 0 deletions src/complex/Utilities/IParallelAlgorithm.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ class COMPLEX_EXPORT IParallelAlgorithm
public:
using AlgorithmArrays = std::vector<const IDataArray*>;

static bool CheckArraysInMemory(const AlgorithmArrays& arrays);

IParallelAlgorithm(const IParallelAlgorithm&) = default;
IParallelAlgorithm(IParallelAlgorithm&&) noexcept = default;
IParallelAlgorithm& operator=(const IParallelAlgorithm&) = default;
Expand Down

0 comments on commit 4ab6dc0

Please sign in to comment.