From 8fcfb9b1eeffb2587eba51ad86956ead45ac43c9 Mon Sep 17 00:00:00 2001 From: peterhillman Date: Fri, 3 Dec 2021 10:12:25 +1300 Subject: [PATCH] Reduce memory consumption with very large deepscanline images (#1208) * DeepScanlineInputFile now uses chunk size test from DeepTiledInputFile Signed-off-by: Peter Hillman * don't cache sampleCount in large deepscanline images Signed-off-by: Peter Hillman --- src/lib/OpenEXR/ImfDeepScanLineInputFile.cpp | 447 +++++++++++------- .../OpenEXRTest/testDeepScanLineBasic.cpp | 159 +++++-- 2 files changed, 390 insertions(+), 216 deletions(-) diff --git a/src/lib/OpenEXR/ImfDeepScanLineInputFile.cpp b/src/lib/OpenEXR/ImfDeepScanLineInputFile.cpp index 364a1f04b..2960990ba 100644 --- a/src/lib/OpenEXR/ImfDeepScanLineInputFile.cpp +++ b/src/lib/OpenEXR/ImfDeepScanLineInputFile.cpp @@ -126,7 +126,7 @@ struct LineBuffer int number; bool hasException; string exception; - + Array2D _tempCountBuffer; LineBuffer (); ~LineBuffer (); @@ -197,8 +197,11 @@ struct DeepScanLineInputFile::Data MultiPartInputFile* multiPartFile; // for multipart files opened as single part bool memoryMapped; // if the stream is memory mapped + bool bigFile; // if file has large dataWindow, do not use 'sampleCount' cache; read samples + // each time + Array2D sampleCount; // the number of samples - // in each pixel + // in each pixel unless bigFile is true Array lineSampleCount; // the number of samples // in each line @@ -247,6 +250,7 @@ DeepScanLineInputFile::Data::Data (int numThreads): multiPartBackwardSupport(false), multiPartFile(NULL), memoryMapped(false), + bigFile(false), frameBufferValid(false), _streamData(NULL), _deleteStream(false) @@ -414,7 +418,7 @@ readPixelData (InputStreamMutex *streamData, // if necessary. // - if (!isMultiPart(ifd->version)) + if (!isMultiPart(ifd->version) && !ifd->bigFile) { if (ifd->nextLineBufferMinY != minY) streamData->is->seekg (lineOffset); @@ -424,6 +428,7 @@ readPixelData (InputStreamMutex *streamData, // // In a multi-part file, the file pointer may have been moved by // other parts, so we have to ask tellg() where we are. + // big files also move the pointer since they have re-read the sampleCount buffer // if (streamData->is->tellg() != ifd->lineOffsets[lineBufferNumber]) streamData->is->seekg (lineOffset); @@ -514,6 +519,185 @@ readPixelData (InputStreamMutex *streamData, ifd->nextLineBufferMinY = minY - ifd->linesInBuffer; } + + +void +readSampleCountForLineBlock(InputStreamMutex* streamData, + DeepScanLineInputFile::Data* data, + int lineBlockId, + Array2D* sampleCountBuffer, + int sampleCountMinY, + bool writeToSlice + ) +{ + streamData->is->seekg(data->lineOffsets[lineBlockId]); + + if (isMultiPart(data->version)) + { + int partNumber; + OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read (*streamData->is, partNumber); + + if (partNumber != data->partNumber) + throw IEX_NAMESPACE::ArgExc("Unexpected part number."); + } + + int minY; + OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read (*streamData->is, minY); + + // + // Check the correctness of minY. + // + + if (minY != data->minY + lineBlockId * data->linesInBuffer) + throw IEX_NAMESPACE::ArgExc("Unexpected data block y coordinate."); + + int maxY; + maxY = min(minY + data->linesInBuffer - 1, data->maxY); + + uint64_t sampleCountTableDataSize; + OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read (*streamData->is, sampleCountTableDataSize); + + + + if(sampleCountTableDataSize>static_cast(data->maxSampleCountTableSize)) + { + THROW (IEX_NAMESPACE::ArgExc, "Bad sampleCountTableDataSize read from chunk "<< lineBlockId << ": expected " << data->maxSampleCountTableSize << " or less, got "<< sampleCountTableDataSize); + } + + uint64_t packedDataSize; + uint64_t unpackedDataSize; + OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read (*streamData->is, packedDataSize); + OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read (*streamData->is, unpackedDataSize); + + + + // + // We make a check on the data size requirements here. + // Whilst we wish to store 64bit sizes on disk, not all the compressors + // have been made to work with such data sizes and are still limited to + // using signed 32 bit (int) for the data size. As such, this version + // insists that we validate that the data size does not exceed the data + // type max limit. + // @TODO refactor the compressor code to ensure full 64-bit support. + // + + uint64_t compressorMaxDataSize = static_cast(std::numeric_limits::max()); + if (packedDataSize > compressorMaxDataSize || + unpackedDataSize > compressorMaxDataSize || + sampleCountTableDataSize > compressorMaxDataSize) + { + THROW (IEX_NAMESPACE::ArgExc, "This version of the library does not" + << "support the allocation of data with size > " + << compressorMaxDataSize + << " file table size :" << sampleCountTableDataSize + << " file unpacked size :" << unpackedDataSize + << " file packed size :" << packedDataSize << ".\n"); + } + + + streamData->is->read(data->sampleCountTableBuffer, static_cast(sampleCountTableDataSize)); + + const char* readPtr; + + // + // If the sample count table is compressed, we'll uncompress it. + // + + + if (sampleCountTableDataSize < static_cast(data->maxSampleCountTableSize)) + { + if(!data->sampleCountTableComp) + { + THROW(IEX_NAMESPACE::ArgExc,"Deep scanline data corrupt at chunk " << lineBlockId << " (sampleCountTableDataSize error)"); + } + data->sampleCountTableComp->uncompress(data->sampleCountTableBuffer, + static_cast(sampleCountTableDataSize), + minY, + readPtr); + } + else readPtr = data->sampleCountTableBuffer; + + char* base = data->sampleCountSliceBase; + int xStride = data->sampleCountXStride; + int yStride = data->sampleCountYStride; + + // total number of samples in block: used to check samplecount table doesn't + // reference more data than exists + + size_t cumulative_total_samples=0; + + for (int y = minY; y <= maxY; y++) + { + int yInDataWindow = y - data->minY; + data->lineSampleCount[yInDataWindow] = 0; + + int lastAccumulatedCount = 0; + for (int x = data->minX; x <= data->maxX; x++) + { + int accumulatedCount, count; + + // + // Read the sample count for pixel (x, y). + // + + Xdr::read (readPtr, accumulatedCount); + + // sample count table should always contain monotonically + // increasing values. + if (accumulatedCount < lastAccumulatedCount) + { + THROW(IEX_NAMESPACE::ArgExc,"Deep scanline sampleCount data corrupt at chunk " << lineBlockId << " (negative sample count detected)"); + } + + count = accumulatedCount - lastAccumulatedCount; + lastAccumulatedCount = accumulatedCount; + + // + // Store the data in internal data structure. + // + + if(sampleCountBuffer) + { + (*sampleCountBuffer)[y-sampleCountMinY][x - data->minX] = count; + } + data->lineSampleCount[yInDataWindow] += count; + + // + // Store the data in external slice + // + if (writeToSlice) + { + sampleCount(base, xStride, yStride, x, y) = count; + } + } + cumulative_total_samples+=data->lineSampleCount[yInDataWindow]; + if(cumulative_total_samples*data->combinedSampleSize > unpackedDataSize) + { + THROW(IEX_NAMESPACE::ArgExc,"Deep scanline sampleCount data corrupt at chunk " << lineBlockId << ": pixel data only contains " << unpackedDataSize + << " bytes of data but table references at least " << cumulative_total_samples*data->combinedSampleSize << " bytes of sample data" ); + } + + data->gotSampleCount[y - data->minY] = true; + } +} + + +void +fillSampleCountFromCache(int y, DeepScanLineInputFile::Data* data) +{ + int yInDataWindow = y - data->minY; + char* base = data->sampleCountSliceBase; + int xStride = data->sampleCountXStride; + int yStride = data->sampleCountYStride; + + for (int x = data->minX; x <= data->maxX; x++) + { + unsigned int count = data->sampleCount[yInDataWindow][x - data->minX]; + sampleCount(base, xStride, yStride, x, y) = count; + } +} + + // // A LineBufferTask encapsulates the task uncompressing a set of // scanlines (line buffer) and copying them into the frame buffer. @@ -706,9 +890,22 @@ LineBufferTask::execute () int width = (_ifd->maxX - _ifd->minX + 1); - ptrdiff_t base = reinterpret_cast(&_ifd->sampleCount[0][0]); - base -= sizeof(unsigned int)*_ifd->minX; - base -= sizeof(unsigned int)*static_cast(_ifd->minY) * static_cast(width); + + ptrdiff_t base; + + if( _ifd->bigFile) + { + base = reinterpret_cast(&_lineBuffer->_tempCountBuffer[0][0]); + base -= sizeof(unsigned int)*_ifd->minX; + base -= sizeof(unsigned int)*static_cast(_lineBuffer->minY) * static_cast(width); + } + else + { + base = reinterpret_cast(&_ifd->sampleCount[0][0]); + base -= sizeof(unsigned int)*_ifd->minX; + base -= sizeof(unsigned int)*static_cast(_ifd->minY) * static_cast(width); + + } copyIntoDeepFrameBuffer (readPtr, slice.base, reinterpret_cast(base), @@ -777,6 +974,32 @@ newLineBufferTask lineBuffer->number = number; lineBuffer->uncompressedData = 0; + if (ifd->bigFile) + { + + if (lineBuffer->_tempCountBuffer.height() != ifd->linesInBuffer || + lineBuffer->_tempCountBuffer.width() != ifd->maxX - ifd->minX + 1) + { + lineBuffer->_tempCountBuffer.resizeErase(ifd->linesInBuffer,ifd->maxX - ifd->minX + 1); + } + + // + // read sample counts into internal 'tempCountBuffer' only, not into external buffer + // + + int lineBlockId = ( lineBuffer->minY - ifd->minY ) / ifd->linesInBuffer; + + + readSampleCountForLineBlock(ifd->_streamData, + ifd, + lineBlockId, + &lineBuffer->_tempCountBuffer, + lineBuffer->minY, + false + ); + + } + readPixelData (ifd->_streamData, ifd, lineBuffer->minY, lineBuffer->buffer, lineBuffer->packedDataSize, @@ -816,6 +1039,16 @@ newLineBufferTask scanLineMin, scanLineMax); } +// +// when handling files with dataWindows with a large number of pixels, +// the sampleCount values are not precached and Data::sampleCount is not asigned +// instead, the sampleCount is read every time readPixels() is called +// and sample counts are stored in LineBuffer::_tempCountBuffer instead +// (A square image that is 16k by 16k pixels has gBigFileDataWindowSize pixels, +// andthe sampleCount table would take 1GiB of memory to store) +// +const uint64_t gBigFileDataWindowSize = (1<<28); + } // namespace @@ -853,8 +1086,18 @@ void DeepScanLineInputFile::initialize(const Header& header) _data->minY = dataWindow.min.y; _data->maxY = dataWindow.max.y; - _data->sampleCount.resizeErase(_data->maxY - _data->minY + 1, - _data->maxX - _data->minX + 1); + + if ( static_cast(_data->maxX-_data->minX+1)* + static_cast(_data->maxY-_data->minY+1) + > gBigFileDataWindowSize) + { + _data->bigFile = true; + } + else + { + _data->sampleCount.resizeErase(_data->maxY - _data->minY + 1, + _data->maxX - _data->minX + 1); + } _data->lineSampleCount.resizeErase(_data->maxY - _data->minY + 1); Compressor* compressor = newCompressor(_data->header.compression(), @@ -1513,6 +1756,9 @@ union bytesOruint64_t uint64_t i; }; } + + + void DeepScanLineInputFile::rawPixelData (int firstScanLine, char *pixelData, @@ -1902,171 +2148,6 @@ void DeepScanLineInputFile::readPixelSampleCounts (const char* rawPixelData, -namespace -{ - -void -readSampleCountForLineBlock(InputStreamMutex* streamData, - DeepScanLineInputFile::Data* data, - int lineBlockId) -{ - streamData->is->seekg(data->lineOffsets[lineBlockId]); - - if (isMultiPart(data->version)) - { - int partNumber; - OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read (*streamData->is, partNumber); - - if (partNumber != data->partNumber) - throw IEX_NAMESPACE::ArgExc("Unexpected part number."); - } - - int minY; - OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read (*streamData->is, minY); - - // - // Check the correctness of minY. - // - - if (minY != data->minY + lineBlockId * data->linesInBuffer) - throw IEX_NAMESPACE::ArgExc("Unexpected data block y coordinate."); - - int maxY; - maxY = min(minY + data->linesInBuffer - 1, data->maxY); - - uint64_t sampleCountTableDataSize; - OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read (*streamData->is, sampleCountTableDataSize); - - - - if(sampleCountTableDataSize>static_cast(data->maxSampleCountTableSize)) - { - THROW (IEX_NAMESPACE::ArgExc, "Bad sampleCountTableDataSize read from chunk "<< lineBlockId << ": expected " << data->maxSampleCountTableSize << " or less, got "<< sampleCountTableDataSize); - } - - uint64_t packedDataSize; - uint64_t unpackedDataSize; - OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read (*streamData->is, packedDataSize); - OPENEXR_IMF_INTERNAL_NAMESPACE::Xdr::read (*streamData->is, unpackedDataSize); - - - - // - // We make a check on the data size requirements here. - // Whilst we wish to store 64bit sizes on disk, not all the compressors - // have been made to work with such data sizes and are still limited to - // using signed 32 bit (int) for the data size. As such, this version - // insists that we validate that the data size does not exceed the data - // type max limit. - // @TODO refactor the compressor code to ensure full 64-bit support. - // - - uint64_t compressorMaxDataSize = static_cast(std::numeric_limits::max()); - if (packedDataSize > compressorMaxDataSize || - unpackedDataSize > compressorMaxDataSize || - sampleCountTableDataSize > compressorMaxDataSize) - { - THROW (IEX_NAMESPACE::ArgExc, "This version of the library does not" - << "support the allocation of data with size > " - << compressorMaxDataSize - << " file table size :" << sampleCountTableDataSize - << " file unpacked size :" << unpackedDataSize - << " file packed size :" << packedDataSize << ".\n"); - } - - - streamData->is->read(data->sampleCountTableBuffer, static_cast(sampleCountTableDataSize)); - - const char* readPtr; - - // - // If the sample count table is compressed, we'll uncompress it. - // - - - if (sampleCountTableDataSize < static_cast(data->maxSampleCountTableSize)) - { - if(!data->sampleCountTableComp) - { - THROW(IEX_NAMESPACE::ArgExc,"Deep scanline data corrupt at chunk " << lineBlockId << " (sampleCountTableDataSize error)"); - } - data->sampleCountTableComp->uncompress(data->sampleCountTableBuffer, - static_cast(sampleCountTableDataSize), - minY, - readPtr); - } - else readPtr = data->sampleCountTableBuffer; - - char* base = data->sampleCountSliceBase; - int xStride = data->sampleCountXStride; - int yStride = data->sampleCountYStride; - - // total number of samples in block: used to check samplecount table doesn't - // reference more data than exists - - size_t cumulative_total_samples=0; - - for (int y = minY; y <= maxY; y++) - { - int yInDataWindow = y - data->minY; - data->lineSampleCount[yInDataWindow] = 0; - - int lastAccumulatedCount = 0; - for (int x = data->minX; x <= data->maxX; x++) - { - int accumulatedCount, count; - - // - // Read the sample count for pixel (x, y). - // - - Xdr::read (readPtr, accumulatedCount); - - // sample count table should always contain monotonically - // increasing values. - if (accumulatedCount < lastAccumulatedCount) - { - THROW(IEX_NAMESPACE::ArgExc,"Deep scanline sampleCount data corrupt at chunk " << lineBlockId << " (negative sample count detected)"); - } - - count = accumulatedCount - lastAccumulatedCount; - lastAccumulatedCount = accumulatedCount; - - // - // Store the data in both internal and external data structure. - // - - data->sampleCount[yInDataWindow][x - data->minX] = count; - data->lineSampleCount[yInDataWindow] += count; - sampleCount(base, xStride, yStride, x, y) = count; - } - cumulative_total_samples+=data->lineSampleCount[yInDataWindow]; - if(cumulative_total_samples*data->combinedSampleSize > unpackedDataSize) - { - THROW(IEX_NAMESPACE::ArgExc,"Deep scanline sampleCount data corrupt at chunk " << lineBlockId << ": pixel data only contains " << unpackedDataSize - << " bytes of data but table references at least " << cumulative_total_samples*data->combinedSampleSize << " bytes of sample data" ); - } - data->gotSampleCount[y - data->minY] = true; - } -} - - -void -fillSampleCountFromCache(int y, DeepScanLineInputFile::Data* data) -{ - int yInDataWindow = y - data->minY; - char* base = data->sampleCountSliceBase; - int xStride = data->sampleCountXStride; - int yStride = data->sampleCountYStride; - - for (int x = data->minX; x <= data->maxX; x++) - { - unsigned int count = data->sampleCount[yInDataWindow][x - data->minX]; - sampleCount(base, xStride, yStride, x, y) = count; - } -} - -} // namespace void DeepScanLineInputFile::readPixelSampleCounts (int scanline1, int scanline2) @@ -2095,18 +2176,28 @@ DeepScanLineInputFile::readPixelSampleCounts (int scanline1, int scanline2) for (int i = scanLineMin; i <= scanLineMax; i++) { // - // if scanline is already read, it'll be in the cache + // if scanline is already read, and file is small enough to cache, count data will be in the cache // otherwise, read from file, store in cache and in caller's framebuffer // - if (_data->gotSampleCount[i - _data->minY]) + if ( !_data->bigFile && _data->gotSampleCount[i - _data->minY]) { fillSampleCountFromCache(i,_data); - - }else{ + } + else + { int lineBlockId = ( i - _data->minY ) / _data->linesInBuffer; - readSampleCountForLineBlock ( _data->_streamData, _data, lineBlockId ); + + // + // read samplecount data into external buffer + // also cache to internal buffer unless in bigFile mode + // + readSampleCountForLineBlock ( _data->_streamData, _data, lineBlockId, + _data->bigFile ? nullptr : &_data->sampleCount, + _data->minY, + true + ); int minYInLineBuffer = lineBlockId * _data->linesInBuffer + _data->minY; int maxYInLineBuffer = min ( minYInLineBuffer + _data->linesInBuffer - 1, _data->maxY ); diff --git a/src/test/OpenEXRTest/testDeepScanLineBasic.cpp b/src/test/OpenEXRTest/testDeepScanLineBasic.cpp index dc5a9fd82..1d256ecfc 100644 --- a/src/test/OpenEXRTest/testDeepScanLineBasic.cpp +++ b/src/test/OpenEXRTest/testDeepScanLineBasic.cpp @@ -33,12 +33,7 @@ using namespace ILMTHREAD_NAMESPACE; namespace { -const int width = 273; -const int height = 173; -const int minX = 10; -const int minY = 11; -const Box2i dataWindow(V2i(minX, minY), V2i(minX + width - 1, minY + height - 1)); -const Box2i displayWindow(V2i(0, 0), V2i(minX + width * 2, minY + height * 2)); + vector channelTypes; Array2D sampleCount; @@ -47,7 +42,10 @@ Header header; void generateRandomFile (const std::string filename, int channelCount, Compression compression, - bool bulkWrite) + bool bulkWrite, + const Box2i& dataWindow, + const Box2i& displayWindow + ) { cout << "generating " << flush; header = Header(displayWindow, dataWindow, @@ -59,6 +57,10 @@ void generateRandomFile (const std::string filename, cout << "compression " << compression << " " << flush; + int width = dataWindow.max.x - dataWindow.min.x+1; + int height = dataWindow.max.y - dataWindow.min.y+1; + + // // Add channels. // @@ -133,6 +135,10 @@ void generateRandomFile (const std::string filename, file.setFrameBuffer(frameBuffer); + int maxSamples = 10; + + bool bigFile = width>1000 || height>1000; + cout << "writing " << flush; if (bulkWrite) { @@ -145,23 +151,41 @@ void generateRandomFile (const std::string filename, for (int j = 0; j < width; j++) { - sampleCount[i][j] = random_int(10) + 1; + int samples = random_int(maxSamples); + + // big files write very sparse data for efficiency: most pixels have no samples + + if(bigFile && (i%63!=0 || j%63!=0)) + { + samples=0; + } + sampleCount[i][j] = samples; + + for (int k = 0; k < channelCount; k++) { - if (channelTypes[k] == 0) - data[k][i][j] = new unsigned int[sampleCount[i][j]]; - if (channelTypes[k] == 1) - data[k][i][j] = new half[sampleCount[i][j]]; - if (channelTypes[k] == 2) - data[k][i][j] = new float[sampleCount[i][j]]; - for (unsigned int l = 0; l < sampleCount[i][j]; l++) + if( samples>0 ) { + if (channelTypes[k] == 0) - ((unsigned int*)data[k][i][j])[l] = (i * width + j) % 2049; + data[k][i][j] = new unsigned int[sampleCount[i][j]]; if (channelTypes[k] == 1) - ((half*)data[k][i][j])[l] = (i * width + j) % 2049; + data[k][i][j] = new half[sampleCount[i][j]]; if (channelTypes[k] == 2) - ((float*)data[k][i][j])[l] = (i * width + j) % 2049; + data[k][i][j] = new float[sampleCount[i][j]]; + for (unsigned int l = 0; l < sampleCount[i][j]; l++) + { + if (channelTypes[k] == 0) + ((unsigned int*)data[k][i][j])[l] = (i * width + j) % 2049; + if (channelTypes[k] == 1) + ((half*)data[k][i][j])[l] = (i * width + j) % 2049; + if (channelTypes[k] == 2) + ((float*)data[k][i][j])[l] = (i * width + j) % 2049; + } + } + else + { + data[k][i][j] = nullptr; } } } @@ -180,25 +204,42 @@ void generateRandomFile (const std::string filename, for (int j = 0; j < width; j++) { - sampleCount[i][j] = random_int(10) + 1; + int samples = random_int(maxSamples); + + // big files write very sparse data for efficiency: most pixels have no samples + if(bigFile && (i%63!=0 || j%63!=0)) + { + samples=0; + } + sampleCount[i][j] = samples; + for (int k = 0; k < channelCount; k++) { - if (channelTypes[k] == 0) - data[k][i][j] = new unsigned int[sampleCount[i][j]]; - if (channelTypes[k] == 1) - data[k][i][j] = new half[sampleCount[i][j]]; - if (channelTypes[k] == 2) - data[k][i][j] = new float[sampleCount[i][j]]; - for (unsigned int l = 0; l < sampleCount[i][j]; l++) + + if(samples>0) { if (channelTypes[k] == 0) - ((unsigned int*)data[k][i][j])[l] = (i * width + j) % 2049; + data[k][i][j] = new unsigned int[sampleCount[i][j]]; if (channelTypes[k] == 1) - ((half*)data[k][i][j])[l] = (i * width + j) % 2049; + data[k][i][j] = new half[sampleCount[i][j]]; if (channelTypes[k] == 2) - ((float*)data[k][i][j])[l] = (i * width + j) % 2049; + data[k][i][j] = new float[sampleCount[i][j]]; + for (unsigned int l = 0; l < sampleCount[i][j]; l++) + { + if (channelTypes[k] == 0) + ((unsigned int*)data[k][i][j])[l] = (i * width + j) % 2049; + if (channelTypes[k] == 1) + ((half*)data[k][i][j])[l] = (i * width + j) % 2049; + if (channelTypes[k] == 2) + ((float*)data[k][i][j])[l] = (i * width + j) % 2049; + } + } + else + { + data[k][i][j] = nullptr; } } + } file.writePixels(1); } @@ -244,6 +285,12 @@ void readFile (const std::string & filename, assert (fileHeader.channels() == header.channels()); assert (fileHeader.type() == header.type()); + const Box2i& dataWindow = fileHeader.dataWindow(); + + int width = dataWindow.max.x - dataWindow.min.x+1; + int height = dataWindow.max.y - dataWindow.min.y+1; + + Array2D localSampleCount; localSampleCount.resizeErase(height, width); @@ -348,7 +395,7 @@ void readFile (const std::string & filename, { for (int k = 0; k < channelCount; k++) { - if(!randomChannels || read_channel[k]==1) + if( localSampleCount[i][j]>0 && (!randomChannels || read_channel[k]==1)) { if (channelTypes[k] == 0) data[k][i][j] = new unsigned int[localSampleCount[i][j]]; @@ -357,6 +404,10 @@ void readFile (const std::string & filename, if (channelTypes[k] == 2) data[k][i][j] = new float[localSampleCount[i][j]]; } + else + { + data[k][i][j] = nullptr; + } } for( int f = 0 ; f < fillChannels ; ++f ) { @@ -383,7 +434,7 @@ void readFile (const std::string & filename, { for (int k = 0; k < channelCount; k++) { - if( !randomChannels || read_channel[k]==1) + if( localSampleCount[i][j]>0 && (!randomChannels || read_channel[k]==1)) { if (channelTypes[k] == 0) data[k][i][j] = new unsigned int[localSampleCount[i][j]]; @@ -392,6 +443,10 @@ void readFile (const std::string & filename, if (channelTypes[k] == 2) data[k][i][j] = new float[localSampleCount[i][j]]; } + else + { + data[k][i][j] = nullptr; + } } for( int f = 0 ; f < fillChannels ; ++f ) { @@ -465,7 +520,8 @@ void readFile (const std::string & filename, } } -void readWriteTest(const std::string & tempDir, int channelCount, int testTimes) +void readWriteTest(const std::string & tempDir, int channelCount, int testTimes, + const Box2i& dataWindow,const Box2i& displayWindow) { cout << "Testing files with " << channelCount << " channels " << testTimes << " times." << endl << flush; @@ -489,14 +545,14 @@ void readWriteTest(const std::string & tempDir, int channelCount, int testTimes) break; } - generateRandomFile (filename, channelCount, compression, false); + generateRandomFile (filename, channelCount, compression, false , dataWindow,displayWindow); readFile (filename, channelCount, false , false ); if (channelCount>1) readFile (filename, channelCount, false , true ); remove (filename.c_str()); cout << endl << flush; - generateRandomFile (filename, channelCount, compression, true); + generateRandomFile (filename, channelCount, compression, true, dataWindow,displayWindow); readFile (filename, channelCount, true , false ); if (channelCount>1) readFile (filename, channelCount, true , true ); @@ -580,6 +636,23 @@ void testCompressionTypeChecks() }; // namespace +namespace small +{ +const int width = 273; +const int height = 173; +const int minX = 10; +const int minY = 11; +} + +namespace large +{ +const int width = 10000; +const int height = 5000; +const int minX = -22; +const int minY = -12; +} + + void testDeepScanLineBasic (const std::string &tempDir) { try @@ -593,10 +666,20 @@ void testDeepScanLineBasic (const std::string &tempDir) testCompressionTypeChecks(); - - readWriteTest (tempDir, 1, 100); - readWriteTest (tempDir, 3, 50); - readWriteTest (tempDir,10, 10); + + const Box2i largeDataWindow(V2i(large::minX, large::minY), V2i(large::minX + large::width - 1, large::minY + large::height - 1)); + const Box2i largeDisplayWindow(V2i(0, 0), V2i(large::minX + large::width * 2, large::minY + large::height * 2)); + + readWriteTest (tempDir, 1, 3 , largeDataWindow , largeDisplayWindow); + + const Box2i dataWindow(V2i(small::minX, small::minY), V2i(small::minX + small::width - 1, small::minY + small::height - 1)); + const Box2i displayWindow(V2i(0, 0), V2i(small::minX + small::width * 2, small::minY + small::height * 2)); + + readWriteTest (tempDir, 1, 50 , dataWindow , displayWindow); + readWriteTest (tempDir, 3, 25 , dataWindow , displayWindow); + readWriteTest (tempDir,10, 10 , dataWindow , displayWindow); + + ThreadPool::globalThreadPool().setNumThreads(numThreads);