From 0e08c959c5459e2ffd3b81b654c3ce8b71a4b42c Mon Sep 17 00:00:00 2001 From: peterhillman Date: Wed, 20 Jan 2021 14:37:21 +1300 Subject: [PATCH] Use size_t for DWA buffersize calculation (#901) * Use size_t for DWA buffersize calculation Signed-off-by: Peter Hillman * use Int64 instead of size_t for buffersize calculations Signed-off-by: Peter Hillman --- src/lib/OpenEXR/ImfDwaCompressor.cpp | 42 +++++++++++++++------------- src/lib/OpenEXR/ImfDwaCompressor.h | 20 ++++++------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/src/lib/OpenEXR/ImfDwaCompressor.cpp b/src/lib/OpenEXR/ImfDwaCompressor.cpp index b104afd30..22c75e452 100644 --- a/src/lib/OpenEXR/ImfDwaCompressor.cpp +++ b/src/lib/OpenEXR/ImfDwaCompressor.cpp @@ -2932,10 +2932,10 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // of channels we have. // - size_t maxOutBufferSize = 0; + Int64 maxOutBufferSize = 0; int numLossyDctChans = 0; - int unknownBufferSize = 0; - int rleBufferSize = 0; + Int64 unknownBufferSize = 0; + Int64 rleBufferSize = 0; int maxLossyDctAcSize = (int)ceil ((float)numScanLines() / 8.0f) * (int)ceil ((float)(_max[0] - _min[0] + 1) / 8.0f) * @@ -2945,6 +2945,8 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) (int)ceil ((float)(_max[0] - _min[0] + 1) / 8.0f) * sizeof (unsigned short); + Int64 pixelCount = static_cast(numScanLines()) * static_cast(_max[0] - _min[0] + 1); + for (unsigned int chan = 0; chan < _channelData.size(); ++chan) { switch (_channelData[chan].compression) @@ -2971,8 +2973,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // of the source data. // - int rleAmount = 2 * numScanLines() * (_max[0] - _min[0] + 1) * - OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type); + Int64 rleAmount = 2 * pixelCount * OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type); rleBufferSize += rleAmount; } @@ -2981,8 +2982,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) case UNKNOWN: - unknownBufferSize += numScanLines() * (_max[0] - _min[0] + 1) * - OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type); + unknownBufferSize += pixelCount * OPENEXR_IMF_NAMESPACE::pixelTypeSize (_channelData[chan].type); break; default: @@ -2999,13 +2999,13 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // which could take slightly more space // - maxOutBufferSize += compressBound ((uLongf)rleBufferSize); + maxOutBufferSize += compressBound (rleBufferSize); // // And the same goes for the UNKNOWN data // - maxOutBufferSize += compressBound ((uLongf)unknownBufferSize); + maxOutBufferSize += compressBound (unknownBufferSize); // // Allocate a zip/deflate compressor big enought to hold the DC data @@ -3052,7 +3052,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // to Huffman encoding // - if (static_cast(maxLossyDctAcSize * numLossyDctChans) > _packedAcBufferSize) + if (static_cast(maxLossyDctAcSize * numLossyDctChans) > _packedAcBufferSize) { _packedAcBufferSize = maxLossyDctAcSize * numLossyDctChans; if (_packedAcBuffer != 0) @@ -3064,7 +3064,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // _packedDcBuffer holds one quantized DCT coef per 8x8 block // - if (static_cast(maxLossyDctDcSize * numLossyDctChans) > _packedDcBufferSize) + if (static_cast(maxLossyDctDcSize * numLossyDctChans) > _packedDcBufferSize) { _packedDcBufferSize = maxLossyDctDcSize * numLossyDctChans; if (_packedDcBuffer != 0) @@ -3072,7 +3072,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) _packedDcBuffer = new char[_packedDcBufferSize]; } - if (static_cast(rleBufferSize) > _rleBufferSize) + if ( rleBufferSize > _rleBufferSize ) { _rleBufferSize = rleBufferSize; if (_rleBuffer != 0) @@ -3091,7 +3091,7 @@ DwaCompressor::initializeBuffers (size_t &outBufferSize) // all in one swoop (for each compression scheme). // - int planarUncBufferSize[NUM_COMPRESSOR_SCHEMES]; + Int64 planarUncBufferSize[NUM_COMPRESSOR_SCHEMES]; for (int i=0; i 0) { planarUncBufferSize[UNKNOWN] = - compressBound ((uLongf)planarUncBufferSize[UNKNOWN]); + compressBound (planarUncBufferSize[UNKNOWN]); } for (int i = 0; i < NUM_COMPRESSOR_SCHEMES; ++i) { - if (static_cast(planarUncBufferSize[i]) > _planarUncBufferSize[i]) + if ( planarUncBufferSize[i] > _planarUncBufferSize[i]) { _planarUncBufferSize[i] = planarUncBufferSize[i]; if (_planarUncBuffer[i] != 0) delete[] _planarUncBuffer[i]; + + if (planarUncBufferSize[i] > std::numeric_limits::max()) + { + throw IEX_NAMESPACE::ArgExc("DWA buffers too large"); + } + _planarUncBuffer[i] = new char[planarUncBufferSize[i]]; } } diff --git a/src/lib/OpenEXR/ImfDwaCompressor.h b/src/lib/OpenEXR/ImfDwaCompressor.h index 04550a1da..012819c75 100644 --- a/src/lib/OpenEXR/ImfDwaCompressor.h +++ b/src/lib/OpenEXR/ImfDwaCompressor.h @@ -169,16 +169,16 @@ class DwaCompressor: public Compressor std::vector _cscSets; std::vector _channelRules; - char *_packedAcBuffer; - size_t _packedAcBufferSize; - char *_packedDcBuffer; - size_t _packedDcBufferSize; - char *_rleBuffer; - size_t _rleBufferSize; - char *_outBuffer; - size_t _outBufferSize; - char *_planarUncBuffer[NUM_COMPRESSOR_SCHEMES]; - size_t _planarUncBufferSize[NUM_COMPRESSOR_SCHEMES]; + char* _packedAcBuffer; + Int64 _packedAcBufferSize; + char* _packedDcBuffer; + Int64 _packedDcBufferSize; + char* _rleBuffer; + Int64 _rleBufferSize; + char* _outBuffer; + Int64 _outBufferSize; + char* _planarUncBuffer[NUM_COMPRESSOR_SCHEMES]; + Int64 _planarUncBufferSize[NUM_COMPRESSOR_SCHEMES]; Zip *_zip; float _dwaCompressionLevel;