diff --git a/apps/SimpleStreamer/main.cpp b/apps/SimpleStreamer/main.cpp index e7997e6..6927204 100644 --- a/apps/SimpleStreamer/main.cpp +++ b/apps/SimpleStreamer/main.cpp @@ -59,6 +59,7 @@ bool deflectInteraction = false; bool deflectCompressImage = true; bool deflectStereoStreamLeft = false; bool deflectStereoStreamRight = false; +bool deflectStereoSideBySide = false; unsigned int deflectCompressionQuality = 75; std::string deflectHost; std::string deflectStreamId = "SimpleStreamer"; @@ -115,6 +116,8 @@ void syntax(const int exitStatus) std::cout << " -r enable stereo streaming, right image " "only (default: OFF)" << std::endl; + std::cout << " -2 enable side-by-side stereo (default: OFF)" + << std::endl; exit(exitStatus); } @@ -152,6 +155,9 @@ void readCommandLineArguments(int argc, char** argv) case 'r': deflectStereoStreamRight = true; break; + case '2': + deflectStereoSideBySide = true; + break; case 'h': syntax(EXIT_SUCCESS); default: @@ -253,6 +259,28 @@ struct Image image.height, 4); return image; } + + static Image sideBySide(const Image& left, const Image& right) + { + Image image; + image.width = left.width + right.width; + image.height = std::max(left.height, right.height); + image.data.resize(image.width * image.height * 4); + for (uint j = 0; j < left.height; ++j) + { + const auto lineBegin = left.data.data() + 4 * j * left.width; + const auto lineOut = image.data.data() + 4 * j * image.width; + std::copy(lineBegin, lineBegin + 4 * left.width, lineOut); + } + for (uint j = 0; j < right.height; ++j) + { + const auto lineBegin = right.data.data() + 4 * j * right.width; + const auto lineOut = + image.data.data() + 4 * j * image.width + 4 * left.width; + std::copy(lineBegin, lineBegin + 4 * right.width, lineOut); + } + return image; + } }; bool send(const Image& image, const deflect::View view) @@ -274,6 +302,27 @@ bool timeout(const float sec) return std::chrono::duration{clock::now() - start}.count() > sec; } +void drawLeft() +{ + glClearColor(0.7, 0.3, 0.3, 1.0); + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + glutSolidTeapot(1.f); +} + +void drawRight() +{ + glClearColor(0.3, 0.7, 0.3, 1.0); + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + glutSolidTeapot(1.f); +} + +void drawMono() +{ + glClearColor(0.5, 0.5, 0.5, 1.0); + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + glutSolidTeapot(1.f); +} + void display() { static Camera camera; @@ -282,7 +331,16 @@ void display() bool success = false; bool waitToStart = false; static bool deflectFirstEventReceived = false; - if (deflectStereoStreamLeft || deflectStereoStreamRight) + if (deflectStereoSideBySide) + { + drawLeft(); + const auto leftImage = Image::readGlBuffer(); + drawRight(); + const auto rightImage = Image::readGlBuffer(); + const auto sideBySideImage = Image::sideBySide(leftImage, rightImage); + success = send(sideBySideImage, deflect::View::side_by_side); + } + else if (deflectStereoStreamLeft || deflectStereoStreamRight) { // Poor man's attempt to synchronise the start of separate stereo // streams (waiting on first event from server or 5 sec. timeout). @@ -294,17 +352,13 @@ void display() if (deflectStereoStreamLeft && !waitToStart) { - glClearColor(0.7, 0.3, 0.3, 1.0); - glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); - glutSolidTeapot(1.f); + drawLeft(); const auto leftImage = Image::readGlBuffer(); success = send(leftImage, deflect::View::left_eye); } if (deflectStereoStreamRight && !waitToStart) { - glClearColor(0.3, 0.7, 0.3, 1.0); - glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); - glutSolidTeapot(1.f); + drawRight(); const auto rightImage = Image::readGlBuffer(); success = (!deflectStereoStreamLeft || success) && send(rightImage, deflect::View::right_eye); @@ -312,9 +366,7 @@ void display() } else { - glClearColor(0.5, 0.5, 0.5, 1.0); - glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); - glutSolidTeapot(1.f); + drawMono(); success = send(Image::readGlBuffer(), deflect::View::mono); } diff --git a/deflect/ImageSegmenter.cpp b/deflect/ImageSegmenter.cpp index 3c6ed54..14bfae2 100644 --- a/deflect/ImageSegmenter.cpp +++ b/deflect/ImageSegmenter.cpp @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* Stefan.Eilemann@epfl.ch */ /* All rights reserved. */ @@ -45,16 +45,21 @@ #include "ImageJpegCompressor.h" #endif +#include #include -#include + #include +#include namespace deflect { -ImageSegmenter::ImageSegmenter() - : _nominalSegmentWidth(0) - , _nominalSegmentHeight(0) +namespace +{ +bool _isOnRightSideOfSideBySideImage(const Segment& segment) { + return segment.sourceImage->view == View::side_by_side && + segment.view == View::right_eye; +} } bool ImageSegmenter::generate(const ImageWrapper& image, const Handler& handler) @@ -64,28 +69,28 @@ bool ImageSegmenter::generate(const ImageWrapper& image, const Handler& handler) return _generateRaw(image, handler); } +void ImageSegmenter::setNominalSegmentDimensions(const uint width, + const uint height) +{ + _nominalSegmentWidth = width; + _nominalSegmentHeight = height; +} + bool ImageSegmenter::_generateJpeg(const ImageWrapper& image, const Handler& handler) { #ifdef DEFLECT_USE_LIBJPEGTURBO - const SegmentParametersList& params = _generateSegmentParameters(image); - // The resulting Jpeg segments - std::vector segments; - for (const auto& param : params) - { - Segment segment; - segment.parameters = param; - segment.sourceImage = ℑ - segments.push_back(segment); - } + auto segments = _generateSegments(image); - // create JPEGs for each segment, in parallel + // start creating JPEGs for each segment, in parallel QtConcurrent::map(segments, std::bind(&ImageSegmenter::_computeJpeg, this, std::placeholders::_1)); - // send ready compressed jpeg images from here. It's the thread where the - // socket lives, and Qt insists on that to not violate this contract. + // Sending compressed jpeg segments while they arrive in the queue. + // Note: Qt insists that sending (by calling handler()) should happen + // exclusively from the QThread where the socket lives. Sending from the + // worker threads triggers a qWarning. bool result = true; for (size_t i = 0; i < segments.size(); ++i) if (!handler(_sendQueue.dequeue())) @@ -110,9 +115,15 @@ void ImageSegmenter::_computeJpeg(Segment& segment) segment.parameters.y - segment.sourceImage->y, segment.parameters.width, segment.parameters.height); - ImageJpegCompressor compressor; + if (_isOnRightSideOfSideBySideImage(segment)) + imageRegion.translate(segment.sourceImage->width / 2, 0); + + // turbojpeg handles need to be per thread, and this function is called from + // multiple threads by QtConcurrent::map + static QThreadStorage compressor; segment.imageData = - compressor.computeJpeg(*segment.sourceImage, imageRegion); + compressor.localData().computeJpeg(*segment.sourceImage, imageRegion); + segment.parameters.dataType = DataType::jpeg; _sendQueue.enqueue(segment); #endif } @@ -120,19 +131,15 @@ void ImageSegmenter::_computeJpeg(Segment& segment) bool ImageSegmenter::_generateRaw(const ImageWrapper& image, const Handler& handler) const { - const SegmentParametersList& paramList = _generateSegmentParameters(image); - - // resulting Raw segments - for (SegmentParametersList::const_iterator it = paramList.begin(); - it != paramList.end(); ++it) + auto segments = _generateSegments(image); + for (auto& segment : segments) { - Segment segment; - segment.parameters = *it; segment.imageData.reserve(segment.parameters.width * segment.parameters.height * image.getBytesPerPixel()); + segment.parameters.dataType = DataType::rgba; - if (paramList.size() == 1) + if (segments.size() == 1) { // If we are not segmenting the image, just append the image data segment.imageData.append((const char*)image.data, @@ -141,17 +148,20 @@ bool ImageSegmenter::_generateRaw(const ImageWrapper& image, else // Copy the image subregion { // assume imageBuffer isn't padded - const size_t imagePitch = image.width * image.getBytesPerPixel(); - const size_t offset = - segment.parameters.y * imagePitch + - segment.parameters.x * image.getBytesPerPixel(); + const auto bytesPerPixel = image.getBytesPerPixel(); + const size_t imagePitch = image.width * bytesPerPixel; + size_t offset = segment.parameters.y * imagePitch + + segment.parameters.x * bytesPerPixel; + + if (_isOnRightSideOfSideBySideImage(segment)) + offset += segment.sourceImage->width / 2 * bytesPerPixel; + const char* lineData = (const char*)image.data + offset; - for (unsigned int i = 0; i < segment.parameters.height; ++i) + for (uint i = 0; i < segment.parameters.height; ++i) { - segment.imageData.append(lineData, - segment.parameters.width * - image.getBytesPerPixel()); + segment.imageData.append(lineData, segment.parameters.width * + bytesPerPixel); lineData += imagePitch; } } @@ -163,113 +173,92 @@ bool ImageSegmenter::_generateRaw(const ImageWrapper& image, return true; } -void ImageSegmenter::setNominalSegmentDimensions(const unsigned int width, - const unsigned int height) +Segments ImageSegmenter::_generateSegments(const ImageWrapper& image) const { - _nominalSegmentWidth = width; - _nominalSegmentHeight = height; -} + Segments segments; + for (const auto& params : _makeSegmentParameters(image)) + { + Segment segment; + segment.parameters = params; + segment.view = + image.view == View::side_by_side ? View::left_eye : image.view; + segment.sourceImage = ℑ + segments.push_back(segment); + } -#ifdef UNIORM_SEGMENT_WIDTH -SegmentParametersList ImageSegmenter::generateSegmentParameters( - const ImageWrapper& image) const -{ - unsigned int numSubdivisionsX = 1; - unsigned int numSubdivisionsY = 1; + if (image.view == View::side_by_side) + { + if (image.width % 2 != 0) + throw std::runtime_error("side_by_side image width must be even!"); - unsigned int uniformSegmentWidth = image.width; - unsigned int uniformSegmentHeight = image.height; + // create copy of segments for right view + auto segmentsRight = segments; + for (auto& segment : segmentsRight) + segment.view = View::right_eye; - bool segmentImage = (nominalSegmentWidth_ > 0 && nominalSegmentHeight_ > 0); - if (segmentImage) - { - numSubdivisionsX = (unsigned int)floor( - (float)image.width / (float)nominalSegmentWidth_ + 0.5); - numSubdivisionsY = (unsigned int)floor( - (float)image.height / (float)nominalSegmentHeight_ + 0.5); - - uniformSegmentWidth = - (unsigned int)((float)image.width / (float)numSubdivisionsX); - uniformSegmentHeight = - (unsigned int)((float)image.height / (float)numSubdivisionsY); + segments.insert(segments.end(), segmentsRight.begin(), + segmentsRight.end()); } - // now, create parameters for each segment - SegmentParametersList parameters; + return segments; +} + +SegmentParametersList ImageSegmenter::_makeSegmentParameters( + const ImageWrapper& image) const +{ + const auto info = _makeSegmentationInfo(image); - for (unsigned int i = 0; i < numSubdivisionsX; ++i) + SegmentParametersList parameters; + for (uint j = 0; j < info.countY; ++j) { - for (unsigned int j = 0; j < numSubdivisionsY; ++j) + for (uint i = 0; i < info.countX; ++i) { SegmentParameters p; - - p.x = image.x + i * uniformSegmentWidth; - p.y = image.y + j * uniformSegmentHeight; - p.width = uniformSegmentWidth; - p.height = uniformSegmentHeight; - p.compressed = (image.compressionPolicy == COMPRESSION_ON); - + p.x = image.x + i * info.width; + p.y = image.y + j * info.height; + p.width = (i < info.countX - 1) ? info.width : info.lastWidth; + p.height = (j < info.countY - 1) ? info.height : info.lastHeight; parameters.push_back(p); } } - return parameters; } -#else -SegmentParametersList ImageSegmenter::_generateSegmentParameters( + +ImageSegmenter::SegmentationInfo ImageSegmenter::_makeSegmentationInfo( const ImageWrapper& image) const { - unsigned int numSubdivisionsX = 1; - unsigned int numSubdivisionsY = 1; + const auto imageWidth = + image.view == View::side_by_side ? image.width / 2 : image.width; - unsigned int lastSegmentWidth = image.width; - unsigned int lastSegmentHeight = image.height; + SegmentationInfo info; + info.width = _nominalSegmentWidth; + info.height = _nominalSegmentHeight; - bool segmentImage = (_nominalSegmentWidth > 0 && _nominalSegmentHeight > 0); - if (segmentImage) + if (_nominalSegmentWidth == 0 || _nominalSegmentHeight == 0) { - numSubdivisionsX = image.width / _nominalSegmentWidth + 1; - numSubdivisionsY = image.height / _nominalSegmentHeight + 1; - - lastSegmentWidth = image.width % _nominalSegmentWidth; - lastSegmentHeight = image.height % _nominalSegmentHeight; - - if (lastSegmentWidth == 0) - { - lastSegmentWidth = _nominalSegmentWidth; - --numSubdivisionsX; - } - if (lastSegmentHeight == 0) - { - lastSegmentHeight = _nominalSegmentHeight; - --numSubdivisionsY; - } + info.countX = 1; + info.countY = 1; + info.lastWidth = imageWidth; + info.lastHeight = image.height; + return info; } - // now, create parameters for each segment - SegmentParametersList parameters; - - for (unsigned int j = 0; j < numSubdivisionsY; ++j) - { - for (unsigned int i = 0; i < numSubdivisionsX; ++i) - { - SegmentParameters p; + info.countX = imageWidth / _nominalSegmentWidth + 1; + info.countY = image.height / _nominalSegmentHeight + 1; - p.x = image.x + i * _nominalSegmentWidth; - p.y = image.y + j * _nominalSegmentHeight; - p.width = (i < numSubdivisionsX - 1) ? _nominalSegmentWidth - : lastSegmentWidth; - p.height = (j < numSubdivisionsY - 1) ? _nominalSegmentHeight - : lastSegmentHeight; - p.dataType = (image.compressionPolicy == COMPRESSION_ON) - ? DataType::jpeg - : DataType::rgba; + info.lastWidth = imageWidth % _nominalSegmentWidth; + info.lastHeight = image.height % _nominalSegmentHeight; - parameters.push_back(p); - } + if (info.lastWidth == 0) + { + info.lastWidth = _nominalSegmentWidth; + --info.countX; } - - return parameters; + if (info.lastHeight == 0) + { + info.lastHeight = _nominalSegmentHeight; + --info.countY; + } + return info; } -#endif } diff --git a/deflect/ImageSegmenter.h b/deflect/ImageSegmenter.h index 159995c..84f6f76 100644 --- a/deflect/ImageSegmenter.h +++ b/deflect/ImageSegmenter.h @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright (c) 2013-2016, EPFL/Blue Brain Project */ +/* Copyright (c) 2013-2017, EPFL/Blue Brain Project */ /* Raphael Dumusc */ /* Stefan.Eilemann@epfl.ch */ /* All rights reserved. */ @@ -48,7 +48,6 @@ #include #include -#include namespace deflect { @@ -59,7 +58,7 @@ class ImageSegmenter { public: /** Construct an ImageSegmenter. */ - DEFLECT_API ImageSegmenter(); + DEFLECT_API ImageSegmenter() = default; /** Function called on each segment. */ using Handler = std::function; @@ -90,19 +89,32 @@ class ImageSegmenter * @param width The nominal width of the segments to generate (default: 0) * @param height The nominal height of the segments to generate (default: 0) */ - DEFLECT_API void setNominalSegmentDimensions(unsigned int width, - unsigned int height); + DEFLECT_API void setNominalSegmentDimensions(uint width, uint height); private: - SegmentParametersList _generateSegmentParameters( - const ImageWrapper& image) const; + struct SegmentationInfo + { + uint width = 0; + uint height = 0; + + uint countX = 0; + uint countY = 0; + + uint lastWidth = 0; + uint lastHeight = 0; + }; bool _generateJpeg(const ImageWrapper& image, const Handler& handler); - bool _generateRaw(const ImageWrapper& image, const Handler& handler) const; void _computeJpeg(Segment& task); + bool _generateRaw(const ImageWrapper& image, const Handler& handler) const; + + Segments _generateSegments(const ImageWrapper& image) const; + SegmentParametersList _makeSegmentParameters( + const ImageWrapper& image) const; + SegmentationInfo _makeSegmentationInfo(const ImageWrapper& image) const; - unsigned int _nominalSegmentWidth; - unsigned int _nominalSegmentHeight; + uint _nominalSegmentWidth = 0; + uint _nominalSegmentHeight = 0; MTQueue _sendQueue; }; diff --git a/deflect/StreamSendWorker.cpp b/deflect/StreamSendWorker.cpp index ffd6bf7..9a8e6b0 100644 --- a/deflect/StreamSendWorker.cpp +++ b/deflect/StreamSendWorker.cpp @@ -83,7 +83,7 @@ void StreamSendWorker::run() lock.unlock(); bool success = true; - for (auto&& task : request.tasks) + for (auto& task : request.tasks) { if (!task()) { @@ -190,8 +190,7 @@ bool StreamSendWorker::_sendImage(const ImageWrapper& image) { const auto sendFunc = std::bind(&StreamSendWorker::_sendSegment, this, std::placeholders::_1); - return _sendImageView(image.view) && - _imageSegmenter.generate(image, sendFunc); + return _imageSegmenter.generate(image, sendFunc); } bool StreamSendWorker::_sendImageView(const View view) @@ -202,6 +201,13 @@ bool StreamSendWorker::_sendImageView(const View view) bool StreamSendWorker::_sendSegment(const Segment& segment) { + if (segment.view != _currentView) + { + if (!_sendImageView(segment.view)) + return false; + _currentView = segment.view; + } + auto message = QByteArray{(const char*)(&segment.parameters), sizeof(SegmentParameters)}; message.append(segment.imageData); diff --git a/deflect/StreamSendWorker.h b/deflect/StreamSendWorker.h index f9e6897..8636664 100644 --- a/deflect/StreamSendWorker.h +++ b/deflect/StreamSendWorker.h @@ -108,6 +108,7 @@ class StreamSendWorker : public QThread std::mutex _mutex; std::condition_variable _condition; bool _running = false; + View _currentView = View::mono; /** Main QThread loop doing asynchronous processing of queued tasks. */ void run() final; diff --git a/deflect/types.h b/deflect/types.h index 83a6687..0b30208 100644 --- a/deflect/types.h +++ b/deflect/types.h @@ -54,7 +54,8 @@ enum class View : std::uint8_t { mono, left_eye, - right_eye + right_eye, + side_by_side }; /** Sub-sampling of the image chrominance components in YCbCr color space. */ diff --git a/doc/Changelog.md b/doc/Changelog.md index e7ff683..61aa1e7 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -3,6 +3,8 @@ Changelog {#Changelog} ## Deflect 0.13 (git master) +* [163](https://github.com/BlueBrain/Deflect/pull/163): + Stereo streaming accepts side-by-side images as input. * [161](https://github.com/BlueBrain/Deflect/pull/161): DesktopStreamer OSX: the default hosts are read from a json file in the app bundle. This is useful for external users who want to adapt the official