diff --git a/deflect/ImageJpegCompressor.cpp b/deflect/ImageJpegCompressor.cpp
index 4aedec4..a4bf8d0 100644
--- a/deflect/ImageJpegCompressor.cpp
+++ b/deflect/ImageJpegCompressor.cpp
@@ -109,14 +109,18 @@ QByteArray ImageJpegCompressor::computeJpeg(const ImageWrapper& sourceImage,
     const int tjPitch = sourceImage.width * sourceImage.getBytesPerPixel();
     const int tjHeight = imageRegion.height();
     const int tjPixelFormat = _getTurboJpegFormat(sourceImage.pixelFormat);
-    unsigned char* tjJpegBuf = nullptr;
-    unsigned long tjJpegSize = 0;
+
     const int tjJpegSubsamp = _getTurboJpegSubsamp(sourceImage.subsampling);
+    unsigned long tjJpegSize = tjBufSize(tjWidth, tjHeight, tjJpegSubsamp);
+
+    _tjJpegBuf.resize(tjJpegSize);
+
     const int tjJpegQual = sourceImage.compressionQuality;
-    const int tjFlags = 0; // or: TJFLAG_BOTTOMUP
+    const int tjFlags = TJFLAG_NOREALLOC; // or: TJFLAG_BOTTOMUP
 
+    auto ptr = _tjJpegBuf.data();
     int err = tjCompress2(_tjHandle, tjSrcBuffer, tjWidth, tjPitch, tjHeight,
-                          tjPixelFormat, &tjJpegBuf, &tjJpegSize, tjJpegSubsamp,
+                          tjPixelFormat, &ptr, &tjJpegSize, tjJpegSubsamp,
                           tjJpegQual, tjFlags);
     if (err != 0)
     {
@@ -124,12 +128,6 @@ QByteArray ImageJpegCompressor::computeJpeg(const ImageWrapper& sourceImage,
         return QByteArray();
     }
 
-    // move the JPEG buffer to a byte array
-    const QByteArray jpegData((char*)tjJpegBuf, tjJpegSize);
-
-    // free the libjpeg-turbo allocated memory
-    tjFree(tjJpegBuf);
-
-    return jpegData;
+    return QByteArray((const char*)ptr, tjJpegSize);
 }
 }
diff --git a/deflect/ImageJpegCompressor.h b/deflect/ImageJpegCompressor.h
index b254024..6cf24e7 100644
--- a/deflect/ImageJpegCompressor.h
+++ b/deflect/ImageJpegCompressor.h
@@ -71,6 +71,7 @@ class ImageJpegCompressor
 
 private:
     tjhandle _tjHandle;
+    std::vector<unsigned char> _tjJpegBuf;
 };
 }
 
diff --git a/deflect/ImageSegmenter.cpp b/deflect/ImageSegmenter.cpp
index 14bfae2..ea18ee6 100644
--- a/deflect/ImageSegmenter.cpp
+++ b/deflect/ImageSegmenter.cpp
@@ -69,6 +69,21 @@ bool ImageSegmenter::generate(const ImageWrapper& image, const Handler& handler)
     return _generateRaw(image, handler);
 }
 
+Segment ImageSegmenter::compressSingleSegment(const ImageWrapper& image)
+{
+#ifdef DEFLECT_USE_LIBJPEGTURBO
+    auto segments = _generateSegments(image);
+    if (segments.size() > 1)
+        throw std::runtime_error(
+            "compressSingleSegment only works for small images");
+    ImageSegmenter::_computeJpeg(segments[0], false);
+    return segments[0];
+#else
+    throw std::runtime_error(
+        "LibJpegTurbo not available, needed for compressSingleSegment");
+#endif
+}
+
 void ImageSegmenter::setNominalSegmentDimensions(const uint width,
                                                  const uint height)
 {
@@ -85,7 +100,7 @@ bool ImageSegmenter::_generateJpeg(const ImageWrapper& image,
 
     // start creating JPEGs for each segment, in parallel
     QtConcurrent::map(segments, std::bind(&ImageSegmenter::_computeJpeg, this,
-                                          std::placeholders::_1));
+                                          std::placeholders::_1, true));
 
     // Sending compressed jpeg segments while they arrive in the queue.
     // Note: Qt insists that sending (by calling handler()) should happen
@@ -108,7 +123,7 @@ bool ImageSegmenter::_generateJpeg(const ImageWrapper& image,
 #endif
 }
 
-void ImageSegmenter::_computeJpeg(Segment& segment)
+void ImageSegmenter::_computeJpeg(Segment& segment, const bool sendSegment)
 {
 #ifdef DEFLECT_USE_LIBJPEGTURBO
     QRect imageRegion(segment.parameters.x - segment.sourceImage->x,
@@ -124,7 +139,8 @@ void ImageSegmenter::_computeJpeg(Segment& segment)
     segment.imageData =
         compressor.localData().computeJpeg(*segment.sourceImage, imageRegion);
     segment.parameters.dataType = DataType::jpeg;
-    _sendQueue.enqueue(segment);
+    if (sendSegment)
+        _sendQueue.enqueue(segment);
 #endif
 }
 
diff --git a/deflect/ImageSegmenter.h b/deflect/ImageSegmenter.h
index 84f6f76..9f1de90 100644
--- a/deflect/ImageSegmenter.h
+++ b/deflect/ImageSegmenter.h
@@ -91,6 +91,16 @@ class ImageSegmenter
      */
     DEFLECT_API void setNominalSegmentDimensions(uint width, uint height);
 
+    /**
+     * For a small input image (tested with 64x64, possible for <=512 as well),
+     * directly compress it to a single segment which will be enqueued for
+     * sending.
+     *
+     * @param image The image to be compressed
+     * @return the compressed segment
+     */
+    DEFLECT_API Segment compressSingleSegment(const ImageWrapper& image);
+
 private:
     struct SegmentationInfo
     {
@@ -105,7 +115,7 @@ class ImageSegmenter
     };
 
     bool _generateJpeg(const ImageWrapper& image, const Handler& handler);
-    void _computeJpeg(Segment& task);
+    void _computeJpeg(Segment& task, bool sendSegment);
     bool _generateRaw(const ImageWrapper& image, const Handler& handler) const;
 
     Segments _generateSegments(const ImageWrapper& image) const;
diff --git a/deflect/StreamSendWorker.cpp b/deflect/StreamSendWorker.cpp
index 9a8e6b0..6324f4f 100644
--- a/deflect/StreamSendWorker.cpp
+++ b/deflect/StreamSendWorker.cpp
@@ -49,6 +49,7 @@
 namespace
 {
 const unsigned int SEGMENT_SIZE = 512;
+const unsigned int SMALL_IMAGE_SIZE = 64;
 }
 
 namespace deflect
@@ -128,7 +129,17 @@ Stream::Future StreamSendWorker::enqueueImage(const ImageWrapper& image,
         return promise.get_future();
     }
 
-    auto tasks = std::vector<Task>{[this, image] { return _sendImage(image); }};
+    std::vector<Task> tasks;
+
+    if (image.width <= SMALL_IMAGE_SIZE && image.height <= SMALL_IMAGE_SIZE &&
+        image.compressionPolicy == COMPRESSION_ON)
+    {
+        auto segment = _imageSegmenter.compressSingleSegment(image);
+        tasks.emplace_back([this, segment] { return _sendSegment(segment); });
+    }
+    else
+        tasks.emplace_back([this, image] { return _sendImage(image); });
+
     if (finish)
         tasks.emplace_back([this] { return _sendFinish(); });
 
diff --git a/doc/Changelog.md b/doc/Changelog.md
index 6394c80..4e64a37 100644
--- a/doc/Changelog.md
+++ b/doc/Changelog.md
@@ -5,6 +5,9 @@ Changelog {#Changelog}
 
 ### 0.13.1 (git master)
 
+* [174](https://github.com/BlueBrain/Deflect/pull/174):
+  Performance improvements for sending small images (tested with 64x64):
+  directly compress them in the call thread
 * [173](https://github.com/BlueBrain/Deflect/pull/173):
   Fix server: disabling system proxy which are default starting Qt 5.8