Refactor VideoDecoder and CameraInput

src/plugins/score-plugin-gfx/Gfx/Graph/decoders/YUYV422.hpp

@@ -178,19 +178,12 @@ void main() {
       QRhiResourceUpdateBatch& res,
       AVFrame& frame) override
   {
-    setYPixels(res, frame.data[0], frame.linesize[0]);
-  }
-
-  void setYPixels(
-      QRhiResourceUpdateBatch& res,
-      uint8_t* pixels,
-      int stride) const noexcept
-  {
-    const auto w = decoder.width, h = decoder.height;
     auto y_tex = samplers[0].texture;
 
+    auto pixels = frame.data[0];
+    auto stride = frame.linesize[0];
     QRhiTextureUploadEntry entry{
-        0, 0, createTextureUpload(pixels, w, h, 2, stride)};
+        0, 0, createTextureUpload(pixels, frame.width, frame.height, 2, stride)};
 
     QRhiTextureUploadDescription desc{entry};
     res.uploadTexture(y_tex, desc);

src/plugins/score-plugin-media/Video/CameraInput.cpp

@@ -149,43 +149,49 @@ void CameraInput::close_file() noexcept
 
 AVFrame* CameraInput::read_frame_impl() noexcept
 {
-  AVFrame* res = nullptr;
+  ReadFrame res;
 
   if (m_stream != -1)
   {
-    AVFrame* frame = m_frames.newFrame();
     AVPacket packet;
     memset(&packet, 0, sizeof(AVPacket));
-    bool ok = false;
 
-    while (av_read_frame(m_formatContext, &packet) >= 0)
+    do
+    {
+      res = read_one_frame(m_frames.newFrame(), packet);
+    } while (res.error == AVERROR(EAGAIN));
+  }
+  return res.frame;
+}
+
+ReadFrame CameraInput::read_one_frame(AVFramePointer frame, AVPacket& packet)
+{
+  int res{};
+  while ((res = av_read_frame(m_formatContext, &packet)) >= 0)
+  {
+    if (packet.stream_index == m_stream)
     {
-      if (packet.stream_index == m_stream)
       {
-        if (enqueue_frame(&packet, &frame))
-        {
-          res = frame;
-          ok = true;
-        }
+        SCORE_ASSERT(m_codecContext);
+        auto res = enqueue_frame(&packet, std::move(frame));
 
         av_packet_unref(&packet);
-        packet = AVPacket{};
-        break;
+        return res;
       }
 
       av_packet_unref(&packet);
-      packet = AVPacket{};
+      break;
     }
 
-    if (!ok)
-    {
-      av_frame_free(&frame);
-      res = nullptr;
-    }
+    av_packet_unref(&packet);
   }
-  return res;
+  // if (res != 0 && res != AVERROR_EOF)
+  //   qDebug() << "Error while reading a frame: "
+  //            << av_make_error_string(
+  //                   global_errbuf, sizeof(global_errbuf), res);
+  av_packet_unref(&packet);
+  return {nullptr, res};
 }
-
 void CameraInput::init_scaler() noexcept
 {
   // Allocate a rescale context
@@ -289,51 +295,47 @@ void CameraInput::close_stream() noexcept
   m_stream = -1;
 }
 
-bool CameraInput::enqueue_frame(const AVPacket* pkt, AVFrame** frame) noexcept
+ReadFrame CameraInput::enqueue_frame(const AVPacket* pkt, AVFramePointer frame) noexcept
 {
-  int got_picture_ptr = 0;
-
-  if (m_codecContext && pkt && frame)
+  ReadFrame read = readVideoFrame(m_codecContext, pkt, frame.get());
+  if(!read.frame)
   {
-    int ret = avcodec_send_packet(m_codecContext, pkt);
-    if (ret < 0)
-      return ret == AVERROR_EOF ? 0 : ret;
-
-    ret = avcodec_receive_frame(m_codecContext, *frame);
-    if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
-      return ret;
-
-    if (ret >= 0)
-    {
-      got_picture_ptr = 1;
-
-      if (m_rescale)
-      {
-        // alloc an rgb frame
-        auto m_rgb = av_frame_alloc();
-        m_rgb->width = this->width;
-        m_rgb->height = this->height;
-        m_rgb->format = AV_PIX_FMT_RGBA;
-        av_frame_get_buffer(m_rgb, 0);
-
-        // 2. Resize
-        sws_scale(
-            m_rescale,
-            (*frame)->data,
-            (*frame)->linesize,
-            0,
-            this->height,
-            m_rgb->data,
-            m_rgb->linesize);
-
-        av_frame_free(frame);
-        *frame = m_rgb;
-        return true;
-      }
-    }
+    this->m_frames.enqueue_decoding_error(frame.release());
+    return read;
   }
 
-  return got_picture_ptr == 1;
+  if (m_rescale)
+  {
+    // alloc an rgb frame
+    auto rgb = m_frames.newFrame().release();
+    av_frame_copy_props(rgb, read.frame);
+    rgb->width = this->width;
+    rgb->height = this->height;
+    rgb->format = AV_PIX_FMT_RGBA;
+    av_frame_get_buffer(rgb, 0);
+
+    // 2. Resize
+    sws_scale(
+        m_rescale,
+        read.frame->data,
+        read.frame->linesize,
+        0,
+        this->height,
+        rgb->data,
+        rgb->linesize);
+
+    // 3. Free the old frame data
+    frame.reset();
+
+    // 4. Return the new frame
+    read.frame = rgb;
+  }
+  else
+  {
+    // it is already stored in "read" but well
+    read.frame = frame.release();
+  }
+  return read;
 }
 
 }

src/plugins/score-plugin-media/Video/CameraInput.hpp

@@ -48,7 +48,8 @@ class SCORE_PLUGIN_MEDIA_EXPORT CameraInput final : public VideoInterface
   AVFrame* read_frame_impl() noexcept;
   bool open_stream() noexcept;
   void close_stream() noexcept;
-  bool enqueue_frame(const AVPacket* pkt, AVFrame** frame) noexcept;
+  ReadFrame enqueue_frame(const AVPacket* pkt, AVFramePointer frame) noexcept;
+  ReadFrame read_one_frame(AVFramePointer frame, AVPacket& packet);
   void init_scaler() noexcept;
 
   static const constexpr int frames_to_buffer = 1;

src/plugins/score-plugin-media/Video/FrameQueue.cpp

@@ -1,6 +1,7 @@
 #include <Media/Libav.hpp>
 #if SCORE_HAS_LIBAV
 
+#include <Video/VideoInterface.hpp>
 #include <Video/FrameQueue.hpp>
 
 extern "C"
@@ -11,12 +12,30 @@ extern "C"
 namespace Video
 {
 
-AVFrame* FrameQueue::newFrame() noexcept
+AVFramePointer FrameQueue::newFrame() noexcept
 {
+  // We were working on a frame in this thread (e.g. during a seek, when retrying with EAGAIN..)
+  if (!m_decodeThreadFrameBuffer.empty())
+  {
+    auto f = m_decodeThreadFrameBuffer.back();
+    m_decodeThreadFrameBuffer.pop_back();
+    return AVFramePointer{f};
+  }
+
+  // Frames freed from the rendering thread
+  {
   AVFrame* f{};
   if (released.try_dequeue(f))
-    return f;
-  return av_frame_alloc();
+    return AVFramePointer{f};
+  }
+
+  // We actually need to allocate :throw_up_emoji:
+  return AVFramePointer{av_frame_alloc()};
+}
+
+void FrameQueue::enqueue_decoding_error(AVFrame* f)
+{
+  this->m_decodeThreadFrameBuffer.push_back(f);
 }
 
 void FrameQueue::enqueue(AVFrame* f)
@@ -39,26 +58,66 @@ AVFrame* FrameQueue::dequeue() noexcept
   return f;
 }
 
+void FrameQueue::set_discard_frame(AVFrame* f)
+{
+  m_discardUntil.exchange(f);
+}
+AVFrame* FrameQueue::discard_and_dequeue() noexcept
+{
+  AVFrame* f{};
+  AVFrame* prev_f{};
+
+  if (auto to_discard = m_discardUntil.exchange(nullptr))
+  {
+    while (available.try_dequeue(f) && f != to_discard)
+    {
+      release(f);
+    }
+
+    return to_discard;
+  }
+  // We only want the latest frame
+  while (available.try_dequeue(f))
+  {
+    if (prev_f)
+      release(prev_f);
+    prev_f = f;
+  }
+  return f;
+}
+
 void FrameQueue::release(AVFrame* frame) noexcept
 {
   released.enqueue(frame);
 }
 
 void FrameQueue::drain()
 {
-  AVFrame* frame{};
-  while (available.try_dequeue(frame))
   {
-    av_frame_free(&frame);
+    AVFrame* frame{};
+    while (available.try_dequeue(frame))
+    {
+      av_frame_free(&frame);
+    }
   }
 
   // TODO we must check that this is safe as the queue
   // does not support dequeueing from the same thread as the
   // enqueuing
-  while (released.try_dequeue(frame))
   {
-    av_frame_free(&frame);
+    AVFrame* frame{};
+    while (released.try_dequeue(frame))
+    {
+      av_frame_free(&frame);
+    }
+  }
+
+
+  for(auto f : m_decodeThreadFrameBuffer)
+  {
+    av_frame_free(&f);
   }
+  m_decodeThreadFrameBuffer.clear();
 }
 
 }

src/plugins/score-plugin-media/Video/FrameQueue.hpp

@@ -1,8 +1,10 @@
 #pragma once
 #include <Media/Libav.hpp>
 #if SCORE_HAS_LIBAV
-
+#include <Video/VideoInterface.hpp>
 #include <ossia/detail/lockfree_queue.hpp>
+#include <vector>
+#include <atomic>
 
 #include <score_plugin_media_export.h>
 
@@ -15,11 +17,14 @@ namespace Video
 struct SCORE_PLUGIN_MEDIA_EXPORT FrameQueue
 {
 public:
-  AVFrame* newFrame() noexcept;
+  AVFramePointer newFrame() noexcept;
 
+  void enqueue_decoding_error(AVFrame* f);
   void enqueue(AVFrame* f);
   AVFrame* dequeue() noexcept;
+  AVFrame* discard_and_dequeue() noexcept;
 
+  void set_discard_frame(AVFrame*);
   void release(AVFrame* frame) noexcept;
   void drain();
 
@@ -28,6 +33,9 @@ struct SCORE_PLUGIN_MEDIA_EXPORT FrameQueue
 private:
   ossia::spsc_queue<AVFrame*, 16> available;
   ossia::spsc_queue<AVFrame*, 16> released;
+
+  std::vector<AVFrame*> m_decodeThreadFrameBuffer;
+  std::atomic<AVFrame*> m_discardUntil{};
 };
 }
 #endif