From 3292d6447aae339f090e3bc49a71432f1cd58c03 Mon Sep 17 00:00:00 2001 From: kirksaunders Date: Wed, 13 Dec 2023 22:43:35 -0500 Subject: [PATCH] Add feature to enforce a minimum frame display latency --- app/cli/commandlineparser.cpp | 8 + app/gui/SettingsView.qml | 39 +++++ app/settings/streamingpreferences.cpp | 3 + app/settings/streamingpreferences.h | 5 +- app/streaming/session.cpp | 20 ++- app/streaming/session.h | 2 +- app/streaming/video/decoder.h | 1 + .../video/ffmpeg-renderers/pacer/pacer.cpp | 164 ++++++++++++++++-- .../video/ffmpeg-renderers/pacer/pacer.h | 13 +- app/streaming/video/ffmpeg.cpp | 3 +- 10 files changed, 232 insertions(+), 26 deletions(-) diff --git a/app/cli/commandlineparser.cpp b/app/cli/commandlineparser.cpp index 23fc65cd3..d73dba55c 100644 --- a/app/cli/commandlineparser.cpp +++ b/app/cli/commandlineparser.cpp @@ -374,6 +374,7 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference parser.addChoiceOption("capture-system-keys", "capture system key combos", m_CaptureSysKeysModeMap.keys()); parser.addChoiceOption("video-codec", "video codec", m_VideoCodecMap.keys()); parser.addChoiceOption("video-decoder", "video decoder", m_VideoDecoderMap.keys()); + parser.addValueOption("minimum-latency", "Minimum latency"); if (!parser.parse(args)) { parser.showError(parser.errorText()); @@ -500,6 +501,13 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference preferences->videoDecoderSelection = mapValue(m_VideoDecoderMap, parser.getChoiceOptionValue("video-decoder")); } + if (parser.isSet("minimum-latency")) { + preferences->minimumLatency = parser.getIntOption("minimum-latency"); + if (!inRange(preferences->minimumLatency, 0, 50)) { + parser.showError("Minimum latency must be in range: 0 - 50"); + } + } + // This method will not return and terminates the process if --version or // --help is specified parser.handleHelpAndVersionOptions(); diff --git a/app/gui/SettingsView.qml b/app/gui/SettingsView.qml index da6e310da..b71ea7b23 100644 --- a/app/gui/SettingsView.qml +++ b/app/gui/SettingsView.qml @@ -812,6 +812,45 @@ Flickable { ToolTip.visible: hovered ToolTip.text: qsTr("Frame pacing reduces micro-stutter by delaying frames that come in too early") } + + Label { + width: parent.width + id: minimumLatencyTitle + text: qsTr("Minimum latency:") + font.pointSize: 12 + wrapMode: Text.Wrap + } + + Label { + width: parent.width + id: minimumLatencyDesc + text: qsTr("A lower bound on display latency. Set higher to account for more jitter in your connection, at the cost of increased display latency.") + font.pointSize: 9 + wrapMode: Text.Wrap + } + + Slider { + id: minimumLatencySlider + + value: StreamingPreferences.minimumLatency + + stepSize: 1 + from : 0 + to: 50 + + snapMode: "SnapOnRelease" + width: Math.min(minimumLatencyDesc.implicitWidth, parent.width) + + onValueChanged: { + minimumLatencyTitle.text = qsTr("Minimum latency: %1 msec").arg(value) + StreamingPreferences.minimumLatency = value + } + + Component.onCompleted: { + // Refresh the text after translations change + languageChanged.connect(onValueChanged) + } + } } } diff --git a/app/settings/streamingpreferences.cpp b/app/settings/streamingpreferences.cpp index 6a883bc08..304bb1ded 100644 --- a/app/settings/streamingpreferences.cpp +++ b/app/settings/streamingpreferences.cpp @@ -45,6 +45,7 @@ #define SER_CAPTURESYSKEYS "capturesyskeys" #define SER_KEEPAWAKE "keepawake" #define SER_LANGUAGE "language" +#define SER_MINIMUMLATENCY "minimumlatency" #define CURRENT_DEFAULT_VER 2 @@ -122,6 +123,7 @@ void StreamingPreferences::reload() : UIDisplayMode::UI_MAXIMIZED)).toInt()); language = static_cast(settings.value(SER_LANGUAGE, static_cast(Language::LANG_AUTO)).toInt()); + minimumLatency = settings.value(SER_MINIMUMLATENCY, 0).toInt(); // Perform default settings updates as required based on last default version @@ -295,6 +297,7 @@ void StreamingPreferences::save() settings.setValue(SER_SWAPFACEBUTTONS, swapFaceButtons); settings.setValue(SER_CAPTURESYSKEYS, captureSysKeysMode); settings.setValue(SER_KEEPAWAKE, keepAwake); + settings.setValue(SER_MINIMUMLATENCY, minimumLatency); } int StreamingPreferences::getDefaultBitrate(int width, int height, int fps) diff --git a/app/settings/streamingpreferences.h b/app/settings/streamingpreferences.h index 0e2cf365e..43183ca59 100644 --- a/app/settings/streamingpreferences.h +++ b/app/settings/streamingpreferences.h @@ -134,7 +134,8 @@ class StreamingPreferences : public QObject Q_PROPERTY(bool swapFaceButtons MEMBER swapFaceButtons NOTIFY swapFaceButtonsChanged) Q_PROPERTY(bool keepAwake MEMBER keepAwake NOTIFY keepAwakeChanged) Q_PROPERTY(CaptureSysKeysMode captureSysKeysMode MEMBER captureSysKeysMode NOTIFY captureSysKeysModeChanged) - Q_PROPERTY(Language language MEMBER language NOTIFY languageChanged); + Q_PROPERTY(Language language MEMBER language NOTIFY languageChanged) + Q_PROPERTY(int minimumLatency MEMBER minimumLatency NOTIFY minimumLatencyChanged); Q_INVOKABLE bool retranslate(); @@ -172,6 +173,7 @@ class StreamingPreferences : public QObject UIDisplayMode uiDisplayMode; Language language; CaptureSysKeysMode captureSysKeysMode; + int minimumLatency; signals: void displayModeChanged(); @@ -204,6 +206,7 @@ class StreamingPreferences : public QObject void captureSysKeysModeChanged(); void keepAwakeChanged(); void languageChanged(); + void minimumLatencyChanged(); private: QString getSuffixFromLanguage(Language lang); diff --git a/app/streaming/session.cpp b/app/streaming/session.cpp index 46c43450f..06e77f42a 100644 --- a/app/streaming/session.cpp +++ b/app/streaming/session.cpp @@ -256,7 +256,8 @@ void Session::clSetControllerLED(uint16_t controllerNumber, uint8_t r, uint8_t g bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds, SDL_Window* window, int videoFormat, int width, int height, - int frameRate, bool enableVsync, bool enableFramePacing, bool testOnly, IVideoDecoder*& chosenDecoder) + int frameRate, bool enableVsync, bool enableFramePacing, bool testOnly, IVideoDecoder*& chosenDecoder, + int minimumLatency) { DECODER_PARAMETERS params; @@ -274,6 +275,7 @@ bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds, params.enableFramePacing = enableFramePacing; params.testOnly = testOnly; params.vds = vds; + params.minimumLatency = minimumLatency; SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "V-sync %s", @@ -376,7 +378,7 @@ void Session::getDecoderInfo(SDL_Window* window, // Try an HEVC Main10 decoder first to see if we have HDR support if (chooseDecoder(StreamingPreferences::VDS_FORCE_HARDWARE, window, VIDEO_FORMAT_H265_MAIN10, 1920, 1080, 60, - false, false, true, decoder)) { + false, false, true, decoder, 0)) { isHardwareAccelerated = decoder->isHardwareAccelerated(); isFullScreenOnly = decoder->isAlwaysFullScreen(); isHdrSupported = decoder->isHdrSupported(); @@ -389,7 +391,7 @@ void Session::getDecoderInfo(SDL_Window* window, // Try an AV1 Main10 decoder next to see if we have HDR support if (chooseDecoder(StreamingPreferences::VDS_FORCE_HARDWARE, window, VIDEO_FORMAT_AV1_MAIN10, 1920, 1080, 60, - false, false, true, decoder)) { + false, false, true, decoder, 0)) { // If we've got a working AV1 Main 10-bit decoder, we'll enable the HDR checkbox // but we will still continue probing to get other attributes for HEVC or H.264 // decoders. See the AV1 comment at the top of the function for more info. @@ -405,7 +407,7 @@ void Session::getDecoderInfo(SDL_Window* window, // Try a regular hardware accelerated HEVC decoder now if (chooseDecoder(StreamingPreferences::VDS_FORCE_HARDWARE, window, VIDEO_FORMAT_H265, 1920, 1080, 60, - false, false, true, decoder)) { + false, false, true, decoder, 0)) { isHardwareAccelerated = decoder->isHardwareAccelerated(); isFullScreenOnly = decoder->isAlwaysFullScreen(); maxResolution = decoder->getDecoderMaxResolution(); @@ -418,7 +420,7 @@ void Session::getDecoderInfo(SDL_Window* window, #if 0 // See AV1 comment at the top of this function if (chooseDecoder(StreamingPreferences::VDS_FORCE_HARDWARE, window, VIDEO_FORMAT_AV1_MAIN8, 1920, 1080, 60, - false, false, true, decoder)) { + false, false, true, decoder, 0)) { isHardwareAccelerated = decoder->isHardwareAccelerated(); isFullScreenOnly = decoder->isAlwaysFullScreen(); maxResolution = decoder->getDecoderMaxResolution(); @@ -432,7 +434,7 @@ void Session::getDecoderInfo(SDL_Window* window, // This will fall back to software decoding, so it should always work. if (chooseDecoder(StreamingPreferences::VDS_AUTO, window, VIDEO_FORMAT_H264, 1920, 1080, 60, - false, false, true, decoder)) { + false, false, true, decoder, 0)) { isHardwareAccelerated = decoder->isHardwareAccelerated(); isFullScreenOnly = decoder->isAlwaysFullScreen(); maxResolution = decoder->getDecoderMaxResolution(); @@ -451,7 +453,7 @@ bool Session::isHardwareDecodeAvailable(SDL_Window* window, { IVideoDecoder* decoder; - if (!chooseDecoder(vds, window, videoFormat, width, height, frameRate, false, false, true, decoder)) { + if (!chooseDecoder(vds, window, videoFormat, width, height, frameRate, false, false, true, decoder, 0)) { return false; } @@ -489,7 +491,7 @@ bool Session::populateDecoderProperties(SDL_Window* window) m_StreamConfig.width, m_StreamConfig.height, m_StreamConfig.fps, - false, false, true, decoder)) { + false, false, true, decoder, 0)) { return false; } @@ -1974,7 +1976,7 @@ void Session::execInternal() enableVsync, enableVsync && m_Preferences->framePacing, false, - s_ActiveSession->m_VideoDecoder)) { + s_ActiveSession->m_VideoDecoder, m_Preferences->minimumLatency)) { SDL_AtomicUnlock(&m_DecoderLock); SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Failed to recreate decoder after reset"); diff --git a/app/streaming/session.h b/app/streaming/session.h index 26c5102cb..0a7fa24dc 100644 --- a/app/streaming/session.h +++ b/app/streaming/session.h @@ -101,7 +101,7 @@ class Session : public QObject SDL_Window* window, int videoFormat, int width, int height, int frameRate, bool enableVsync, bool enableFramePacing, bool testOnly, - IVideoDecoder*& chosenDecoder); + IVideoDecoder*& chosenDecoder, int mimimumLatency); static void clStageStarting(int stage); diff --git a/app/streaming/video/decoder.h b/app/streaming/video/decoder.h index 85721b0a6..075ce55f6 100644 --- a/app/streaming/video/decoder.h +++ b/app/streaming/video/decoder.h @@ -43,6 +43,7 @@ typedef struct _DECODER_PARAMETERS { bool enableVsync; bool enableFramePacing; bool testOnly; + int minimumLatency; } DECODER_PARAMETERS, *PDECODER_PARAMETERS; class IVideoDecoder { diff --git a/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp b/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp index 2686c5495..b9c13e8cc 100644 --- a/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp +++ b/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp @@ -31,11 +31,14 @@ Pacer::Pacer(IFFmpegRenderer* renderer, PVIDEO_STATS videoStats) : m_RenderThread(nullptr), m_VsyncThread(nullptr), + m_LatencyThread(nullptr), m_Stopping(false), m_VsyncSource(nullptr), m_VsyncRenderer(renderer), m_MaxVideoFps(0), m_DisplayFps(0), + m_MinimumLatency(0), + m_MaxLatencyQueuedFrames(0), m_VideoStats(videoStats) { @@ -45,6 +48,12 @@ Pacer::~Pacer() { m_Stopping = true; + // Stop the latency thread + if (m_LatencyThread != nullptr) { + m_LatencyQueueNotEmpty.wakeAll(); + SDL_WaitThread(m_LatencyThread, nullptr); + } + // Stop the V-sync thread if (m_VsyncThread != nullptr) { m_PacingQueueNotEmpty.wakeAll(); @@ -173,9 +182,125 @@ int Pacer::renderThread(void* context) return 0; } + +int Pacer::latencyThread(void* context) +{ + Pacer* me = reinterpret_cast(context); + +#if SDL_VERSION_ATLEAST(2, 0, 9) + int threadPriorityResult = SDL_SetThreadPriority(SDL_THREAD_PRIORITY_TIME_CRITICAL); +#else + int threadPriorityResult = SDL_SetThreadPriority(SDL_THREAD_PRIORITY_HIGH); +#endif + if (threadPriorityResult < 0) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "Unable to increase latency thread to higher priority: %s", + SDL_GetError()); + } + + // Make sure initialize() has been called + SDL_assert(m_MaxVideoFps != 0); + + Uint32 startTime = SDL_GetTicks(); + double releasePeriod = 1000.0 / me->m_MaxVideoFps; + double scheduleShift = 0; + + int baseFrameBufferAllowance = 1 + me->m_MinimumLatency * me->m_MaxVideoFps / 1000; + + int iteration = 0; + while (!me->m_Stopping) { + iteration++; + + // Wait for the next frame release target + double nextReleaseTime = startTime + scheduleShift + releasePeriod * iteration; + double timeToNextRelease = nextReleaseTime - SDL_GetTicks(); + if (timeToNextRelease < -releasePeriod * 0.5) { + // We somehow fell far behind our release target (maybe due to thread scheduling or + // a super late frame). Just skip to the next iteration. + continue; + } + if (timeToNextRelease >= 1.0) { + SDL_Delay((Uint32) timeToNextRelease); + } + + if (me->m_Stopping) { + break; + } + + // Acquire the latency queue lock to protect the queue and + // the not empty condition + me->m_LatencyQueueLock.lock(); + + // If the queue length history entries are large, be strict + // about dropping excess frames. + int frameDropTarget = baseFrameBufferAllowance; + + // If we may get more frames per second than we can display, use + // frame history to drop frames only if consistently above the + // our frame buffer allowance. + for (int queueHistoryEntry : me->m_LatencyQueueHistory) { + if (queueHistoryEntry <= baseFrameBufferAllowance) { + // Be lenient as long as the queue length + // resolves before the end of frame history + frameDropTarget = baseFrameBufferAllowance + 2; + break; + } + } + + // Keep a rolling 500 ms window of latency queue history + if (me->m_LatencyQueueHistory.count() == me->m_DisplayFps / 2) { + me->m_LatencyQueueHistory.dequeue(); + } + + me->m_LatencyQueueHistory.enqueue(me->m_LatencyQueue.count()); + + // Drop excess frames if we're above our drop target + while (me->m_LatencyQueue.count() > frameDropTarget) { + AVFrame* frame = me->m_LatencyQueue.dequeue(); + + // Drop the lock while we call av_frame_free() + me->m_LatencyQueueLock.unlock(); + me->m_VideoStats->pacerDroppedFrames++; + av_frame_free(&frame); + me->m_LatencyQueueLock.lock(); + } + + // Wait for a frame. Ideally a frame is already ready, but that may not be the case + while (!me->m_Stopping && me->m_LatencyQueue.empty()) { + me->m_LatencyQueueNotEmpty.wait(&me->m_LatencyQueueLock); + } + + if (me->m_Stopping) { + me->m_LatencyQueueLock.unlock(); + break; + } + + AVFrame* frame = me->m_LatencyQueue.dequeue(); + me->m_LatencyQueueLock.unlock(); + + // Adjust our frame release schedule to target our minimum latency. + int frameLatency = (int) (SDL_GetTicks() - frame->pkt_dts); + scheduleShift += (me->m_MinimumLatency - frameLatency) * 0.1; + + // Enqueue the frame in the next queue + me->m_FrameQueueLock.lock(); + if (me->m_VsyncSource == nullptr) { + me->enqueueFrameForRenderingAndUnlock(frame); + } + else { + me->dropFrameForEnqueue(me->m_PacingQueue, MAX_QUEUED_FRAMES); + me->m_PacingQueue.enqueue(frame); + me->m_FrameQueueLock.unlock(); + me->m_PacingQueueNotEmpty.wakeOne(); + } + } + + return 0; +} + void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame) { - dropFrameForEnqueue(m_RenderQueue); + dropFrameForEnqueue(m_RenderQueue, MAX_QUEUED_FRAMES); m_RenderQueue.enqueue(frame); m_FrameQueueLock.unlock(); @@ -256,11 +381,12 @@ void Pacer::handleVsync(int timeUntilNextVsyncMillis) enqueueFrameForRenderingAndUnlock(m_PacingQueue.dequeue()); } -bool Pacer::initialize(SDL_Window* window, int maxVideoFps, bool enablePacing) +bool Pacer::initialize(SDL_Window* window, int maxVideoFps, bool enablePacing, int minimumLatency) { m_MaxVideoFps = maxVideoFps; m_DisplayFps = StreamUtils::getDisplayRefreshRate(window); m_RendererAttributes = m_VsyncRenderer->getRendererAttributes(); + m_MinimumLatency = minimumLatency; if (enablePacing) { SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, @@ -314,6 +440,11 @@ bool Pacer::initialize(SDL_Window* window, int maxVideoFps, bool enablePacing) m_DisplayFps, m_MaxVideoFps); } + if (m_MinimumLatency > 0) { + m_MaxLatencyQueuedFrames = 3 + m_MinimumLatency * m_MaxVideoFps / 1000; + m_LatencyThread = SDL_CreateThread(Pacer::latencyThread, "PacerLatency", this); + } + if (m_VsyncSource != nullptr) { m_VsyncThread = SDL_CreateThread(Pacer::vsyncThread, "PacerVsync", this); } @@ -387,10 +518,10 @@ void Pacer::renderFrame(AVFrame* frame) m_FrameQueueLock.unlock(); } -void Pacer::dropFrameForEnqueue(QQueue& queue) +void Pacer::dropFrameForEnqueue(QQueue& queue, int maxQueuedFrames) { - SDL_assert(queue.size() <= MAX_QUEUED_FRAMES); - if (queue.size() == MAX_QUEUED_FRAMES) { + SDL_assert(queue.size() <= maxQueuedFrames); + if (queue.size() == maxQueuedFrames) { AVFrame* frame = queue.dequeue(); av_frame_free(&frame); } @@ -402,14 +533,23 @@ void Pacer::submitFrame(AVFrame* frame) SDL_assert(m_MaxVideoFps != 0); // Queue the frame and possibly wake up the render thread - m_FrameQueueLock.lock(); - if (m_VsyncSource != nullptr) { - dropFrameForEnqueue(m_PacingQueue); - m_PacingQueue.enqueue(frame); - m_FrameQueueLock.unlock(); - m_PacingQueueNotEmpty.wakeOne(); + if (m_LatencyThread != nullptr) { + m_LatencyQueueLock.lock(); + dropFrameForEnqueue(m_LatencyQueue, m_MaxLatencyQueuedFrames); + m_LatencyQueue.enqueue(frame); + m_LatencyQueueLock.unlock(); + m_LatencyQueueNotEmpty.wakeOne(); } else { - enqueueFrameForRenderingAndUnlock(frame); + m_FrameQueueLock.lock(); + if (m_VsyncSource != nullptr) { + dropFrameForEnqueue(m_PacingQueue, MAX_QUEUED_FRAMES); + m_PacingQueue.enqueue(frame); + m_FrameQueueLock.unlock(); + m_PacingQueueNotEmpty.wakeOne(); + } + else { + enqueueFrameForRenderingAndUnlock(frame); + } } } diff --git a/app/streaming/video/ffmpeg-renderers/pacer/pacer.h b/app/streaming/video/ffmpeg-renderers/pacer/pacer.h index 11963a5f5..9d1693b18 100644 --- a/app/streaming/video/ffmpeg-renderers/pacer/pacer.h +++ b/app/streaming/video/ffmpeg-renderers/pacer/pacer.h @@ -31,7 +31,7 @@ class Pacer void submitFrame(AVFrame* frame); - bool initialize(SDL_Window* window, int maxVideoFps, bool enablePacing); + bool initialize(SDL_Window* window, int maxVideoFps, bool enablePacing, int minimumLatency); void signalVsync(); @@ -42,30 +42,39 @@ class Pacer static int renderThread(void* context); + static int latencyThread(void* context); + void handleVsync(int timeUntilNextVsyncMillis); void enqueueFrameForRenderingAndUnlock(AVFrame* frame); void renderFrame(AVFrame* frame); - void dropFrameForEnqueue(QQueue& queue); + void dropFrameForEnqueue(QQueue& queue, int maxQueuedFrames); QQueue m_RenderQueue; QQueue m_PacingQueue; + QQueue m_LatencyQueue; QQueue m_PacingQueueHistory; QQueue m_RenderQueueHistory; + QQueue m_LatencyQueueHistory; QMutex m_FrameQueueLock; + QMutex m_LatencyQueueLock; QWaitCondition m_RenderQueueNotEmpty; QWaitCondition m_PacingQueueNotEmpty; + QWaitCondition m_LatencyQueueNotEmpty; QWaitCondition m_VsyncSignalled; SDL_Thread* m_RenderThread; SDL_Thread* m_VsyncThread; + SDL_Thread* m_LatencyThread; bool m_Stopping; IVsyncSource* m_VsyncSource; IFFmpegRenderer* m_VsyncRenderer; int m_MaxVideoFps; int m_DisplayFps; + int m_MinimumLatency; + int m_MaxLatencyQueuedFrames; PVIDEO_STATS m_VideoStats; int m_RendererAttributes; }; diff --git a/app/streaming/video/ffmpeg.cpp b/app/streaming/video/ffmpeg.cpp index e27c88bb6..333953ce3 100644 --- a/app/streaming/video/ffmpeg.cpp +++ b/app/streaming/video/ffmpeg.cpp @@ -422,7 +422,8 @@ bool FFmpegVideoDecoder::completeInitialization(const AVCodec* decoder, enum AVP if (!testFrame) { m_Pacer = new Pacer(m_FrontendRenderer, &m_ActiveWndVideoStats); if (!m_Pacer->initialize(params->window, params->frameRate, - params->enableFramePacing || (params->enableVsync && (m_FrontendRenderer->getRendererAttributes() & RENDERER_ATTRIBUTE_FORCE_PACING)))) { + params->enableFramePacing || (params->enableVsync && (m_FrontendRenderer->getRendererAttributes() & RENDERER_ATTRIBUTE_FORCE_PACING)), + params->minimumLatency)) { return false; } }