Add Video Super Resolution for Windows (AMD, Intel and NVIDIA) and MacOS

.gitignore

@@ -2,6 +2,7 @@
 
 
 **/.vs/
+**/.vscode/
 build/
 config.tests/*/.qmake.stash
 config.tests/*/Makefile

.gitmodules

@@ -17,3 +17,6 @@
 	path = libs
 	url = https://github.com/cgutman/moonlight-qt-prebuilts.git
 	shallow = true
+[submodule "third-party/AMF"]
+	path = third-party/AMF
+	url = https://github.com/GPUOpen-LibrariesAndSDKs/AMF.git

app/app.pro

@@ -165,7 +165,7 @@ macx {
         CONFIG += discord-rpc
     }
 
-    LIBS += -lobjc -framework VideoToolbox -framework AVFoundation -framework CoreVideo -framework CoreGraphics -framework CoreMedia -framework AppKit -framework Metal -framework QuartzCore
+    LIBS += -lobjc -framework VideoToolbox -framework AVFoundation -framework CoreVideo -framework CoreGraphics -framework CoreMedia -framework AppKit -framework Metal -framework MetalFx -framework QuartzCore
 
     # For libsoundio
     LIBS += -framework CoreAudio -framework AudioUnit
@@ -211,6 +211,7 @@ SOURCES += \
     gui/sdlgamepadkeynavigation.cpp \
     streaming/video/overlaymanager.cpp \
     backend/systemproperties.cpp \
+    streaming/video/videoenhancement.cpp \
     wm.cpp
 
 HEADERS += \
@@ -219,6 +220,7 @@ HEADERS += \
     cli/pair.h \
     settings/compatfetcher.h \
     settings/mappingfetcher.h \
+    streaming/video/videoenhancement.h \
     utils.h \
     backend/computerseeker.h \
     backend/identitymanager.h \
@@ -402,6 +404,18 @@ win32:!winrt {
         streaming/video/ffmpeg-renderers/d3d11va.h \
         streaming/video/ffmpeg-renderers/pacer/dxvsyncsource.h
 }
+win32:!winrt {
+    message(AMF enabled for AMD Drivers)
+
+    SOURCES += \
+        ../third-party/AMF/amf/public/common/AMFFactory.cpp \
+        ../third-party/AMF/amf/public/common/AMFSTL.cpp \
+        ../third-party/AMF/amf/public/common/Thread.cpp \
+        ../third-party/AMF/amf/public/common/TraceAdapter.cpp \
+        ../third-party/AMF/amf/public/common/Windows/ThreadWindows.cpp
+
+    INCLUDEPATH += $$PWD/../third-party/AMF/amf
+}
 macx {
     message(VideoToolbox renderer selected)
 

app/backend/systemproperties.cpp

@@ -6,6 +6,7 @@
 
 #include "streaming/session.h"
 #include "streaming/streamutils.h"
+#include "streaming/video/videoenhancement.h"
 
 #ifdef Q_OS_WIN32
 #define WIN32_LEAN_AND_MEAN
@@ -248,3 +249,27 @@ void SystemProperties::refreshDisplaysInternal()
 
     SDL_QuitSubSystem(SDL_INIT_VIDEO);
 }
+
+/**
+ * \brief Inform if the GPU is capable of Video enhancement
+ *
+ * Check if either Video Super-Resolution or SDR-to-HDR features can be used by the GPU.
+ *
+ * \return bool Returns true if the GPU is capable
+ */
+bool SystemProperties::isVideoEnhancementCapable()
+{
+    VideoEnhancement* videoEnhancement = &VideoEnhancement::getInstance();
+    return videoEnhancement->isUIvisible() && (videoEnhancement->isVSRcapable() || videoEnhancement->isHDRcapable());
+}
+
+/**
+ * \brief Inform if the GPU's driver is at an experiemental state of Video enhancement implementation
+ *
+ * \return bool Returns true if it is experimental yet
+ */
+bool SystemProperties::isVideoEnhancementExperimental()
+{
+    VideoEnhancement* videoEnhancement = &VideoEnhancement::getInstance();
+    return videoEnhancement->isExperimental();
+}

app/backend/systemproperties.h

@@ -32,6 +32,8 @@ class SystemProperties : public QObject
     Q_INVOKABLE QRect getNativeResolution(int displayIndex);
     Q_INVOKABLE QRect getSafeAreaResolution(int displayIndex);
     Q_INVOKABLE int getRefreshRate(int displayIndex);
+    Q_INVOKABLE bool isVideoEnhancementCapable();
+    Q_INVOKABLE bool isVideoEnhancementExperimental();
 
 signals:
     void unmappedGamepadsChanged();

app/cli/commandlineparser.cpp

@@ -367,6 +367,7 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference
     parser.addToggleOption("game-optimization", "game optimizations");
     parser.addToggleOption("audio-on-host", "audio on host PC");
     parser.addToggleOption("frame-pacing", "frame pacing");
+    parser.addToggleOption("video-enhancement", "Enhance video with AI");
     parser.addToggleOption("mute-on-focus-loss", "mute audio when Moonlight window loses focus");
     parser.addToggleOption("background-gamepad", "background gamepad input");
     parser.addToggleOption("reverse-scroll-direction", "inverted scroll direction");
@@ -474,6 +475,9 @@ void StreamCommandLineParser::parse(const QStringList &args, StreamingPreference
     // Resolve --frame-pacing and --no-frame-pacing options
     preferences->framePacing = parser.getToggleOptionValue("frame-pacing", preferences->framePacing);
 
+    // Resolve --video-enhancement and --no-video-enhancement options
+    preferences->videoEnhancement = parser.getToggleOptionValue("video-enhancement", preferences->videoEnhancement);
+
     // Resolve --mute-on-focus-loss and --no-mute-on-focus-loss options
     preferences->muteOnFocusLoss = parser.getToggleOptionValue("mute-on-focus-loss", preferences->muteOnFocusLoss);
 

app/gui/SettingsView.qml

@@ -820,6 +820,44 @@ Flickable {
                     ToolTip.visible: hovered
                     ToolTip.text: qsTr("Frame pacing reduces micro-stutter by delaying frames that come in too early")
                 }
+
+                CheckBox {
+                    id: videoEnhancementCheck
+                    width: parent.width
+                    hoverEnabled: true
+                    text: qsTr("Video AI-Enhancement")
+                    font.pointSize:  12
+                    enabled: SystemProperties.isVideoEnhancementCapable()
+                    checked: {
+                        return SystemProperties.isVideoEnhancementCapable() && StreamingPreferences.videoEnhancement
+                    }
+                    property bool keepValue: checked;
+                    onCheckedChanged: {
+                        StreamingPreferences.videoEnhancement = checked
+                    }
+                    ToolTip.delay: 1000
+                    ToolTip.timeout: 5000
+                    ToolTip.visible: hovered
+                    ToolTip.text:
+                        qsTr("Enhance video quality by utilizing the GPU's AI-Enhancement capabilities.")
+                        + qsTr("\nThis feature effectively upscales, reduces compression artifacts and enhances the clarity of streamed content.")
+                        + qsTr("\nNote:")
+                        + qsTr("\n  - If available, ensure that appropriate settings (i.e. RTX Video enhancement) are enabled in your GPU driver configuration.")
+                        + qsTr("\n  - HDR rendering has divers issues depending on the GPU used, we are working on it but we advise to currently use Non-HDR.")
+                        + qsTr("\n  - Be advised that using this feature on laptops running on battery power may lead to significant battery drain.")
+
+                    Component.onCompleted: {
+                        if (!SystemProperties.isVideoEnhancementCapable()){
+                            // VSR or SDR->HDR feature could not be initialized by any GPU available
+                            text = qsTr("Video AI-Enhancement (Not supported by the GPU)")
+                            enabled = false;
+                            checked = false;
+                        } else if(SystemProperties.isVideoEnhancementExperimental()){
+                            // Indicate if the feature is available but not officially deployed by the Vendor
+                            text = qsTr("Video AI-Enhancement (Experimental)")
+                        }
+                    }
+                }
             }
         }
 
@@ -1523,6 +1561,16 @@ Flickable {
                             StreamingPreferences.videoDecoderSelection = decoderListModel.get(currentIndex).val
                         }
                     }
+                    onCurrentIndexChanged: {
+                        if(decoderListModel.get(currentIndex).val === StreamingPreferences.VDS_FORCE_SOFTWARE){
+                            videoEnhancementCheck.enabled = false;
+                            videoEnhancementCheck.keepValue = videoEnhancementCheck.checked;
+                            videoEnhancementCheck.checked = false;
+                        } else {
+                            videoEnhancementCheck.enabled = true;
+                            videoEnhancementCheck.checked = videoEnhancementCheck.keepValue;
+                        }
+                    }
                 }
 
                 Label {

app/settings/streamingpreferences.cpp

@@ -33,6 +33,7 @@
 #define SER_ABSTOUCHMODE "abstouchmode"
 #define SER_STARTWINDOWED "startwindowed"
 #define SER_FRAMEPACING "framepacing"
+#define SER_VIDEOENHANCEMENT "videoenhancement"
 #define SER_CONNWARNINGS "connwarnings"
 #define SER_UIDISPLAYMODE "uidisplaymode"
 #define SER_RICHPRESENCE "richpresence"
@@ -131,6 +132,7 @@ void StreamingPreferences::reload()
     absoluteMouseMode = settings.value(SER_ABSMOUSEMODE, false).toBool();
     absoluteTouchMode = settings.value(SER_ABSTOUCHMODE, true).toBool();
     framePacing = settings.value(SER_FRAMEPACING, false).toBool();
+    videoEnhancement = settings.value(SER_VIDEOENHANCEMENT, false).toBool();
     connectionWarnings = settings.value(SER_CONNWARNINGS, true).toBool();
     richPresence = settings.value(SER_RICHPRESENCE, true).toBool();
     gamepadMouse = settings.value(SER_GAMEPADMOUSE, true).toBool();
@@ -321,6 +323,7 @@ void StreamingPreferences::save()
     settings.setValue(SER_ABSMOUSEMODE, absoluteMouseMode);
     settings.setValue(SER_ABSTOUCHMODE, absoluteTouchMode);
     settings.setValue(SER_FRAMEPACING, framePacing);
+    settings.setValue(SER_VIDEOENHANCEMENT, videoEnhancement);
     settings.setValue(SER_CONNWARNINGS, connectionWarnings);
     settings.setValue(SER_RICHPRESENCE, richPresence);
     settings.setValue(SER_GAMEPADMOUSE, gamepadMouse);

app/settings/streamingpreferences.h

@@ -119,6 +119,7 @@ class StreamingPreferences : public QObject
     Q_PROPERTY(bool absoluteMouseMode MEMBER absoluteMouseMode NOTIFY absoluteMouseModeChanged)
     Q_PROPERTY(bool absoluteTouchMode MEMBER absoluteTouchMode NOTIFY absoluteTouchModeChanged)
     Q_PROPERTY(bool framePacing MEMBER framePacing NOTIFY framePacingChanged)
+    Q_PROPERTY(bool videoEnhancement MEMBER videoEnhancement NOTIFY videoEnhancementChanged)
     Q_PROPERTY(bool connectionWarnings MEMBER connectionWarnings NOTIFY connectionWarningsChanged)
     Q_PROPERTY(bool richPresence MEMBER richPresence NOTIFY richPresenceChanged)
     Q_PROPERTY(bool gamepadMouse MEMBER gamepadMouse NOTIFY gamepadMouseChanged)
@@ -158,6 +159,7 @@ class StreamingPreferences : public QObject
     bool absoluteMouseMode;
     bool absoluteTouchMode;
     bool framePacing;
+    bool videoEnhancement;
     bool connectionWarnings;
     bool richPresence;
     bool gamepadMouse;
@@ -202,6 +204,7 @@ class StreamingPreferences : public QObject
     void uiDisplayModeChanged();
     void windowModeChanged();
     void framePacingChanged();
+    void videoEnhancementChanged();
     void connectionWarningsChanged();
     void richPresenceChanged();
     void gamepadMouseChanged();

app/streaming/session.cpp

@@ -261,7 +261,7 @@ void Session::clSetControllerLED(uint16_t controllerNumber, uint8_t r, uint8_t g
 
 bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
                             SDL_Window* window, int videoFormat, int width, int height,
-                            int frameRate, bool enableVsync, bool enableFramePacing, bool testOnly, IVideoDecoder*& chosenDecoder)
+                            int frameRate, bool enableVsync, bool enableFramePacing, bool enableVideoEnhancement, bool testOnly, IVideoDecoder*& chosenDecoder)
 {
     DECODER_PARAMETERS params;
 
@@ -277,6 +277,7 @@ bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
     params.window = window;
     params.enableVsync = enableVsync;
     params.enableFramePacing = enableFramePacing;
+    params.enableVideoEnhancement = enableVideoEnhancement;
     params.testOnly = testOnly;
     params.vds = vds;
 
@@ -381,7 +382,7 @@ void Session::getDecoderInfo(SDL_Window* window,
     // Try an HEVC Main10 decoder first to see if we have HDR support
     if (chooseDecoder(StreamingPreferences::VDS_FORCE_HARDWARE,
                       window, VIDEO_FORMAT_H265_MAIN10, 1920, 1080, 60,
-                      false, false, true, decoder)) {
+                      false, false, false, true, decoder)) {
         isHardwareAccelerated = decoder->isHardwareAccelerated();
         isFullScreenOnly = decoder->isAlwaysFullScreen();
         isHdrSupported = decoder->isHdrSupported();
@@ -394,7 +395,7 @@ void Session::getDecoderInfo(SDL_Window* window,
     // Try an AV1 Main10 decoder next to see if we have HDR support
     if (chooseDecoder(StreamingPreferences::VDS_FORCE_HARDWARE,
                       window, VIDEO_FORMAT_AV1_MAIN10, 1920, 1080, 60,
-                      false, false, true, decoder)) {
+                      false, false, false, true, decoder)) {
         // If we've got a working AV1 Main 10-bit decoder, we'll enable the HDR checkbox
         // but we will still continue probing to get other attributes for HEVC or H.264
         // decoders. See the AV1 comment at the top of the function for more info.
@@ -406,10 +407,10 @@ void Session::getDecoderInfo(SDL_Window* window,
         // that supports HDR rendering with software decoded frames.
         if (chooseDecoder(StreamingPreferences::VDS_FORCE_SOFTWARE,
                           window, VIDEO_FORMAT_H265_MAIN10, 1920, 1080, 60,
-                          false, false, true, decoder) ||
+                          false, false, false, true, decoder) ||
             chooseDecoder(StreamingPreferences::VDS_FORCE_SOFTWARE,
                           window, VIDEO_FORMAT_AV1_MAIN10, 1920, 1080, 60,
-                          false, false, true, decoder)) {
+                          false, false, false, true, decoder)) {
             isHdrSupported = decoder->isHdrSupported();
             delete decoder;
         }
@@ -423,7 +424,7 @@ void Session::getDecoderInfo(SDL_Window* window,
     // Try a regular hardware accelerated HEVC decoder now
     if (chooseDecoder(StreamingPreferences::VDS_FORCE_HARDWARE,
                       window, VIDEO_FORMAT_H265, 1920, 1080, 60,
-                      false, false, true, decoder)) {
+                      false, false, false, true, decoder)) {
         isHardwareAccelerated = decoder->isHardwareAccelerated();
         isFullScreenOnly = decoder->isAlwaysFullScreen();
         maxResolution = decoder->getDecoderMaxResolution();
@@ -450,7 +451,7 @@ void Session::getDecoderInfo(SDL_Window* window,
     // This will fall back to software decoding, so it should always work.
     if (chooseDecoder(StreamingPreferences::VDS_AUTO,
                       window, VIDEO_FORMAT_H264, 1920, 1080, 60,
-                      false, false, true, decoder)) {
+                      false, false, false, true, decoder)) {
         isHardwareAccelerated = decoder->isHardwareAccelerated();
         isFullScreenOnly = decoder->isAlwaysFullScreen();
         maxResolution = decoder->getDecoderMaxResolution();
@@ -470,7 +471,7 @@ Session::getDecoderAvailability(SDL_Window* window,
 {
     IVideoDecoder* decoder;
 
-    if (!chooseDecoder(vds, window, videoFormat, width, height, frameRate, false, false, true, decoder)) {
+    if (!chooseDecoder(vds, window, videoFormat, width, height, frameRate, false, false, false, true, decoder)) {
         return DecoderAvailability::None;
     }
 
@@ -491,7 +492,7 @@ bool Session::populateDecoderProperties(SDL_Window* window)
                        m_StreamConfig.width,
                        m_StreamConfig.height,
                        m_StreamConfig.fps,
-                       false, false, true, decoder)) {
+                       false, false, false, true, decoder)) {
         return false;
     }
 
@@ -2219,6 +2220,7 @@ void Session::execInternal()
                                    m_ActiveVideoHeight, m_ActiveVideoFrameRate,
                                    enableVsync,
                                    enableVsync && m_Preferences->framePacing,
+                                   m_Preferences->videoEnhancement,
                                    false,
                                    s_ActiveSession->m_VideoDecoder)) {
                     SDL_AtomicUnlock(&m_DecoderLock);

app/streaming/session.h

@@ -187,7 +187,7 @@ class Session : public QObject
     bool chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
                        SDL_Window* window, int videoFormat, int width, int height,
                        int frameRate, bool enableVsync, bool enableFramePacing,
-                       bool testOnly,
+                       bool enableVideoEnhancement, bool testOnly,
                        IVideoDecoder*& chosenDecoder);
 
     static

app/streaming/video/decoder.h

@@ -42,6 +42,7 @@ typedef struct _DECODER_PARAMETERS {
     int frameRate;
     bool enableVsync;
     bool enableFramePacing;
+    bool enableVideoEnhancement;
     bool testOnly;
 } DECODER_PARAMETERS, *PDECODER_PARAMETERS;