Add preferred H.264 encoder configuration in gfx.toml

metalefty · Nov 28, 2024 · 93ad14f · 93ad14f
1 parent ddab8d3
commit 93ad14f
Show file tree

Hide file tree

Showing 5 changed files with 85 additions and 38 deletions.
diff --git a/xrdp/gfx.toml b/xrdp/gfx.toml
@@ -1,6 +1,12 @@
 [codec]
 order = [ "H.264", "RFX" ]
 
+# Specify a preferred H.264 encoder, "x264" or "OpenH264".
+# This parameter takes effect only when more than one encoder is
+# enabled at compile time. If only one encoder is enabled, the encoder
+# will be used regardless the value of this parameter.
+h264_encoder = "OpenH264"
+
 [x264.default]
 preset = "ultrafast"
 tune = "zerolatency"
@@ -38,3 +44,9 @@ preset = "fast"
 tune = "zerolatency"
 vbv_max_bitrate = 1200
 vbv_buffer_size = 50
+
+[OpenH264.default]
+TargetBitrate = 0
+MaxBitrate = 0
+
+
diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c
@@ -314,44 +314,31 @@ xrdp_encoder_create(struct xrdp_mm *mm)
     self->frames_in_flight = MAX(self->frames_in_flight, 1);
 
 #if defined(XRDP_X264) && defined(XRDP_OPENH264)
-    if (self->h264_flags == 0)
-    {
-        char *env = g_getenv("XRDP_PREFER_OPENH264");
-        if (env != NULL)
-        {
-            LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: found env var "
-                "XRDP_PREFER_OPENH264=%s", env);
-            if (g_text2bool(env))
-            {
-                ENC_SET_BIT(self->h264_flags, ENC_FLAGS_PREFER_OPENH264_BIT);
-            }
-        }
-        else
-        {
-            LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: "
-                "XRDP_PREFER_OPENH264 not found");
-        }
-    }
-    if (ENC_IS_BIT_SET(self->h264_flags, ENC_FLAGS_PREFER_OPENH264_BIT))
-    {
-        LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using openh264 for "
-            "software encoder");
-        self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create;
-        self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete;
-        self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode;
-    }
-    else
+    struct xrdp_tconfig_gfx gfxconfig;
+    tconfig_load_gfx(GFX_CONF, &gfxconfig);
+
+    switch (gfxconfig.h264_encoder)
     {
-        LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for "
-            "software encoder");
-        self->xrdp_encoder_h264_create = xrdp_encoder_x264_create;
-        self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete;
-        self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode;
+        case XTC_H264_OPENH264:
+            LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using OpenH264 for "
+                "software encoder");
+            self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create;
+            self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete;
+            self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode;
+            break;
+        case XTC_H264_X264:
+        default:
+            /* x264 is the default H.264 software encoder */
+            LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for "
+                "software encoder");
+            self->xrdp_encoder_h264_create = xrdp_encoder_x264_create;
+            self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete;
+            self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode;
+            break;
     }
 #elif defined(XRDP_OPENH264)
-    LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using openh264 for "
+    LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using OpenH264 for "
         "software encoder");
-    self->h264_flags |= 1;
     self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create;
     self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete;
     self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode;

diff --git a/xrdp/xrdp_encoder.h b/xrdp/xrdp_encoder.h
@@ -24,9 +24,6 @@ typedef int (*xrdp_encoder_h264_encode_proc)(
     char *cdata, int *cdata_bytes,
     int connection_type, int *flags_ptr);
 
-/* h264_flags */
-#define ENC_FLAGS_PREFER_OPENH264_BIT   0
-
 /* for codec mode operations */
 struct xrdp_encoder
 {
@@ -59,7 +56,6 @@ struct xrdp_encoder
     int quant_idx_y;
     int quant_idx_u;
     int quant_idx_v;
-    int h264_flags;
     int pad0;
     xrdp_encoder_h264_create_proc xrdp_encoder_h264_create;
     xrdp_encoder_h264_delete_proc xrdp_encoder_h264_delete;

diff --git a/xrdp/xrdp_tconfig.c b/xrdp/xrdp_tconfig.c
@@ -247,6 +247,49 @@ tconfig_load_gfx_x264_ct(toml_table_t *tfile, const int connection_type,
     return 0;
 }
 
+static int tconfig_load_gfx_h264_encoder(toml_table_t *tfile, struct xrdp_tconfig_gfx *config)
+{
+    TCLOG(LOG_LEVEL_TRACE, "[h264 encoder]");
+
+    toml_table_t *codec;
+    int valid_encoder_found = 0;
+
+    if ((codec = toml_table_in(tfile, "codec")) != NULL)
+    {
+        toml_datum_t h264_encoder = toml_string_in(codec, "h264_encoder");
+
+        if (h264_encoder.ok)
+        {
+            if (g_strcasecmp(h264_encoder.u.s, "x264") == 0)
+            {
+                TCLOG(LOG_LEVEL_DEBUG, "[h264 encoder] x264");
+                valid_encoder_found = 1;
+                config->h264_encoder = XTC_H264_X264;
+            }
+            if (g_strcasecmp(h264_encoder.u.s, "OpenH264") == 0)
+            {
+                TCLOG(LOG_LEVEL_DEBUG, "[h264 encoder] OpenH264");
+                valid_encoder_found = 1;
+                config->h264_encoder = XTC_H264_OPENH264;
+            }
+
+            free(h264_encoder.u.s);
+        }
+    }
+
+    if (valid_encoder_found == 0)
+    {
+        TCLOG(LOG_LEVEL_WARNING, "[h264 encoder] could not get valid H.264 encoder , "
+              "using default \"x264\"");
+
+        /* default to x264 */
+        config->h264_encoder = XTC_H264_X264;
+        return 1;
+    }
+
+    return 0;
+}
+
 static int tconfig_load_gfx_order(toml_table_t *tfile, struct xrdp_tconfig_gfx *config)
 {
     char buff[64];
@@ -393,6 +436,8 @@ tconfig_load_gfx(const char *filename, struct xrdp_tconfig_gfx *config)
 
     /* Load GFX codec order */
     tconfig_load_gfx_order(tfile, config);
+    /* Load H.264 encoder */
+    tconfig_load_gfx_h264_encoder(tfile, config);
 
     /* H.264 configuration */
     if (codec_enabled(&config->codec, XTC_H264))

diff --git a/xrdp/xrdp_tconfig.h b/xrdp/xrdp_tconfig.h
@@ -53,6 +53,12 @@ enum xrdp_tconfig_codecs
     XTC_RFX
 };
 
+enum xrdp_tconfig_h264_encoders
+{
+    XTC_H264_X264,
+    XTC_H264_OPENH264
+};
+
 struct xrdp_tconfig_gfx_codec_order
 {
     enum xrdp_tconfig_codecs codecs[2];
@@ -62,6 +68,7 @@ struct xrdp_tconfig_gfx_codec_order
 struct xrdp_tconfig_gfx
 {
     struct xrdp_tconfig_gfx_codec_order codec;
+    enum xrdp_tconfig_h264_encoders h264_encoder;
     /* store x264 parameters for each connection type */
     struct xrdp_tconfig_gfx_x264_param x264_param[NUM_CONNECTION_TYPES];
     struct xrdp_tconfig_gfx_openh264_param