Merge pull request #3324 from metalefty/v0.10-h264-openh264

[v0.10] add OpenH264 support

.github/workflows/build.yml

@@ -114,7 +114,7 @@ jobs:
       CONF_FLAGS_amd64_max: "--enable-ipv6 --enable-jpeg --enable-fuse --enable-mp3lame
                   --enable-fdkaac --enable-opus --enable-rfxcodec --enable-painter
                   --enable-pixman --with-imlib2 --with-freetype2 --enable-tests
-                  --enable-x264"
+                  --enable-x264 --enable-openh264"
       CONF_FLAGS_i386_max: "--enable-ipv6 --enable-jpeg --enable-mp3lame
                   --enable-opus --enable-rfxcodec --enable-painter
                   --disable-pixman --with-imlib2 --with-freetype2

configure.ac

@@ -171,6 +171,10 @@ AC_ARG_ENABLE(x264, AS_HELP_STRING([--enable-x264],
               [Use x264 library (default: no)]),
               [], [enable_x264=no])
 AM_CONDITIONAL(XRDP_X264, [test x$enable_x264 = xyes])
+AC_ARG_ENABLE(openh264, AS_HELP_STRING([--enable-openh264],
+              [Use Cisco OpenH264 library (default: no)]),
+              [], [enable_openh264=no])
+AM_CONDITIONAL(XRDP_OPENH264, [test x$enable_openh264 = xyes])
 AC_ARG_ENABLE(painter, AS_HELP_STRING([--disable-painter],
               [Do not use included painter library (default: no)]),
               [], [enable_painter=yes])
@@ -472,6 +476,8 @@ AS_IF( [test "x$enable_pixman" = "xyes"] , [PKG_CHECK_MODULES(PIXMAN, pixman-1 >
 
 AS_IF( [test "x$enable_x264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_X264, x264 >= 0.3.0)] )
 
+AS_IF( [test "x$enable_openh264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_OPENH264, openh264 >= 2.0.0)] )
+
 # checking for TurboJPEG
 if test "x$enable_tjpeg" = "xyes"
 then
@@ -630,6 +636,7 @@ echo "  jpeg                    $enable_jpeg"
 echo "  turbo jpeg              $enable_tjpeg"
 echo "  rfxcodec                $enable_rfxcodec"
 echo "  x264                    $enable_x264"
+echo "  openh264                $enable_openh264"
 echo "  painter                 $enable_painter"
 echo "  pixman                  $enable_pixman"
 echo "  fuse                    $enable_fuse"

scripts/install_xrdp_build_dependencies_with_apt.sh

@@ -96,7 +96,8 @@ in
                     libimlib2-dev \
                     libopus-dev \
                     libpixman-1-dev \
-                    libx264-dev"
+                    libx264-dev
+                    libopenh264-dev"
                 ;;
             *)
                 echo "unsupported feature set: $FEATURE_SET"

tests/xrdp/Makefile.am

@@ -20,13 +20,18 @@ EXTRA_DIST = \
   test_not4_24bit.bmp \
   test1.jpg \
   test_alpha_blend.png \
-  gfx/gfx.toml\
-  gfx/gfx_codec_order_undefined.toml \
-  gfx/gfx_codec_h264_preferred.toml \
+  gfx/gfx.toml \
   gfx/gfx_codec_h264_only.toml \
-  gfx/gfx_codec_rfx_preferred.toml \
+  gfx/gfx_codec_h264_preferred.toml \
+  gfx/gfx_codec_order_undefined.toml \
+  gfx/gfx_codec_rfx_only.toml \
   gfx/gfx_codec_rfx_preferred_odd.toml \
-  gfx/gfx_codec_rfx_only.toml
+  gfx/gfx_codec_rfx_preferred.toml \
+  gfx/gfx_h264_encoder_invalid.toml \
+  gfx/gfx_h264_encoder_openh264.toml \
+  gfx/gfx_h264_encoder_undefined.toml \
+  gfx/gfx_h264_encoder_x264.toml \
+  gfx/gfx_missing_h264.toml
 
 TESTS = test_xrdp
 check_PROGRAMS = test_xrdp
@@ -83,3 +88,10 @@ test_xrdp_LDADD += \
     $(top_builddir)/xrdp/xrdp_encoder_x264.o \
     $(XRDP_X264_LIBS)
 endif
+
+if XRDP_OPENH264
+AM_CPPFLAGS += -DXRDP_X264 $(XRDP_OPENH264_CFLAGS)
+test_xrdp_LDADD += \
+    $(top_builddir)/xrdp/xrdp_encoder_openh264.o \
+    $(XRDP_OPENH264_LIBS)
+endif

tests/xrdp/gfx/gfx.toml

@@ -1,40 +1,86 @@
 [codec]
 order = [ "H.264", "RFX" ]
 
+# Specify a preferred H.264 encoder, "x264" or "OpenH264".
+# This parameter takes effect only when more than one encoder is
+# enabled at compile time. If only one encoder is enabled, the encoder
+# will be used regardless the value of this parameter.
+h264_encoder = "OpenH264"
+
+#
+# Configurations for x264
+#
 [x264.default]
+# NOTE: x264 specifies bitrate in unit of kbps.
 preset = "ultrafast"
 tune = "zerolatency"
 profile = "main"     # profile is forced to baseline if preset == ultrafast
 vbv_max_bitrate = 0
 vbv_buffer_size = 0
-fps_num = 24
+fps_num = 60
 fps_den = 1
 
 [x264.lan]
 # inherits default
 
 [x264.wan]
-vbv_max_bitrate = 15000
-vbv_buffer_size = 1500
+vbv_max_bitrate = 15_000
+vbv_buffer_size = 1_500
 
 [x264.broadband_high]
 preset = "superfast"
-vbv_max_bitrate = 8000
+vbv_max_bitrate = 8_000
 vbv_buffer_Size = 800
 
 [x264.satellite]
 preset = "superfast"
-vbv_max_bitrate = 5000
+vbv_max_bitrate = 5_000
 vbv_buffer_size = 500
 
 [x264.broadband_low]
 preset = "veryfast"
-tune = "zerolatency"
-vbv_max_bitrate = 1600
+vbv_max_bitrate = 1_600
 vbv_buffer_size = 66
 
 [x264.modem]
 preset = "fast"
-tune = "zerolatency"
-vbv_max_bitrate = 1200
+vbv_max_bitrate = 1_200
 vbv_buffer_size = 50
+
+#
+# Configurations for OpenH264
+#
+[OpenH264.default]
+# NOTE: OpenH264 specifies bitrate in unit of bps, not kbps.
+EnableFrameSkip = false
+TargetBitrate = 20_000_000
+MaxBitrate =    0          # unspecified
+MaxFrameRate = 60.0
+
+[OpenH264.lan]
+# inherits default
+
+[OpenH264.wan]
+TargetBitrate = 10_000_000
+MaxBitrate =    12_000_000
+
+[OpenH264.broadband_high]
+EnableFrameSkip = true
+TargetBitrate = 8_000_000
+MaxBitrate =   10_000_000
+
+[OpenH264.satellite]
+EnableFrameSkip = true
+TargetBitrate = 4_000_000
+MaxBitrate =    6_000_000
+
+[OpenH264.broadband_low]
+EnableFrameSkip = true
+TargetBitrate = 1_600_000
+MaxBitrate =    1_800_000
+
+[OpenH264.modem]
+EnableFrameSkip = true
+TargetBitrate =   600_000
+MaxBitrate =    1_200_000
+

tests/xrdp/gfx/gfx_h264_encoder_invalid.toml

@@ -0,0 +1,86 @@
+[codec]
+order = [ "H.264", "RFX" ]
+
+# Specify a preferred H.264 encoder, "x264" or "OpenH264".
+# This parameter takes effect only when more than one encoder is
+# enabled at compile time. If only one encoder is enabled, the encoder
+# will be used regardless the value of this parameter.
+h264_encoder = "FreeH264"
+
+#
+# Configurations for x264
+#
+[x264.default]
+# NOTE: x264 specifies bitrate in unit of kbps.
+preset = "ultrafast"
+tune = "zerolatency"
+profile = "main"     # profile is forced to baseline if preset == ultrafast
+vbv_max_bitrate = 0
+vbv_buffer_size = 0
+fps_num = 60
+fps_den = 1
+
+[x264.lan]
+# inherits default
+
+[x264.wan]
+vbv_max_bitrate = 15_000
+vbv_buffer_size = 1_500
+
+[x264.broadband_high]
+preset = "superfast"
+vbv_max_bitrate = 8_000
+vbv_buffer_Size = 800
+
+[x264.satellite]
+preset = "superfast"
+vbv_max_bitrate = 5_000
+vbv_buffer_size = 500
+
+[x264.broadband_low]
+preset = "veryfast"
+vbv_max_bitrate = 1_600
+vbv_buffer_size = 66
+
+[x264.modem]
+preset = "fast"
+vbv_max_bitrate = 1_200
+vbv_buffer_size = 50
+
+#
+# Configurations for OpenH264
+#
+[OpenH264.default]
+# NOTE: OpenH264 specifies bitrate in unit of bps, not kbps.
+EnableFrameSkip = false
+TargetBitrate = 20_000_000
+MaxBitrate =    0          # unspecified
+MaxFrameRate = 60.0
+
+[OpenH264.lan]
+# inherits default
+
+[OpenH264.wan]
+TargetBitrate = 10_000_000
+MaxBitrate =    12_000_000
+
+[OpenH264.broadband_high]
+EnableFrameSkip = true
+TargetBitrate = 8_000_000
+MaxBitrate =   10_000_000
+
+[OpenH264.satellite]
+EnableFrameSkip = true
+TargetBitrate = 4_000_000
+MaxBitrate =    6_000_000
+
+[OpenH264.broadband_low]
+EnableFrameSkip = true
+TargetBitrate = 1_600_000
+MaxBitrate =    1_800_000
+
+[OpenH264.modem]
+EnableFrameSkip = true
+TargetBitrate =   600_000
+MaxBitrate =    1_200_000
+

tests/xrdp/gfx/gfx_h264_encoder_openh264.toml

@@ -0,0 +1,86 @@
+[codec]
+order = [ "H.264", "RFX" ]
+
+# Specify a preferred H.264 encoder, "x264" or "OpenH264".
+# This parameter takes effect only when more than one encoder is
+# enabled at compile time. If only one encoder is enabled, the encoder
+# will be used regardless the value of this parameter.
+h264_encoder = "OpenH264"
+
+#
+# Configurations for x264
+#
+[x264.default]
+# NOTE: x264 specifies bitrate in unit of kbps.
+preset = "ultrafast"
+tune = "zerolatency"
+profile = "main"     # profile is forced to baseline if preset == ultrafast
+vbv_max_bitrate = 0
+vbv_buffer_size = 0
+fps_num = 60
+fps_den = 1
+
+[x264.lan]
+# inherits default
+
+[x264.wan]
+vbv_max_bitrate = 15_000
+vbv_buffer_size = 1_500
+
+[x264.broadband_high]
+preset = "superfast"
+vbv_max_bitrate = 8_000
+vbv_buffer_Size = 800
+
+[x264.satellite]
+preset = "superfast"
+vbv_max_bitrate = 5_000
+vbv_buffer_size = 500
+
+[x264.broadband_low]
+preset = "veryfast"
+vbv_max_bitrate = 1_600
+vbv_buffer_size = 66
+
+[x264.modem]
+preset = "fast"
+vbv_max_bitrate = 1_200
+vbv_buffer_size = 50
+
+#
+# Configurations for OpenH264
+#
+[OpenH264.default]
+# NOTE: OpenH264 specifies bitrate in unit of bps, not kbps.
+EnableFrameSkip = false
+TargetBitrate = 20_000_000
+MaxBitrate =    0          # unspecified
+MaxFrameRate = 60.0
+
+[OpenH264.lan]
+# inherits default
+
+[OpenH264.wan]
+TargetBitrate = 10_000_000
+MaxBitrate =    12_000_000
+
+[OpenH264.broadband_high]
+EnableFrameSkip = true
+TargetBitrate = 8_000_000
+MaxBitrate =   10_000_000
+
+[OpenH264.satellite]
+EnableFrameSkip = true
+TargetBitrate = 4_000_000
+MaxBitrate =    6_000_000
+
+[OpenH264.broadband_low]
+EnableFrameSkip = true
+TargetBitrate = 1_600_000
+MaxBitrate =    1_800_000
+
+[OpenH264.modem]
+EnableFrameSkip = true
+TargetBitrate =   600_000
+MaxBitrate =    1_200_000
+