Mainline merge shm

.gitignore

@@ -61,3 +61,4 @@ xrdp/xrdp.ini
 xrdp_configure_options.h
 xrdpapi/xrdp-xrdpapi-simple
 .vscode/*
+xorgxrdp_helper/xorgxrdp_helper

Makefile.am

@@ -70,7 +70,8 @@ SUBDIRS = \
  $(XRDPVRDIR) \
  $(ULALACADIR) \
  tests \
- tools
+ tools \
+ xorgxrdp_helper
 
 distclean-local:
  -rm -f xrdp_configure_options.h

common/Makefile.am

@@ -29,6 +29,10 @@ AM_CPPFLAGS = \
  -DXRDP_PID_PATH=\"${localstatedir}/run\" \
  -DXRDP_LOG_PATH=\"${localstatedir}/log\"
 
+if XRDP_NVENC
+AM_CPPFLAGS += -DXRDP_NVENC
+endif
+
 # -no-suppress is an automake-specific flag which is needed
 # to prevent us missing compiler errors in some circumstances
 # (see https://github.com/neutrinolabs/xrdp/pull/1843 )

common/os_calls.c

@@ -158,6 +158,50 @@ g_init(const char *app_name)
  WSADATA wsadata;
 
  WSAStartup(2, &wsadata);
+#endif
+ /* In order to get g_mbstowcs and g_wcstombs to work properly with
+ UTF-8 non-ASCII characters, LC_CTYPE cannot be "C" or blank.
+ To select UTF-8 encoding without specifying any countries/languages,
+ "C.UTF-8" is used but provided in few systems.
+
+ See also: https://sourceware.org/glibc/wiki/Proposals/C.UTF-8 */
+ char *lc_ctype;
+ lc_ctype = setlocale(LC_CTYPE, "C.UTF-8");
+ if (lc_ctype == NULL)
+ {
+ /* use en_US.UTF-8 instead if not available */
+ setlocale(LC_CTYPE, "en_US.UTF-8");
+ }
+
+#if defined(XRDP_NVENC)
+ if (g_strcmp(app_name, "xrdp") == 0)
+ {
+ /* call cuInit() to initalize the nvidia drivers */
+ /* TODO create an issue on nvidia forums to figure out why we need to
+ * do this */
+ if (g_fork() == 0)
+ {
+ typedef int (*cu_init_proc)(int flags);
+ cu_init_proc cu_init;
+ long lib;
+ char cuda_lib_name[] = "libcuda.so";
+ char cuda_func_name[] = "cuInit";
+
+ lib = g_load_library(cuda_lib_name);
+ if (lib != 0)
+ {
+ cu_init = (cu_init_proc)
+ g_get_proc_address(lib, cuda_func_name);
+ if (cu_init != NULL)
+ {
+ cu_init(0);
+ }
+ }
+ log_end();
+ g_deinit();
+ g_exit(0);
+ }
+ }
 #endif
 }
 

common/trans.h

@@ -57,6 +57,8 @@ enum xrdp_source
  XRDP_SOURCE_SESMAN,
  XRDP_SOURCE_CHANSRV,
  XRDP_SOURCE_MOD,
+ XORGXRDP_SOURCE_XORG,
+ XORGXRDP_SOURCE_XRDP,
 
  XRDP_SOURCE_MAX_COUNT
 };

common/xrdp_client_info.h

@@ -213,7 +213,6 @@ struct xrdp_client_info
  * data */
  unsigned int session_physical_width; /* in mm */
  unsigned int session_physical_height; /* in mm */
-
  int large_pointer_support_flags;
  int gfx;
 };
@@ -222,11 +221,16 @@ enum xrdp_encoder_flags
 {
  NONE = 0,
  ENCODE_COMPLETE = 1 << 0,
- GFX_PROGRESSIVE_RFX = 1 << 1
+ CONTAINS_DUAL_FRAME_AVC444 = 1 << 1,
+ CONTAINS_SINGLE_FRAME_AVC444_YUV420 = 1 << 2,
+ CONTAINS_SINGLE_FRAME_AVC444_CHROMA420 = 1 << 3,
+ CONTAINS_KEY_FRAME = 1 << 4,
+ KEY_FRAME_REQUESTED = 1 << 5,
+ GFX_H264 = 1 << 6,
+ GFX_PROGRESSIVE_RFX = 1 << 7
 };
 
 /* yyyymmdd of last incompatible change to xrdp_client_info */
-/* also used for changes to all the xrdp installed headers */
-#define CLIENT_INFO_CURRENT_VERSION 20230425
+#define CLIENT_INFO_CURRENT_VERSION 20230830
 
 #endif

configure.ac

@@ -1,7 +1,7 @@
 # Process this file with autoconf to produce a configure script
 
-AC_PREREQ([2.69])
-AC_INIT([xrdp], [0.9.80], [xrdp-devel@googlegroups.com])
+AC_PREREQ([2.71])
+AC_INIT([xrdp],[0.9.80],[xrdp-devel@googlegroups.com])
 AC_CONFIG_HEADERS(config_ac.h:config_ac-h.in)
 AM_INIT_AUTOMAKE([1.7.2 foreign])
 AC_CONFIG_MACRO_DIR([m4])
@@ -167,6 +167,25 @@ AC_ARG_ENABLE(pixman, AS_HELP_STRING([--enable-pixman],
  [], [enable_pixman=no])
 AM_CONDITIONAL(XRDP_PIXMAN, [test x$enable_pixman = xyes])
 
+#example XRDP_NVENC_CFLAGS="-I/usr/local/cuda-11.3/include -I$HOME/Video_Codec_SDK_11.0.10/Interface" XRDP_NVENC_LIBS="-lcuda -lnvidia-encode" ./configure --enable-nvenc
+AC_ARG_ENABLE(nvenc, AS_HELP_STRING([--enable-nvenc],
+ [Use nvenc library (default: no), env vars XRDP_NVENC_CFLAGS and
+ XRDP_NVENC_LIBS should be set if used]),
+ [], [enable_nvenc=no])
+AM_CONDITIONAL(XRDP_NVENC, [test x$enable_nvenc = xyes])
+AC_ARG_ENABLE(yami, AS_HELP_STRING([--enable-yami],
+ [Use yami library (default: no)]),
+ [], [enable_yami=no])
+AM_CONDITIONAL(XRDP_YAMI, [test x$enable_yami = xyes])
+AC_ARG_ENABLE(x264, AS_HELP_STRING([--enable-x264],
+ [Use x264 library (default: no)]),
+ [], [enable_x264=no])
+AM_CONDITIONAL(XRDP_X264, [test x$enable_x264 = xyes])
+AC_ARG_ENABLE(avc444, AS_HELP_STRING([--enable-avc444],
+ [Enable AVC444 mode if X264 is enabled.]),
+ [], [enable_avc444=no])
+AM_CONDITIONAL(XRDP_AVC444, [test x$enable_avc444 = xyes])
+
 AC_ARG_ENABLE(painter, AS_HELP_STRING([--disable-painter],
  [Do not use included painter library (default: no)]),
  [], [enable_painter=yes])
@@ -466,6 +485,32 @@ fi
 
 AS_IF( [test "x$enable_pixman" = "xyes"] , [PKG_CHECK_MODULES(PIXMAN, pixman-1 >= 0.1.0)] )
 
+AS_IF( [test "x$enable_x264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_X264, x264 >= 0.3.0)] )
+
+if test "x$enable_nvenc" = "xyes"
+then
+if test ! -z "$XRDP_NVENC_CFLAGS"
+then
+ AC_SUBST(XRDP_NVENC_CFLAGS, ["$XRDP_NVENC_CFLAGS"])
+fi
+if test ! -z "$XRDP_NVENC_LIBS"
+then
+ AC_SUBST(XRDP_NVENC_LIBS, ["$XRDP_NVENC_LIBS"])
+fi
+fi
+
+if test "x$enable_yami" = "xyes"
+then
+if test ! -z "$XRDP_YAMI_CFLAGS"
+then
+ AC_SUBST(XRDP_YAMI_CFLAGS, ["$XRDP_YAMI_CFLAGS"])
+fi
+if test ! -z "$XRDP_YAMI_LIBS"
+then
+ AC_SUBST(XRDP_YAMI_LIBS, ["$XRDP_YAMI_LIBS"])
+fi
+fi
+
 # checking for TurboJPEG
 if test "x$enable_tjpeg" = "xyes"
 then
@@ -609,6 +654,7 @@ AC_CONFIG_FILES([
  xup/Makefile
  third_party/Makefile
  third_party/tomlc99/Makefile
+ xorgxrdp_helper/Makefile
 ])
 
 AC_REQUIRE_AUX_FILE([tap-driver.sh])
@@ -623,6 +669,10 @@ echo " fdkaac $enable_fdkaac"
 echo " jpeg $enable_jpeg"
 echo " turbo jpeg $enable_tjpeg"
 echo " rfxcodec $enable_rfxcodec"
+echo " nvenc $enable_nvenc"
+echo " yami $enable_yami"
+echo " x264 $enable_x264"
+echo " avc444 $enable_avc444"
 echo " painter $enable_painter"
 echo " pixman $enable_pixman"
 echo " fuse $enable_fuse"

instfiles/xrdp.service.in

@@ -10,7 +10,9 @@ EnvironmentFile=-@sysconfdir@/sysconfig/xrdp
 EnvironmentFile=-@sysconfdir@/default/xrdp
 ExecStart=@sbindir@/xrdp $XRDP_OPTIONS --nodaemon
 SystemCallArchitectures=native
-SystemCallFilter=@basic-io @file-system @io-event @ipc @network-io @process @signal ioctl madvise sysinfo uname
+SystemCallFilter=@basic-io @file-system @io-event @ipc @network-io @process
+SystemCallFilter=@signal @system-service brk ioctl madvise sysinfo uname
+SystemCallErrorNumber=EPERM
 
 [Install]
 WantedBy=multi-user.target

libxrdp/libxrdp.c

@@ -2277,6 +2277,7 @@ libxrdp_process_monitor_ex_stream(struct stream *s,
 
  return 0;
 }
+
 /*****************************************************************************/
 int EXPORT_CC
 libxrdp_planar_compress(char *in_data, int width, int height,

libxrdp/libxrdpinc.h

@@ -314,6 +314,7 @@ libxrdp_planar_compress(char *in_data, int width, int height,
  struct stream *s, int bpp, int byte_limit,
  int start_line, struct stream *temp_s,
  int e, int flags);
+
 /**
  * Processes a stream that is based on either
  * 2.2.1.3.6 Client Monitor Data (TS_UD_CS_MONITOR) or 2.2.2.2 DISPLAYCONTROL_MONITOR_LAYOUT_PDU

libxrdp/xrdp_sec.c

@@ -24,8 +24,11 @@
 
 #include "libxrdp.h"
 #include "ms-rdpbcgr.h"
+#include "ms-rdpedisp.h"
 #include "log.h"
 #include "string_calls.h"
+#include <limits.h>
+
 
 /* some compilers need unsigned char to avoid warnings */
 static tui8 g_pad_54[40] =

scripts/install_xrdp_build_dependencies_with_apt.sh

@@ -78,8 +78,9 @@ in
  libssl-dev \
  libx11-dev \
  libxrandr-dev \
- libxfixes-dev"
-
+ libxfixes-dev \
+ libepoxy-dev \
+ libepoxy0"
  case "$FEATURE_SET"
  in
  min)
@@ -118,6 +119,7 @@ in
  $LIBFREETYPE_DEV:i386 \
  libgl1-mesa-dev:i386 \
  libglu1-mesa-dev:i386 \
+ libegl1-mesa-dev:i386 \
  libjpeg-dev:i386 \
  libimlib2-dev:i386 \
  libmp3lame-dev:i386 \
@@ -131,8 +133,9 @@ in
  libxrender-dev:i386 \
  libsubunit-dev:i386 \
  check:i386 \
- libcmocka-dev:i386"
-
+ libcmocka-dev:i386 \
+ libepoxy-dev:i386 \
+ libepoxy0:i386"
  dpkg --add-architecture i386
  dpkg --print-architecture
  dpkg --print-foreign-architectures

sesman/Makefile.am

@@ -41,8 +41,25 @@ xrdp_sesman_LDADD = \
 
 sesmansysconfdir=$(sysconfdir)/xrdp
 
+if XRDP_YAMI
+ldlibrarypath=LD_LIBRARY_PATH=/opt/yami/lib
+libvadriverspath=LIBVA_DRIVERS_PATH=/opt/yami/lib/dri
+else
+ldlibrarypath=
+libvadriverspath=
+endif
+
+if XRDP_NVENC
+xorgconfigfile=xrdp/xorg_nvidia.conf
+else
+xorgconfigfile=xrdp/xorg.conf
+endif
+
 SUBST_VARS = sed \
- -e 's|@sesmansysconfdir[@]|$(sesmansysconfdir)|g'
+ -e 's|@sesmansysconfdir[@]|$(sesmansysconfdir)|g' \
+ -e 's|@ldlibrarypath[@]|$(ldlibrarypath)|g' \
+ -e 's|@libvadriverspath[@]|$(libvadriverspath)|g' \
+ -e 's|@xorgconfigfile[@]|$(xorgconfigfile)|g'
 
 subst_verbose = $(subst_verbose_@AM_V@)
 subst_verbose_ = $(subst_verbose_@AM_DEFAULT_V@)

sesman/sesman.ini.in

@@ -143,7 +143,7 @@ EnableSyslog=true
 param=Xorg
 ; Leave the rest parameters as-is unless you understand what will happen.
 param=-config
-param=xrdp/xorg.conf
+param=@xorgconfigfile@
 param=-noreset
 param=-nolisten
 param=tcp
@@ -206,3 +206,8 @@ EnableSyslog=true
 
 [SessionVariables]
 PULSE_SCRIPT=@sesmansysconfdir@/pulse/default.pa
+XRDP_USE_HELPER=1
+XRDP_NVENC_RATE_CONTROL_MODE=NV_ENC_PARAMS_RC_CONSTQP
+XRDP_NVENC_QP=22
+@ldlibrarypath@
+@libvadriverspath@

tests/xrdp/Makefile.am

@@ -23,6 +23,16 @@ EXTRA_DIST = \
 TESTS = test_xrdp
 check_PROGRAMS = test_xrdp
 
+XRDP_EXTRA_LIBS =
+
+if XRDP_X264
+AM_CPPFLAGS += -DXRDP_X264
+AM_CPPFLAGS += $(XRDP_X264_CFLAGS)
+XRDP_EXTRA_LIBS += \
+ $(top_builddir)/xrdp/xrdp_encoder_x264.o \
+ $(XRDP_X264_LIBS)
+endif
+
 test_xrdp_SOURCES = \
  test_xrdp.h \
  test_xrdp_main.c \
@@ -63,5 +73,6 @@ test_xrdp_LDADD = \
  $(top_builddir)/xrdp/xrdp_main_utils.o \
  $(PIXMAN_LIBS) \
  $(IMLIB2_LIBS) \
+ $(XRDP_EXTRA_LIBS) \
  @CHECK_LIBS@ \
  @CMOCKA_LIBS@

xorgxrdp_helper/Makefile.am

@@ -0,0 +1,46 @@
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/common
+
+EXTRA_DIST = xorgxrdp_helper_shaders.c
+
+XRDP_EXTRA_LIBS =
+XRDP_EXTRA_SOURCES =
+
+if XRDP_NVENC
+AM_CPPFLAGS += -DXRDP_NVENC
+AM_CPPFLAGS += $(XRDP_NVENC_CFLAGS)
+XRDP_EXTRA_LIBS += $(XRDP_NVENC_LIBS)
+XRDP_EXTRA_SOURCES += xorgxrdp_helper_nvenc.c xorgxrdp_helper_nvenc.h \
+encoder_headers/nvEncodeAPI_12_1_14.h
+endif
+
+if XRDP_YAMI
+AM_CPPFLAGS += -DXRDP_YAMI
+AM_CPPFLAGS += $(XRDP_YAMI_CFLAGS)
+XRDP_EXTRA_LIBS += $(XRDP_YAMI_LIBS)
+XRDP_EXTRA_SOURCES += xorgxrdp_helper_yami.c xorgxrdp_helper_yami.h encoder_headers/yami_inf.h
+endif
+
+if XRDP_AVC444
+AM_CPPFLAGS += -DXRDP_AVC444
+endif
+
+bin_PROGRAMS = \
+ xorgxrdp_helper
+
+xorgxrdp_helper_SOURCES = \
+ xorgxrdp_helper.c \
+ xorgxrdp_helper.h \
+ xorgxrdp_helper_x11.c \
+ xorgxrdp_helper_x11.h \
+ xorgxrdp_helper_egl.c \
+ xorgxrdp_helper_egl.h \
+ xorgxrdp_helper_glx.c \
+ xorgxrdp_helper_glx.h \
+ $(XRDP_EXTRA_SOURCES)
+
+xorgxrdp_helper_LDADD = \
+ $(top_builddir)/common/libcommon.la \
+ $(XRDP_EXTRA_LIBS) \
+ -lX11 -lepoxy
+