From e42c591b37c15882e0ad1adcc6566aa1b7637d98 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Mon, 30 Jan 2023 11:48:34 -0500 Subject: [PATCH 01/39] Resolution switching without reconnecting. - Based on https://github.com/jsorg71/xrdp/tree/dynamic_monitor - Tested with xorgxrdp - Tested with vnc - Only works with single monitor. - Update documentation to clarify the difference between MSTSC and Microsoft Remote Desktop. - Compatible with NVENC and xorgxrdp_helper - Compatible with Nvidia. - Updates to include ms-rdpedisp.h header for the 2.2.2 specification of the protocol. - State machine for resizing. - Mechanisms to make sure the key frame is always sent on resize. - Adds new struct that shares the number of monitors with xrdp_client_info.h - Modification to the original resize setup that works with /gfx. - YAMI compat (For Intel hardware acceleration) - Updating to librfxcodec branch that is also a merge from mainline. - Made sure RFX progressive works. - Add AVC444 prototype. - Latest version of NVENC encoding. - Use BT709 Full Range. - Switch to 460 optimized shaders. - Optimized YUV420 shader. - Using posix shared memory. --- .gitignore | 1 + Makefile.am | 3 +- common/Makefile.am | 4 + common/os_calls.c | 44 + common/trans.h | 2 + common/xrdp_client_info.h | 11 +- configure.ac | 54 +- libxrdp/libxrdp.c | 1 + libxrdp/libxrdpinc.h | 1 + libxrdp/xrdp_sec.c | 3 + ...nstall_xrdp_build_dependencies_with_apt.sh | 11 +- sesman/Makefile.am | 19 +- sesman/sesman.ini.in | 7 +- tests/xrdp/Makefile.am | 11 + xorgxrdp_helper/Makefile.am | 42 + .../encoder_headers/nvEncodeAPI_12_1_14.h | 4444 +++++++++++++++++ xorgxrdp_helper/encoder_headers/yami_inf.h | 173 + xorgxrdp_helper/xorgxrdp_helper.c | 852 ++++ xorgxrdp_helper/xorgxrdp_helper.h | 45 + xorgxrdp_helper/xorgxrdp_helper_egl.c | 159 + xorgxrdp_helper/xorgxrdp_helper_egl.h | 33 + xorgxrdp_helper/xorgxrdp_helper_glx.c | 149 + xorgxrdp_helper/xorgxrdp_helper_glx.h | 33 + xorgxrdp_helper/xorgxrdp_helper_nvenc.c | 400 ++ xorgxrdp_helper/xorgxrdp_helper_nvenc.h | 33 + xorgxrdp_helper/xorgxrdp_helper_shaders.c | 410 ++ xorgxrdp_helper/xorgxrdp_helper_x11.c | 788 +++ xorgxrdp_helper/xorgxrdp_helper_x11.h | 46 + xorgxrdp_helper/xorgxrdp_helper_yami.c | 274 + xorgxrdp_helper/xorgxrdp_helper_yami.h | 33 + xrdp/Makefile.am | 13 +- xrdp/xrdp.h | 5 +- xrdp/xrdp_encoder.c | 418 +- xrdp/xrdp_encoder.h | 4 + xrdp/xrdp_encoder_x264.c | 202 + xrdp/xrdp_encoder_x264.h | 36 + xrdp/xrdp_mm.c | 704 ++- xrdp/xrdp_painter.c | 2 - xrdp/xrdp_types.h | 12 +- xrdp/xrdp_wm.c | 2 - xup/xup.c | 211 +- 41 files changed, 9445 insertions(+), 250 deletions(-) create mode 100644 xorgxrdp_helper/Makefile.am create mode 100644 xorgxrdp_helper/encoder_headers/nvEncodeAPI_12_1_14.h create mode 100644 xorgxrdp_helper/encoder_headers/yami_inf.h create mode 100644 xorgxrdp_helper/xorgxrdp_helper.c create mode 100644 xorgxrdp_helper/xorgxrdp_helper.h create mode 100644 xorgxrdp_helper/xorgxrdp_helper_egl.c create mode 100644 xorgxrdp_helper/xorgxrdp_helper_egl.h create mode 100644 xorgxrdp_helper/xorgxrdp_helper_glx.c create mode 100644 xorgxrdp_helper/xorgxrdp_helper_glx.h create mode 100644 xorgxrdp_helper/xorgxrdp_helper_nvenc.c create mode 100644 xorgxrdp_helper/xorgxrdp_helper_nvenc.h create mode 100644 xorgxrdp_helper/xorgxrdp_helper_shaders.c create mode 100644 xorgxrdp_helper/xorgxrdp_helper_x11.c create mode 100644 xorgxrdp_helper/xorgxrdp_helper_x11.h create mode 100644 xorgxrdp_helper/xorgxrdp_helper_yami.c create mode 100644 xorgxrdp_helper/xorgxrdp_helper_yami.h create mode 100644 xrdp/xrdp_encoder_x264.c create mode 100644 xrdp/xrdp_encoder_x264.h diff --git a/.gitignore b/.gitignore index dedde9c33c..01977f109e 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,4 @@ xrdp/xrdp.ini xrdp_configure_options.h xrdpapi/xrdp-xrdpapi-simple .vscode/* +xorgxrdp_helper/xorgxrdp_helper diff --git a/Makefile.am b/Makefile.am index 8a3bca7c1a..4cb278ee4f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -70,7 +70,8 @@ SUBDIRS = \ $(XRDPVRDIR) \ $(ULALACADIR) \ tests \ - tools + tools \ + xorgxrdp_helper distclean-local: -rm -f xrdp_configure_options.h diff --git a/common/Makefile.am b/common/Makefile.am index baed9f0d54..f3ab39c14f 100644 --- a/common/Makefile.am +++ b/common/Makefile.am @@ -29,6 +29,10 @@ AM_CPPFLAGS = \ -DXRDP_PID_PATH=\"${localstatedir}/run\" \ -DXRDP_LOG_PATH=\"${localstatedir}/log\" +if XRDP_NVENC +AM_CPPFLAGS += -DXRDP_NVENC +endif + # -no-suppress is an automake-specific flag which is needed # to prevent us missing compiler errors in some circumstances # (see https://github.com/neutrinolabs/xrdp/pull/1843 ) diff --git a/common/os_calls.c b/common/os_calls.c index 765ad47284..6a85d18b03 100644 --- a/common/os_calls.c +++ b/common/os_calls.c @@ -158,6 +158,50 @@ g_init(const char *app_name) WSADATA wsadata; WSAStartup(2, &wsadata); +#endif + /* In order to get g_mbstowcs and g_wcstombs to work properly with + UTF-8 non-ASCII characters, LC_CTYPE cannot be "C" or blank. + To select UTF-8 encoding without specifying any countries/languages, + "C.UTF-8" is used but provided in few systems. + + See also: https://sourceware.org/glibc/wiki/Proposals/C.UTF-8 */ + char *lc_ctype; + lc_ctype = setlocale(LC_CTYPE, "C.UTF-8"); + if (lc_ctype == NULL) + { + /* use en_US.UTF-8 instead if not available */ + setlocale(LC_CTYPE, "en_US.UTF-8"); + } + +#if defined(XRDP_NVENC) + if (g_strcmp(app_name, "xrdp") == 0) + { + /* call cuInit() to initalize the nvidia drivers */ + /* TODO create an issue on nvidia forums to figure out why we need to + * do this */ + if (g_fork() == 0) + { + typedef int (*cu_init_proc)(int flags); + cu_init_proc cu_init; + long lib; + char cuda_lib_name[] = "libcuda.so"; + char cuda_func_name[] = "cuInit"; + + lib = g_load_library(cuda_lib_name); + if (lib != 0) + { + cu_init = (cu_init_proc) + g_get_proc_address(lib, cuda_func_name); + if (cu_init != NULL) + { + cu_init(0); + } + } + log_end(); + g_deinit(); + g_exit(0); + } + } #endif } diff --git a/common/trans.h b/common/trans.h index 2837b236d1..cf26553479 100644 --- a/common/trans.h +++ b/common/trans.h @@ -57,6 +57,8 @@ enum xrdp_source XRDP_SOURCE_SESMAN, XRDP_SOURCE_CHANSRV, XRDP_SOURCE_MOD, + XORGXRDP_SOURCE_XORG, + XORGXRDP_SOURCE_XRDP, XRDP_SOURCE_MAX_COUNT }; diff --git a/common/xrdp_client_info.h b/common/xrdp_client_info.h index a588f16c56..5508025e9b 100644 --- a/common/xrdp_client_info.h +++ b/common/xrdp_client_info.h @@ -213,7 +213,6 @@ struct xrdp_client_info * data */ unsigned int session_physical_width; /* in mm */ unsigned int session_physical_height; /* in mm */ - int large_pointer_support_flags; int gfx; }; @@ -222,11 +221,15 @@ enum xrdp_encoder_flags { NONE = 0, ENCODE_COMPLETE = 1 << 0, - GFX_PROGRESSIVE_RFX = 1 << 1 + GFX_PROGRESSIVE_RFX = 1 << 1, + CONTAINS_DUAL_FRAME_AVC444 = 1 << 2, + CONTAINS_SINGLE_FRAME_AVC444_YUV420 = 1 << 3, + CONTAINS_SINGLE_FRAME_AVC444_CHROMA420 = 1 << 4, + CONTAINS_KEY_FRAME = 1 << 5, + KEY_FRAME_REQUESTED = 1 << 6 }; /* yyyymmdd of last incompatible change to xrdp_client_info */ -/* also used for changes to all the xrdp installed headers */ -#define CLIENT_INFO_CURRENT_VERSION 20230425 +#define CLIENT_INFO_CURRENT_VERSION 20230830 #endif diff --git a/configure.ac b/configure.ac index c6cb5ef7c8..a5f8ed2ab5 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ # Process this file with autoconf to produce a configure script -AC_PREREQ([2.69]) -AC_INIT([xrdp], [0.9.80], [xrdp-devel@googlegroups.com]) +AC_PREREQ([2.71]) +AC_INIT([xrdp],[0.9.80],[xrdp-devel@googlegroups.com]) AC_CONFIG_HEADERS(config_ac.h:config_ac-h.in) AM_INIT_AUTOMAKE([1.7.2 foreign]) AC_CONFIG_MACRO_DIR([m4]) @@ -167,6 +167,25 @@ AC_ARG_ENABLE(pixman, AS_HELP_STRING([--enable-pixman], [], [enable_pixman=no]) AM_CONDITIONAL(XRDP_PIXMAN, [test x$enable_pixman = xyes]) +#example XRDP_NVENC_CFLAGS="-I/usr/local/cuda-11.3/include -I$HOME/Video_Codec_SDK_11.0.10/Interface" XRDP_NVENC_LIBS="-lcuda -lnvidia-encode" ./configure --enable-nvenc +AC_ARG_ENABLE(nvenc, AS_HELP_STRING([--enable-nvenc], + [Use nvenc library (default: no), env vars XRDP_NVENC_CFLAGS and + XRDP_NVENC_LIBS should be set if used]), + [], [enable_nvenc=no]) +AM_CONDITIONAL(XRDP_NVENC, [test x$enable_nvenc = xyes]) +AC_ARG_ENABLE(yami, AS_HELP_STRING([--enable-yami], + [Use yami library (default: no)]), + [], [enable_yami=no]) +AM_CONDITIONAL(XRDP_YAMI, [test x$enable_yami = xyes]) +AC_ARG_ENABLE(x264, AS_HELP_STRING([--enable-x264], + [Use x264 library (default: no)]), + [], [enable_x264=no]) +AM_CONDITIONAL(XRDP_X264, [test x$enable_x264 = xyes]) +AC_ARG_ENABLE(avc444, AS_HELP_STRING([--enable-avc444], + [Enable AVC444 mode if X264 is enabled.]), + [], [enable_avc444=no]) +AM_CONDITIONAL(XRDP_AVC444, [test x$enable_avc444 = xyes]) + AC_ARG_ENABLE(painter, AS_HELP_STRING([--disable-painter], [Do not use included painter library (default: no)]), [], [enable_painter=yes]) @@ -466,6 +485,32 @@ fi AS_IF( [test "x$enable_pixman" = "xyes"] , [PKG_CHECK_MODULES(PIXMAN, pixman-1 >= 0.1.0)] ) +AS_IF( [test "x$enable_x264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_X264, x264 >= 0.3.0)] ) + +if test "x$enable_nvenc" = "xyes" +then +if test ! -z "$XRDP_NVENC_CFLAGS" +then + AC_SUBST(XRDP_NVENC_CFLAGS, ["$XRDP_NVENC_CFLAGS"]) +fi +if test ! -z "$XRDP_NVENC_LIBS" +then + AC_SUBST(XRDP_NVENC_LIBS, ["$XRDP_NVENC_LIBS"]) +fi +fi + +if test "x$enable_yami" = "xyes" +then +if test ! -z "$XRDP_YAMI_CFLAGS" +then + AC_SUBST(XRDP_YAMI_CFLAGS, ["$XRDP_YAMI_CFLAGS"]) +fi +if test ! -z "$XRDP_YAMI_LIBS" +then + AC_SUBST(XRDP_YAMI_LIBS, ["$XRDP_YAMI_LIBS"]) +fi +fi + # checking for TurboJPEG if test "x$enable_tjpeg" = "xyes" then @@ -609,6 +654,7 @@ AC_CONFIG_FILES([ xup/Makefile third_party/Makefile third_party/tomlc99/Makefile + xorgxrdp_helper/Makefile ]) AC_REQUIRE_AUX_FILE([tap-driver.sh]) @@ -623,6 +669,10 @@ echo " fdkaac $enable_fdkaac" echo " jpeg $enable_jpeg" echo " turbo jpeg $enable_tjpeg" echo " rfxcodec $enable_rfxcodec" +echo " nvenc $enable_nvenc" +echo " yami $enable_yami" +echo " x264 $enable_x264" +echo " avc444 $enable_avc444" echo " painter $enable_painter" echo " pixman $enable_pixman" echo " fuse $enable_fuse" diff --git a/libxrdp/libxrdp.c b/libxrdp/libxrdp.c index 358a441868..227f071d60 100644 --- a/libxrdp/libxrdp.c +++ b/libxrdp/libxrdp.c @@ -2277,6 +2277,7 @@ libxrdp_process_monitor_ex_stream(struct stream *s, return 0; } + /*****************************************************************************/ int EXPORT_CC libxrdp_planar_compress(char *in_data, int width, int height, diff --git a/libxrdp/libxrdpinc.h b/libxrdp/libxrdpinc.h index 5d1c8fff47..65aa764c32 100644 --- a/libxrdp/libxrdpinc.h +++ b/libxrdp/libxrdpinc.h @@ -314,6 +314,7 @@ libxrdp_planar_compress(char *in_data, int width, int height, struct stream *s, int bpp, int byte_limit, int start_line, struct stream *temp_s, int e, int flags); + /** * Processes a stream that is based on either * 2.2.1.3.6 Client Monitor Data (TS_UD_CS_MONITOR) or 2.2.2.2 DISPLAYCONTROL_MONITOR_LAYOUT_PDU diff --git a/libxrdp/xrdp_sec.c b/libxrdp/xrdp_sec.c index 6c2057aeb1..7490bc9817 100644 --- a/libxrdp/xrdp_sec.c +++ b/libxrdp/xrdp_sec.c @@ -24,8 +24,11 @@ #include "libxrdp.h" #include "ms-rdpbcgr.h" +#include "ms-rdpedisp.h" #include "log.h" #include "string_calls.h" +#include + /* some compilers need unsigned char to avoid warnings */ static tui8 g_pad_54[40] = diff --git a/scripts/install_xrdp_build_dependencies_with_apt.sh b/scripts/install_xrdp_build_dependencies_with_apt.sh index dec91a65f4..1f11385989 100755 --- a/scripts/install_xrdp_build_dependencies_with_apt.sh +++ b/scripts/install_xrdp_build_dependencies_with_apt.sh @@ -78,8 +78,9 @@ in libssl-dev \ libx11-dev \ libxrandr-dev \ - libxfixes-dev" - + libxfixes-dev \ + libepoxy-dev \ + libepoxy0" case "$FEATURE_SET" in min) @@ -118,6 +119,7 @@ in $LIBFREETYPE_DEV:i386 \ libgl1-mesa-dev:i386 \ libglu1-mesa-dev:i386 \ + libegl1-mesa-dev:i386 \ libjpeg-dev:i386 \ libimlib2-dev:i386 \ libmp3lame-dev:i386 \ @@ -131,8 +133,9 @@ in libxrender-dev:i386 \ libsubunit-dev:i386 \ check:i386 \ - libcmocka-dev:i386" - + libcmocka-dev:i386 \ + libepoxy-dev:i386 \ + libepoxy0:i386" dpkg --add-architecture i386 dpkg --print-architecture dpkg --print-foreign-architectures diff --git a/sesman/Makefile.am b/sesman/Makefile.am index 7b34470c58..56c1131ee3 100644 --- a/sesman/Makefile.am +++ b/sesman/Makefile.am @@ -41,8 +41,25 @@ xrdp_sesman_LDADD = \ sesmansysconfdir=$(sysconfdir)/xrdp +if XRDP_YAMI +ldlibrarypath=LD_LIBRARY_PATH=/opt/yami/lib +libvadriverspath=LIBVA_DRIVERS_PATH=/opt/yami/lib/dri +else +ldlibrarypath= +libvadriverspath= +endif + +if XRDP_NVENC +xorgconfigfile=xrdp/xorg_nvidia.conf +else +xorgconfigfile=xrdp/xorg.conf +endif + SUBST_VARS = sed \ - -e 's|@sesmansysconfdir[@]|$(sesmansysconfdir)|g' + -e 's|@sesmansysconfdir[@]|$(sesmansysconfdir)|g' \ + -e 's|@ldlibrarypath[@]|$(ldlibrarypath)|g' \ + -e 's|@libvadriverspath[@]|$(libvadriverspath)|g' \ + -e 's|@xorgconfigfile[@]|$(xorgconfigfile)|g' subst_verbose = $(subst_verbose_@AM_V@) subst_verbose_ = $(subst_verbose_@AM_DEFAULT_V@) diff --git a/sesman/sesman.ini.in b/sesman/sesman.ini.in index 685af28b7e..0d89bbc54e 100644 --- a/sesman/sesman.ini.in +++ b/sesman/sesman.ini.in @@ -143,7 +143,7 @@ EnableSyslog=true param=Xorg ; Leave the rest parameters as-is unless you understand what will happen. param=-config -param=xrdp/xorg.conf +param=@xorgconfigfile@ param=-noreset param=-nolisten param=tcp @@ -206,3 +206,8 @@ EnableSyslog=true [SessionVariables] PULSE_SCRIPT=@sesmansysconfdir@/pulse/default.pa +XRDP_USE_HELPER=1 +XRDP_NVENC_RATE_CONTROL_MODE=NV_ENC_PARAMS_RC_CONSTQP +XRDP_NVENC_QP=22 +@ldlibrarypath@ +@libvadriverspath@ diff --git a/tests/xrdp/Makefile.am b/tests/xrdp/Makefile.am index 014cf36db8..dc0692c3ed 100644 --- a/tests/xrdp/Makefile.am +++ b/tests/xrdp/Makefile.am @@ -23,6 +23,16 @@ EXTRA_DIST = \ TESTS = test_xrdp check_PROGRAMS = test_xrdp +XRDP_EXTRA_LIBS = + +if XRDP_X264 +AM_CPPFLAGS += -DXRDP_X264 +AM_CPPFLAGS += $(XRDP_X264_CFLAGS) +XRDP_EXTRA_LIBS += \ + $(top_builddir)/xrdp/xrdp_encoder_x264.o \ + $(XRDP_X264_LIBS) +endif + test_xrdp_SOURCES = \ test_xrdp.h \ test_xrdp_main.c \ @@ -63,5 +73,6 @@ test_xrdp_LDADD = \ $(top_builddir)/xrdp/xrdp_main_utils.o \ $(PIXMAN_LIBS) \ $(IMLIB2_LIBS) \ + $(XRDP_EXTRA_LIBS) \ @CHECK_LIBS@ \ @CMOCKA_LIBS@ diff --git a/xorgxrdp_helper/Makefile.am b/xorgxrdp_helper/Makefile.am new file mode 100644 index 0000000000..5f9aeb3216 --- /dev/null +++ b/xorgxrdp_helper/Makefile.am @@ -0,0 +1,42 @@ +AM_CPPFLAGS = \ + -I$(top_srcdir)/common + +EXTRA_DIST = xorgxrdp_helper_shaders.c + +XRDP_EXTRA_LIBS = +XRDP_EXTRA_SOURCES = + +if XRDP_NVENC +AM_CPPFLAGS += -DXRDP_NVENC +AM_CPPFLAGS += $(XRDP_NVENC_CFLAGS) +XRDP_EXTRA_LIBS += $(XRDP_NVENC_LIBS) +XRDP_EXTRA_SOURCES += xorgxrdp_helper_nvenc.c xorgxrdp_helper_nvenc.h \ +encoder_headers/nvEncodeAPI_12_1_14.h +endif + +if XRDP_YAMI +AM_CPPFLAGS += -DXRDP_YAMI +AM_CPPFLAGS += $(XRDP_YAMI_CFLAGS) +XRDP_EXTRA_LIBS += $(XRDP_YAMI_LIBS) +XRDP_EXTRA_SOURCES += xorgxrdp_helper_yami.c xorgxrdp_helper_yami.h encoder_headers/yami_inf.h +endif + +bin_PROGRAMS = \ + xorgxrdp_helper + +xorgxrdp_helper_SOURCES = \ + xorgxrdp_helper.c \ + xorgxrdp_helper.h \ + xorgxrdp_helper_x11.c \ + xorgxrdp_helper_x11.h \ + xorgxrdp_helper_egl.c \ + xorgxrdp_helper_egl.h \ + xorgxrdp_helper_glx.c \ + xorgxrdp_helper_glx.h \ + $(XRDP_EXTRA_SOURCES) + +xorgxrdp_helper_LDADD = \ + $(top_builddir)/common/libcommon.la \ + $(XRDP_EXTRA_LIBS) \ + -lX11 -lepoxy + diff --git a/xorgxrdp_helper/encoder_headers/nvEncodeAPI_12_1_14.h b/xorgxrdp_helper/encoder_headers/nvEncodeAPI_12_1_14.h new file mode 100644 index 0000000000..b4a158f4ea --- /dev/null +++ b/xorgxrdp_helper/encoder_headers/nvEncodeAPI_12_1_14.h @@ -0,0 +1,4444 @@ +/* + * This copyright notice applies to this header file only: + * + * Copyright (c) 2010-2023 NVIDIA Corporation + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the software, and to permit persons to whom the + * software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/** + * \file nvEncodeAPI.h + * NVIDIA GPUs - beginning with the Kepler generation - contain a hardware-based encoder + * (referred to as NVENC) which provides fully-accelerated hardware-based video encoding. + * NvEncodeAPI provides the interface for NVIDIA video encoder (NVENC). + * \date 2011-2022 + * This file contains the interface constants, structure definitions and function prototypes. + */ + +#ifndef _NV_ENCODEAPI_H_ +#define _NV_ENCODEAPI_H_ + +#include + +#ifdef _WIN32 +#include +#endif + +#ifdef _MSC_VER +#ifndef _STDINT +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef short int16_t; +typedef unsigned short uint16_t; +#endif +#else +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \addtogroup ENCODER_STRUCTURE NvEncodeAPI Data structures + * @{ + */ + +#ifdef _WIN32 +#define NVENCAPI __stdcall +typedef RECT NVENC_RECT; +#else +#define NVENCAPI +// ========================================================================================= +#ifndef GUID_DEFINED +#define GUID_DEFINED +/*! + * \struct GUID + * Abstracts the GUID structure for non-windows platforms. + */ +// ========================================================================================= +typedef struct _GUID +{ + uint32_t Data1; /**< [in]: Specifies the first 8 hexadecimal digits of the GUID. */ + uint16_t Data2; /**< [in]: Specifies the first group of 4 hexadecimal digits. */ + uint16_t Data3; /**< [in]: Specifies the second group of 4 hexadecimal digits. */ + uint8_t Data4[8]; /**< [in]: Array of 8 bytes. The first 2 bytes contain the third group of 4 hexadecimal digits. + The remaining 6 bytes contain the final 12 hexadecimal digits. */ +} GUID, *LPGUID; +#endif // GUID + +/** + * \struct _NVENC_RECT + * Defines a Rectangle. Used in ::NV_ENC_PREPROCESS_FRAME. + */ +typedef struct _NVENC_RECT +{ + uint32_t left; /**< [in]: X coordinate of the upper left corner of rectangular area to be specified. */ + uint32_t top; /**< [in]: Y coordinate of the upper left corner of the rectangular area to be specified. */ + uint32_t right; /**< [in]: X coordinate of the bottom right corner of the rectangular area to be specified. */ + uint32_t bottom; /**< [in]: Y coordinate of the bottom right corner of the rectangular area to be specified. */ +} NVENC_RECT; + +#endif // _WIN32 + +/** @} */ /* End of GUID and NVENC_RECT structure grouping*/ + +typedef void *NV_ENC_INPUT_PTR; /**< NVENCODE API input buffer */ +typedef void *NV_ENC_OUTPUT_PTR; /**< NVENCODE API output buffer*/ +typedef void *NV_ENC_REGISTERED_PTR; /**< A Resource that has been registered with NVENCODE API*/ +typedef void *NV_ENC_CUSTREAM_PTR; /**< Pointer to CUstream*/ + +#define NVENCAPI_MAJOR_VERSION 12 +#define NVENCAPI_MINOR_VERSION 1 + +#define NVENCAPI_VERSION (NVENCAPI_MAJOR_VERSION | (NVENCAPI_MINOR_VERSION << 24)) + +/** + * Macro to generate per-structure version for use with API. + */ +#define NVENCAPI_STRUCT_VERSION(ver) ((uint32_t)NVENCAPI_VERSION | ((ver)<<16) | (0x7 << 28)) + + +#define NVENC_INFINITE_GOPLENGTH 0xffffffff + +#define NV_MAX_SEQ_HDR_LEN (512) + +#ifdef __GNUC__ +#define NV_ENC_DEPRECATED __attribute__ ((deprecated("WILL BE REMOVED IN A FUTURE VIDEO CODEC SDK VERSION"))) +#elif defined(_MSC_VER) +#define NV_ENC_DEPRECATED __declspec(deprecated("WILL BE REMOVED IN A FUTURE VIDEO CODEC SDK VERSION")) +#endif + +// ========================================================================================= +// Encode Codec GUIDS supported by the NvEncodeAPI interface. +// ========================================================================================= + +// {6BC82762-4E63-4ca4-AA85-1E50F321F6BF} +static const GUID NV_ENC_CODEC_H264_GUID = +{ 0x6bc82762, 0x4e63, 0x4ca4, { 0xaa, 0x85, 0x1e, 0x50, 0xf3, 0x21, 0xf6, 0xbf } }; + +// {790CDC88-4522-4d7b-9425-BDA9975F7603} +static const GUID NV_ENC_CODEC_HEVC_GUID = +{ 0x790cdc88, 0x4522, 0x4d7b, { 0x94, 0x25, 0xbd, 0xa9, 0x97, 0x5f, 0x76, 0x3 } }; + +// {0A352289-0AA7-4759-862D-5D15CD16D254} +static const GUID NV_ENC_CODEC_AV1_GUID = +{ 0x0a352289, 0x0aa7, 0x4759, { 0x86, 0x2d, 0x5d, 0x15, 0xcd, 0x16, 0xd2, 0x54 } }; + + + +// ========================================================================================= +// * Encode Profile GUIDS supported by the NvEncodeAPI interface. +// ========================================================================================= + +// {BFD6F8E7-233C-4341-8B3E-4818523803F4} +static const GUID NV_ENC_CODEC_PROFILE_AUTOSELECT_GUID = +{ 0xbfd6f8e7, 0x233c, 0x4341, { 0x8b, 0x3e, 0x48, 0x18, 0x52, 0x38, 0x3, 0xf4 } }; + +// {0727BCAA-78C4-4c83-8C2F-EF3DFF267C6A} +static const GUID NV_ENC_H264_PROFILE_BASELINE_GUID = +{ 0x727bcaa, 0x78c4, 0x4c83, { 0x8c, 0x2f, 0xef, 0x3d, 0xff, 0x26, 0x7c, 0x6a } }; + +// {60B5C1D4-67FE-4790-94D5-C4726D7B6E6D} +static const GUID NV_ENC_H264_PROFILE_MAIN_GUID = +{ 0x60b5c1d4, 0x67fe, 0x4790, { 0x94, 0xd5, 0xc4, 0x72, 0x6d, 0x7b, 0x6e, 0x6d } }; + +// {E7CBC309-4F7A-4b89-AF2A-D537C92BE310} +static const GUID NV_ENC_H264_PROFILE_HIGH_GUID = +{ 0xe7cbc309, 0x4f7a, 0x4b89, { 0xaf, 0x2a, 0xd5, 0x37, 0xc9, 0x2b, 0xe3, 0x10 } }; + +// {7AC663CB-A598-4960-B844-339B261A7D52} +static const GUID NV_ENC_H264_PROFILE_HIGH_444_GUID = +{ 0x7ac663cb, 0xa598, 0x4960, { 0xb8, 0x44, 0x33, 0x9b, 0x26, 0x1a, 0x7d, 0x52 } }; + +// {40847BF5-33F7-4601-9084-E8FE3C1DB8B7} +static const GUID NV_ENC_H264_PROFILE_STEREO_GUID = +{ 0x40847bf5, 0x33f7, 0x4601, { 0x90, 0x84, 0xe8, 0xfe, 0x3c, 0x1d, 0xb8, 0xb7 } }; + +// {B405AFAC-F32B-417B-89C4-9ABEED3E5978} +static const GUID NV_ENC_H264_PROFILE_PROGRESSIVE_HIGH_GUID = +{ 0xb405afac, 0xf32b, 0x417b, { 0x89, 0xc4, 0x9a, 0xbe, 0xed, 0x3e, 0x59, 0x78 } }; + +// {AEC1BD87-E85B-48f2-84C3-98BCA6285072} +static const GUID NV_ENC_H264_PROFILE_CONSTRAINED_HIGH_GUID = +{ 0xaec1bd87, 0xe85b, 0x48f2, { 0x84, 0xc3, 0x98, 0xbc, 0xa6, 0x28, 0x50, 0x72 } }; + +// {B514C39A-B55B-40fa-878F-F1253B4DFDEC} +static const GUID NV_ENC_HEVC_PROFILE_MAIN_GUID = +{ 0xb514c39a, 0xb55b, 0x40fa, { 0x87, 0x8f, 0xf1, 0x25, 0x3b, 0x4d, 0xfd, 0xec } }; + +// {fa4d2b6c-3a5b-411a-8018-0a3f5e3c9be5} +static const GUID NV_ENC_HEVC_PROFILE_MAIN10_GUID = +{ 0xfa4d2b6c, 0x3a5b, 0x411a, { 0x80, 0x18, 0x0a, 0x3f, 0x5e, 0x3c, 0x9b, 0xe5 } }; + +// For HEVC Main 444 8 bit and HEVC Main 444 10 bit profiles only +// {51ec32b5-1b4c-453c-9cbd-b616bd621341} +static const GUID NV_ENC_HEVC_PROFILE_FREXT_GUID = +{ 0x51ec32b5, 0x1b4c, 0x453c, { 0x9c, 0xbd, 0xb6, 0x16, 0xbd, 0x62, 0x13, 0x41 } }; + +// {5f2a39f5-f14e-4f95-9a9e-b76d568fcf97} +static const GUID NV_ENC_AV1_PROFILE_MAIN_GUID = +{ 0x5f2a39f5, 0xf14e, 0x4f95, { 0x9a, 0x9e, 0xb7, 0x6d, 0x56, 0x8f, 0xcf, 0x97 } }; + +// ========================================================================================= +// * Preset GUIDS supported by the NvEncodeAPI interface. +// ========================================================================================= +// Performance degrades and quality improves as we move from P1 to P7. Presets P3 to P7 for H264 and Presets P2 to P7 for HEVC have B frames enabled by default +// for HIGH_QUALITY and LOSSLESS tuning info, and will not work with Weighted Prediction enabled. In case Weighted Prediction is required, disable B frames by +// setting frameIntervalP = 1 +// {FC0A8D3E-45F8-4CF8-80C7-298871590EBF} +static const GUID NV_ENC_PRESET_P1_GUID = +{ 0xfc0a8d3e, 0x45f8, 0x4cf8, { 0x80, 0xc7, 0x29, 0x88, 0x71, 0x59, 0xe, 0xbf } }; + +// {F581CFB8-88D6-4381-93F0-DF13F9C27DAB} +static const GUID NV_ENC_PRESET_P2_GUID = +{ 0xf581cfb8, 0x88d6, 0x4381, { 0x93, 0xf0, 0xdf, 0x13, 0xf9, 0xc2, 0x7d, 0xab } }; + +// {36850110-3A07-441F-94D5-3670631F91F6} +static const GUID NV_ENC_PRESET_P3_GUID = +{ 0x36850110, 0x3a07, 0x441f, { 0x94, 0xd5, 0x36, 0x70, 0x63, 0x1f, 0x91, 0xf6 } }; + +// {90A7B826-DF06-4862-B9D2-CD6D73A08681} +static const GUID NV_ENC_PRESET_P4_GUID = +{ 0x90a7b826, 0xdf06, 0x4862, { 0xb9, 0xd2, 0xcd, 0x6d, 0x73, 0xa0, 0x86, 0x81 } }; + +// {21C6E6B4-297A-4CBA-998F-B6CBDE72ADE3} +static const GUID NV_ENC_PRESET_P5_GUID = +{ 0x21c6e6b4, 0x297a, 0x4cba, { 0x99, 0x8f, 0xb6, 0xcb, 0xde, 0x72, 0xad, 0xe3 } }; + +// {8E75C279-6299-4AB6-8302-0B215A335CF5} +static const GUID NV_ENC_PRESET_P6_GUID = +{ 0x8e75c279, 0x6299, 0x4ab6, { 0x83, 0x2, 0xb, 0x21, 0x5a, 0x33, 0x5c, 0xf5 } }; + +// {84848C12-6F71-4C13-931B-53E283F57974} +static const GUID NV_ENC_PRESET_P7_GUID = +{ 0x84848c12, 0x6f71, 0x4c13, { 0x93, 0x1b, 0x53, 0xe2, 0x83, 0xf5, 0x79, 0x74 } }; + +/** + * \addtogroup ENCODER_STRUCTURE NvEncodeAPI Data structures + * @{ + */ + +/** + * Input frame encode modes + */ +typedef enum _NV_ENC_PARAMS_FRAME_FIELD_MODE +{ + NV_ENC_PARAMS_FRAME_FIELD_MODE_FRAME = 0x01, /**< Frame mode */ + NV_ENC_PARAMS_FRAME_FIELD_MODE_FIELD = 0x02, /**< Field mode */ + NV_ENC_PARAMS_FRAME_FIELD_MODE_MBAFF = 0x03 /**< MB adaptive frame/field */ +} NV_ENC_PARAMS_FRAME_FIELD_MODE; + +/** + * Rate Control Modes + */ +typedef enum _NV_ENC_PARAMS_RC_MODE +{ + NV_ENC_PARAMS_RC_CONSTQP = 0x0, /**< Constant QP mode */ + NV_ENC_PARAMS_RC_VBR = 0x1, /**< Variable bitrate mode */ + NV_ENC_PARAMS_RC_CBR = 0x2, /**< Constant bitrate mode */ +} NV_ENC_PARAMS_RC_MODE; + +/** + * Multi Pass encoding + */ +typedef enum _NV_ENC_MULTI_PASS +{ + NV_ENC_MULTI_PASS_DISABLED = 0x0, /**< Single Pass */ + NV_ENC_TWO_PASS_QUARTER_RESOLUTION = 0x1, /**< Two Pass encoding is enabled where first Pass is quarter resolution */ + NV_ENC_TWO_PASS_FULL_RESOLUTION = 0x2, /**< Two Pass encoding is enabled where first Pass is full resolution */ +} NV_ENC_MULTI_PASS; + +/** + * Restore Encoder state + */ +typedef enum _NV_ENC_STATE_RESTORE_TYPE +{ + NV_ENC_STATE_RESTORE_FULL = 0x01, /**< Restore full encoder state */ + NV_ENC_STATE_RESTORE_RATE_CONTROL = 0x02, /**< Restore only rate control state */ + NV_ENC_STATE_RESTORE_ENCODE = 0x03, /**< Restore full encoder state except for rate control state */ +} NV_ENC_STATE_RESTORE_TYPE; + +typedef enum _NV_ENC_OUTPUT_STATS_LEVEL +{ + NV_ENC_OUTPUT_STATS_NONE = 0, /** No output stats */ + NV_ENC_OUTPUT_STATS_BLOCK_LEVEL = 1, /** Output stats for every block. + Block represents a CTB for HEVC, macroblock for H.264, super block for AV1 */ + NV_ENC_OUTPUT_STATS_ROW_LEVEL = 2, /** Output stats for every row. + Row represents a CTB row for HEVC, macroblock row for H.264, super block row for AV1 */ +} NV_ENC_OUTPUT_STATS_LEVEL; + +/** + * Emphasis Levels + */ +typedef enum _NV_ENC_EMPHASIS_MAP_LEVEL +{ + NV_ENC_EMPHASIS_MAP_LEVEL_0 = 0x0, /**< Emphasis Map Level 0, for zero Delta QP value */ + NV_ENC_EMPHASIS_MAP_LEVEL_1 = 0x1, /**< Emphasis Map Level 1, for very low Delta QP value */ + NV_ENC_EMPHASIS_MAP_LEVEL_2 = 0x2, /**< Emphasis Map Level 2, for low Delta QP value */ + NV_ENC_EMPHASIS_MAP_LEVEL_3 = 0x3, /**< Emphasis Map Level 3, for medium Delta QP value */ + NV_ENC_EMPHASIS_MAP_LEVEL_4 = 0x4, /**< Emphasis Map Level 4, for high Delta QP value */ + NV_ENC_EMPHASIS_MAP_LEVEL_5 = 0x5 /**< Emphasis Map Level 5, for very high Delta QP value */ +} NV_ENC_EMPHASIS_MAP_LEVEL; + +/** + * QP MAP MODE + */ +typedef enum _NV_ENC_QP_MAP_MODE +{ + NV_ENC_QP_MAP_DISABLED = 0x0, /**< Value in NV_ENC_PIC_PARAMS::qpDeltaMap have no effect. */ + NV_ENC_QP_MAP_EMPHASIS = 0x1, /**< Value in NV_ENC_PIC_PARAMS::qpDeltaMap will be treated as Emphasis level. Currently this is only supported for H264 */ + NV_ENC_QP_MAP_DELTA = 0x2, /**< Value in NV_ENC_PIC_PARAMS::qpDeltaMap will be treated as QP delta map. */ + NV_ENC_QP_MAP = 0x3, /**< Currently This is not supported. Value in NV_ENC_PIC_PARAMS::qpDeltaMap will be treated as QP value. */ +} NV_ENC_QP_MAP_MODE; + +/** + * Input picture structure + */ +typedef enum _NV_ENC_PIC_STRUCT +{ + NV_ENC_PIC_STRUCT_FRAME = 0x01, /**< Progressive frame */ + NV_ENC_PIC_STRUCT_FIELD_TOP_BOTTOM = 0x02, /**< Field encoding top field first */ + NV_ENC_PIC_STRUCT_FIELD_BOTTOM_TOP = 0x03 /**< Field encoding bottom field first */ +} NV_ENC_PIC_STRUCT; + +/** + * Display picture structure + * Currently, this enum is only used for deciding the number of clock timestamp sets in Picture Timing SEI / Time Code SEI + * Otherwise, this has no impact on encoder behavior + */ +typedef enum _NV_ENC_DISPLAY_PIC_STRUCT +{ + NV_ENC_PIC_STRUCT_DISPLAY_FRAME = 0x00, /**< Field encoding top field first */ + NV_ENC_PIC_STRUCT_DISPLAY_FIELD_TOP_BOTTOM = 0x01, /**< Field encoding top field first */ + NV_ENC_PIC_STRUCT_DISPLAY_FIELD_BOTTOM_TOP = 0x02, /**< Field encoding bottom field first */ + NV_ENC_PIC_STRUCT_DISPLAY_FRAME_DOUBLING = 0x03, /**< Frame doubling */ + NV_ENC_PIC_STRUCT_DISPLAY_FRAME_TRIPLING = 0x04 /**< Field tripling */ +} NV_ENC_DISPLAY_PIC_STRUCT; + +/** + * Input picture type + */ +typedef enum _NV_ENC_PIC_TYPE +{ + NV_ENC_PIC_TYPE_P = 0x0, /**< Forward predicted */ + NV_ENC_PIC_TYPE_B = 0x01, /**< Bi-directionally predicted picture */ + NV_ENC_PIC_TYPE_I = 0x02, /**< Intra predicted picture */ + NV_ENC_PIC_TYPE_IDR = 0x03, /**< IDR picture */ + NV_ENC_PIC_TYPE_BI = 0x04, /**< Bi-directionally predicted with only Intra MBs */ + NV_ENC_PIC_TYPE_SKIPPED = 0x05, /**< Picture is skipped */ + NV_ENC_PIC_TYPE_INTRA_REFRESH = 0x06, /**< First picture in intra refresh cycle */ + NV_ENC_PIC_TYPE_NONREF_P = 0x07, /**< Non reference P picture */ + NV_ENC_PIC_TYPE_UNKNOWN = 0xFF /**< Picture type unknown */ +} NV_ENC_PIC_TYPE; + +/** + * Motion vector precisions + */ +typedef enum _NV_ENC_MV_PRECISION +{ + NV_ENC_MV_PRECISION_DEFAULT = 0x0, /**< Driver selects Quarter-Pel motion vector precision by default */ + NV_ENC_MV_PRECISION_FULL_PEL = 0x01, /**< Full-Pel motion vector precision */ + NV_ENC_MV_PRECISION_HALF_PEL = 0x02, /**< Half-Pel motion vector precision */ + NV_ENC_MV_PRECISION_QUARTER_PEL = 0x03 /**< Quarter-Pel motion vector precision */ +} NV_ENC_MV_PRECISION; + + +/** + * Input buffer formats + */ +typedef enum _NV_ENC_BUFFER_FORMAT +{ + NV_ENC_BUFFER_FORMAT_UNDEFINED = 0x00000000, /**< Undefined buffer format */ + + NV_ENC_BUFFER_FORMAT_NV12 = 0x00000001, /**< Semi-Planar YUV [Y plane followed by interleaved UV plane] */ + NV_ENC_BUFFER_FORMAT_YV12 = 0x00000010, /**< Planar YUV [Y plane followed by V and U planes] */ + NV_ENC_BUFFER_FORMAT_IYUV = 0x00000100, /**< Planar YUV [Y plane followed by U and V planes] */ + NV_ENC_BUFFER_FORMAT_YUV444 = 0x00001000, /**< Planar YUV [Y plane followed by U and V planes] */ + NV_ENC_BUFFER_FORMAT_YUV420_10BIT = 0x00010000, /**< 10 bit Semi-Planar YUV [Y plane followed by interleaved UV plane]. Each pixel of size 2 bytes. Most Significant 10 bits contain pixel data. */ + NV_ENC_BUFFER_FORMAT_YUV444_10BIT = 0x00100000, /**< 10 bit Planar YUV444 [Y plane followed by U and V planes]. Each pixel of size 2 bytes. Most Significant 10 bits contain pixel data. */ + NV_ENC_BUFFER_FORMAT_ARGB = 0x01000000, /**< 8 bit Packed A8R8G8B8. This is a word-ordered format + where a pixel is represented by a 32-bit word with B + in the lowest 8 bits, G in the next 8 bits, R in the + 8 bits after that and A in the highest 8 bits. */ + NV_ENC_BUFFER_FORMAT_ARGB10 = 0x02000000, /**< 10 bit Packed A2R10G10B10. This is a word-ordered format + where a pixel is represented by a 32-bit word with B + in the lowest 10 bits, G in the next 10 bits, R in the + 10 bits after that and A in the highest 2 bits. */ + NV_ENC_BUFFER_FORMAT_AYUV = 0x04000000, /**< 8 bit Packed A8Y8U8V8. This is a word-ordered format + where a pixel is represented by a 32-bit word with V + in the lowest 8 bits, U in the next 8 bits, Y in the + 8 bits after that and A in the highest 8 bits. */ + NV_ENC_BUFFER_FORMAT_ABGR = 0x10000000, /**< 8 bit Packed A8B8G8R8. This is a word-ordered format + where a pixel is represented by a 32-bit word with R + in the lowest 8 bits, G in the next 8 bits, B in the + 8 bits after that and A in the highest 8 bits. */ + NV_ENC_BUFFER_FORMAT_ABGR10 = 0x20000000, /**< 10 bit Packed A2B10G10R10. This is a word-ordered format + where a pixel is represented by a 32-bit word with R + in the lowest 10 bits, G in the next 10 bits, B in the + 10 bits after that and A in the highest 2 bits. */ + NV_ENC_BUFFER_FORMAT_U8 = 0x40000000, /**< Buffer format representing one-dimensional buffer. + This format should be used only when registering the + resource as output buffer, which will be used to write + the encoded bit stream or H.264 ME only mode output. */ +} NV_ENC_BUFFER_FORMAT; + +#define NV_ENC_BUFFER_FORMAT_NV12_PL NV_ENC_BUFFER_FORMAT_NV12 +#define NV_ENC_BUFFER_FORMAT_YV12_PL NV_ENC_BUFFER_FORMAT_YV12 +#define NV_ENC_BUFFER_FORMAT_IYUV_PL NV_ENC_BUFFER_FORMAT_IYUV +#define NV_ENC_BUFFER_FORMAT_YUV444_PL NV_ENC_BUFFER_FORMAT_YUV444 + +/** + * Encoding levels + */ +typedef enum _NV_ENC_LEVEL +{ + NV_ENC_LEVEL_AUTOSELECT = 0, + + NV_ENC_LEVEL_H264_1 = 10, + NV_ENC_LEVEL_H264_1b = 9, + NV_ENC_LEVEL_H264_11 = 11, + NV_ENC_LEVEL_H264_12 = 12, + NV_ENC_LEVEL_H264_13 = 13, + NV_ENC_LEVEL_H264_2 = 20, + NV_ENC_LEVEL_H264_21 = 21, + NV_ENC_LEVEL_H264_22 = 22, + NV_ENC_LEVEL_H264_3 = 30, + NV_ENC_LEVEL_H264_31 = 31, + NV_ENC_LEVEL_H264_32 = 32, + NV_ENC_LEVEL_H264_4 = 40, + NV_ENC_LEVEL_H264_41 = 41, + NV_ENC_LEVEL_H264_42 = 42, + NV_ENC_LEVEL_H264_5 = 50, + NV_ENC_LEVEL_H264_51 = 51, + NV_ENC_LEVEL_H264_52 = 52, + NV_ENC_LEVEL_H264_60 = 60, + NV_ENC_LEVEL_H264_61 = 61, + NV_ENC_LEVEL_H264_62 = 62, + + NV_ENC_LEVEL_HEVC_1 = 30, + NV_ENC_LEVEL_HEVC_2 = 60, + NV_ENC_LEVEL_HEVC_21 = 63, + NV_ENC_LEVEL_HEVC_3 = 90, + NV_ENC_LEVEL_HEVC_31 = 93, + NV_ENC_LEVEL_HEVC_4 = 120, + NV_ENC_LEVEL_HEVC_41 = 123, + NV_ENC_LEVEL_HEVC_5 = 150, + NV_ENC_LEVEL_HEVC_51 = 153, + NV_ENC_LEVEL_HEVC_52 = 156, + NV_ENC_LEVEL_HEVC_6 = 180, + NV_ENC_LEVEL_HEVC_61 = 183, + NV_ENC_LEVEL_HEVC_62 = 186, + + NV_ENC_TIER_HEVC_MAIN = 0, + NV_ENC_TIER_HEVC_HIGH = 1, + + NV_ENC_LEVEL_AV1_2 = 0, + NV_ENC_LEVEL_AV1_21 = 1, + NV_ENC_LEVEL_AV1_22 = 2, + NV_ENC_LEVEL_AV1_23 = 3, + NV_ENC_LEVEL_AV1_3 = 4, + NV_ENC_LEVEL_AV1_31 = 5, + NV_ENC_LEVEL_AV1_32 = 6, + NV_ENC_LEVEL_AV1_33 = 7, + NV_ENC_LEVEL_AV1_4 = 8, + NV_ENC_LEVEL_AV1_41 = 9, + NV_ENC_LEVEL_AV1_42 = 10, + NV_ENC_LEVEL_AV1_43 = 11, + NV_ENC_LEVEL_AV1_5 = 12, + NV_ENC_LEVEL_AV1_51 = 13, + NV_ENC_LEVEL_AV1_52 = 14, + NV_ENC_LEVEL_AV1_53 = 15, + NV_ENC_LEVEL_AV1_6 = 16, + NV_ENC_LEVEL_AV1_61 = 17, + NV_ENC_LEVEL_AV1_62 = 18, + NV_ENC_LEVEL_AV1_63 = 19, + NV_ENC_LEVEL_AV1_7 = 20, + NV_ENC_LEVEL_AV1_71 = 21, + NV_ENC_LEVEL_AV1_72 = 22, + NV_ENC_LEVEL_AV1_73 = 23, + NV_ENC_LEVEL_AV1_AUTOSELECT, + + NV_ENC_TIER_AV1_0 = 0, + NV_ENC_TIER_AV1_1 = 1 +} NV_ENC_LEVEL; + +/** + * Error Codes + */ +typedef enum _NVENCSTATUS +{ + /** + * This indicates that API call returned with no errors. + */ + NV_ENC_SUCCESS, + + /** + * This indicates that no encode capable devices were detected. + */ + NV_ENC_ERR_NO_ENCODE_DEVICE, + + /** + * This indicates that devices pass by the client is not supported. + */ + NV_ENC_ERR_UNSUPPORTED_DEVICE, + + /** + * This indicates that the encoder device supplied by the client is not + * valid. + */ + NV_ENC_ERR_INVALID_ENCODERDEVICE, + + /** + * This indicates that device passed to the API call is invalid. + */ + NV_ENC_ERR_INVALID_DEVICE, + + /** + * This indicates that device passed to the API call is no longer available and + * needs to be reinitialized. The clients need to destroy the current encoder + * session by freeing the allocated input output buffers and destroying the device + * and create a new encoding session. + */ + NV_ENC_ERR_DEVICE_NOT_EXIST, + + /** + * This indicates that one or more of the pointers passed to the API call + * is invalid. + */ + NV_ENC_ERR_INVALID_PTR, + + /** + * This indicates that completion event passed in ::NvEncEncodePicture() call + * is invalid. + */ + NV_ENC_ERR_INVALID_EVENT, + + /** + * This indicates that one or more of the parameter passed to the API call + * is invalid. + */ + NV_ENC_ERR_INVALID_PARAM, + + /** + * This indicates that an API call was made in wrong sequence/order. + */ + NV_ENC_ERR_INVALID_CALL, + + /** + * This indicates that the API call failed because it was unable to allocate + * enough memory to perform the requested operation. + */ + NV_ENC_ERR_OUT_OF_MEMORY, + + /** + * This indicates that the encoder has not been initialized with + * ::NvEncInitializeEncoder() or that initialization has failed. + * The client cannot allocate input or output buffers or do any encoding + * related operation before successfully initializing the encoder. + */ + NV_ENC_ERR_ENCODER_NOT_INITIALIZED, + + /** + * This indicates that an unsupported parameter was passed by the client. + */ + NV_ENC_ERR_UNSUPPORTED_PARAM, + + /** + * This indicates that the ::NvEncLockBitstream() failed to lock the output + * buffer. This happens when the client makes a non blocking lock call to + * access the output bitstream by passing NV_ENC_LOCK_BITSTREAM::doNotWait flag. + * This is not a fatal error and client should retry the same operation after + * few milliseconds. + */ + NV_ENC_ERR_LOCK_BUSY, + + /** + * This indicates that the size of the user buffer passed by the client is + * insufficient for the requested operation. + */ + NV_ENC_ERR_NOT_ENOUGH_BUFFER, + + /** + * This indicates that an invalid struct version was used by the client. + */ + NV_ENC_ERR_INVALID_VERSION, + + /** + * This indicates that ::NvEncMapInputResource() API failed to map the client + * provided input resource. + */ + NV_ENC_ERR_MAP_FAILED, + + /** + * This indicates encode driver requires more input buffers to produce an output + * bitstream. If this error is returned from ::NvEncEncodePicture() API, this + * is not a fatal error. If the client is encoding with B frames then, + * ::NvEncEncodePicture() API might be buffering the input frame for re-ordering. + * + * A client operating in synchronous mode cannot call ::NvEncLockBitstream() + * API on the output bitstream buffer if ::NvEncEncodePicture() returned the + * ::NV_ENC_ERR_NEED_MORE_INPUT error code. + * The client must continue providing input frames until encode driver returns + * ::NV_ENC_SUCCESS. After receiving ::NV_ENC_SUCCESS status the client can call + * ::NvEncLockBitstream() API on the output buffers in the same order in which + * it has called ::NvEncEncodePicture(). + */ + NV_ENC_ERR_NEED_MORE_INPUT, + + /** + * This indicates that the HW encoder is busy encoding and is unable to encode + * the input. The client should call ::NvEncEncodePicture() again after few + * milliseconds. + */ + NV_ENC_ERR_ENCODER_BUSY, + + /** + * This indicates that the completion event passed in ::NvEncEncodePicture() + * API has not been registered with encoder driver using ::NvEncRegisterAsyncEvent(). + */ + NV_ENC_ERR_EVENT_NOT_REGISTERD, + + /** + * This indicates that an unknown internal error has occurred. + */ + NV_ENC_ERR_GENERIC, + + /** + * This indicates that the client is attempting to use a feature + * that is not available for the license type for the current system. + */ + NV_ENC_ERR_INCOMPATIBLE_CLIENT_KEY, + + /** + * This indicates that the client is attempting to use a feature + * that is not implemented for the current version. + */ + NV_ENC_ERR_UNIMPLEMENTED, + + /** + * This indicates that the ::NvEncRegisterResource API failed to register the resource. + */ + NV_ENC_ERR_RESOURCE_REGISTER_FAILED, + + /** + * This indicates that the client is attempting to unregister a resource + * that has not been successfully registered. + */ + NV_ENC_ERR_RESOURCE_NOT_REGISTERED, + + /** + * This indicates that the client is attempting to unmap a resource + * that has not been successfully mapped. + */ + NV_ENC_ERR_RESOURCE_NOT_MAPPED, + + /** + * This indicates encode driver requires more output buffers to write an output + * bitstream. If this error is returned from ::NvEncRestoreEncoderState() API, this + * is not a fatal error. If the client is encoding with B frames then, + * ::NvEncRestoreEncoderState() API might be requiring the extra output buffer for accomodating overlay frame output in a separate buffer, for AV1 codec. + * In this case, client must call NvEncRestoreEncoderState() API again with NV_ENC_RESTORE_ENCODER_STATE_PARAMS::outputBitstream as input along with + * the parameters in the previous call. When operating in asynchronous mode of encoding, client must also specify NV_ENC_RESTORE_ENCODER_STATE_PARAMS::completionEvent. + */ + NV_ENC_ERR_NEED_MORE_OUTPUT, + +} NVENCSTATUS; + +/** + * Encode Picture encode flags. + */ +typedef enum _NV_ENC_PIC_FLAGS +{ + NV_ENC_PIC_FLAG_FORCEINTRA = 0x1, /**< Encode the current picture as an Intra picture */ + NV_ENC_PIC_FLAG_FORCEIDR = 0x2, /**< Encode the current picture as an IDR picture. + This flag is only valid when Picture type decision is taken by the Encoder + [_NV_ENC_INITIALIZE_PARAMS::enablePTD == 1]. */ + NV_ENC_PIC_FLAG_OUTPUT_SPSPPS = 0x4, /**< Write the sequence and picture header in encoded bitstream of the current picture */ + NV_ENC_PIC_FLAG_EOS = 0x8, /**< Indicates end of the input stream */ + NV_ENC_PIC_FLAG_DISABLE_ENC_STATE_ADVANCE = 0x10, /**< Do not advance encoder state during encode */ + NV_ENC_PIC_FLAG_OUTPUT_RECON_FRAME = 0x20, /**< Write reconstructed frame */ +} NV_ENC_PIC_FLAGS; + +/** + * Memory heap to allocate input and output buffers. + */ +typedef enum _NV_ENC_MEMORY_HEAP +{ + NV_ENC_MEMORY_HEAP_AUTOSELECT = 0, /**< Memory heap to be decided by the encoder driver based on the usage */ + NV_ENC_MEMORY_HEAP_VID = 1, /**< Memory heap is in local video memory */ + NV_ENC_MEMORY_HEAP_SYSMEM_CACHED = 2, /**< Memory heap is in cached system memory */ + NV_ENC_MEMORY_HEAP_SYSMEM_UNCACHED = 3 /**< Memory heap is in uncached system memory */ +} NV_ENC_MEMORY_HEAP; + +/** + * B-frame used as reference modes + */ +typedef enum _NV_ENC_BFRAME_REF_MODE +{ + NV_ENC_BFRAME_REF_MODE_DISABLED = 0x0, /**< B frame is not used for reference */ + NV_ENC_BFRAME_REF_MODE_EACH = 0x1, /**< Each B-frame will be used for reference */ + NV_ENC_BFRAME_REF_MODE_MIDDLE = 0x2, /**< Only(Number of B-frame)/2 th B-frame will be used for reference */ +} NV_ENC_BFRAME_REF_MODE; + +/** + * H.264 entropy coding modes. + */ +typedef enum _NV_ENC_H264_ENTROPY_CODING_MODE +{ + NV_ENC_H264_ENTROPY_CODING_MODE_AUTOSELECT = 0x0, /**< Entropy coding mode is auto selected by the encoder driver */ + NV_ENC_H264_ENTROPY_CODING_MODE_CABAC = 0x1, /**< Entropy coding mode is CABAC */ + NV_ENC_H264_ENTROPY_CODING_MODE_CAVLC = 0x2 /**< Entropy coding mode is CAVLC */ +} NV_ENC_H264_ENTROPY_CODING_MODE; + +/** + * H.264 specific BDirect modes + */ +typedef enum _NV_ENC_H264_BDIRECT_MODE +{ + NV_ENC_H264_BDIRECT_MODE_AUTOSELECT = 0x0, /**< BDirect mode is auto selected by the encoder driver */ + NV_ENC_H264_BDIRECT_MODE_DISABLE = 0x1, /**< Disable BDirect mode */ + NV_ENC_H264_BDIRECT_MODE_TEMPORAL = 0x2, /**< Temporal BDirect mode */ + NV_ENC_H264_BDIRECT_MODE_SPATIAL = 0x3 /**< Spatial BDirect mode */ +} NV_ENC_H264_BDIRECT_MODE; + +/** + * H.264 specific FMO usage + */ +typedef enum _NV_ENC_H264_FMO_MODE +{ + NV_ENC_H264_FMO_AUTOSELECT = 0x0, /**< FMO usage is auto selected by the encoder driver */ + NV_ENC_H264_FMO_ENABLE = 0x1, /**< Enable FMO */ + NV_ENC_H264_FMO_DISABLE = 0x2, /**< Disable FMO */ +} NV_ENC_H264_FMO_MODE; + +/** + * H.264 specific Adaptive Transform modes + */ +typedef enum _NV_ENC_H264_ADAPTIVE_TRANSFORM_MODE +{ + NV_ENC_H264_ADAPTIVE_TRANSFORM_AUTOSELECT = 0x0, /**< Adaptive Transform 8x8 mode is auto selected by the encoder driver*/ + NV_ENC_H264_ADAPTIVE_TRANSFORM_DISABLE = 0x1, /**< Adaptive Transform 8x8 mode disabled */ + NV_ENC_H264_ADAPTIVE_TRANSFORM_ENABLE = 0x2, /**< Adaptive Transform 8x8 mode should be used */ +} NV_ENC_H264_ADAPTIVE_TRANSFORM_MODE; + +/** + * Stereo frame packing modes. + */ +typedef enum _NV_ENC_STEREO_PACKING_MODE +{ + NV_ENC_STEREO_PACKING_MODE_NONE = 0x0, /**< No Stereo packing required */ + NV_ENC_STEREO_PACKING_MODE_CHECKERBOARD = 0x1, /**< Checkerboard mode for packing stereo frames */ + NV_ENC_STEREO_PACKING_MODE_COLINTERLEAVE = 0x2, /**< Column Interleave mode for packing stereo frames */ + NV_ENC_STEREO_PACKING_MODE_ROWINTERLEAVE = 0x3, /**< Row Interleave mode for packing stereo frames */ + NV_ENC_STEREO_PACKING_MODE_SIDEBYSIDE = 0x4, /**< Side-by-side mode for packing stereo frames */ + NV_ENC_STEREO_PACKING_MODE_TOPBOTTOM = 0x5, /**< Top-Bottom mode for packing stereo frames */ + NV_ENC_STEREO_PACKING_MODE_FRAMESEQ = 0x6 /**< Frame Sequential mode for packing stereo frames */ +} NV_ENC_STEREO_PACKING_MODE; + +/** + * Input Resource type + */ +typedef enum _NV_ENC_INPUT_RESOURCE_TYPE +{ + NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX = 0x0, /**< input resource type is a directx9 surface*/ + NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR = 0x1, /**< input resource type is a cuda device pointer surface*/ + NV_ENC_INPUT_RESOURCE_TYPE_CUDAARRAY = 0x2, /**< input resource type is a cuda array surface. + This array must be a 2D array and the CUDA_ARRAY3D_SURFACE_LDST + flag must have been specified when creating it. */ + NV_ENC_INPUT_RESOURCE_TYPE_OPENGL_TEX = 0x3 /**< input resource type is an OpenGL texture */ +} NV_ENC_INPUT_RESOURCE_TYPE; + +/** + * Buffer usage + */ +typedef enum _NV_ENC_BUFFER_USAGE +{ + NV_ENC_INPUT_IMAGE = 0x0, /**< Registered surface will be used for input image */ + NV_ENC_OUTPUT_MOTION_VECTOR = 0x1, /**< Registered surface will be used for output of H.264 ME only mode. + This buffer usage type is not supported for HEVC ME only mode. */ + NV_ENC_OUTPUT_BITSTREAM = 0x2, /**< Registered surface will be used for output bitstream in encoding */ + NV_ENC_OUTPUT_RECON = 0x4, /**< Registered surface will be used for output reconstructed frame in encoding */ +} NV_ENC_BUFFER_USAGE; + +/** + * Encoder Device type + */ +typedef enum _NV_ENC_DEVICE_TYPE +{ + NV_ENC_DEVICE_TYPE_DIRECTX = 0x0, /**< encode device type is a directx9 device */ + NV_ENC_DEVICE_TYPE_CUDA = 0x1, /**< encode device type is a cuda device */ + NV_ENC_DEVICE_TYPE_OPENGL = 0x2 /**< encode device type is an OpenGL device. + Use of this device type is supported only on Linux */ +} NV_ENC_DEVICE_TYPE; + +/** + * Number of reference frames + */ +typedef enum _NV_ENC_NUM_REF_FRAMES +{ + NV_ENC_NUM_REF_FRAMES_AUTOSELECT = 0x0, /**< Number of reference frames is auto selected by the encoder driver */ + NV_ENC_NUM_REF_FRAMES_1 = 0x1, /**< Number of reference frames equal to 1 */ + NV_ENC_NUM_REF_FRAMES_2 = 0x2, /**< Number of reference frames equal to 2 */ + NV_ENC_NUM_REF_FRAMES_3 = 0x3, /**< Number of reference frames equal to 3 */ + NV_ENC_NUM_REF_FRAMES_4 = 0x4, /**< Number of reference frames equal to 4 */ + NV_ENC_NUM_REF_FRAMES_5 = 0x5, /**< Number of reference frames equal to 5 */ + NV_ENC_NUM_REF_FRAMES_6 = 0x6, /**< Number of reference frames equal to 6 */ + NV_ENC_NUM_REF_FRAMES_7 = 0x7 /**< Number of reference frames equal to 7 */ +} NV_ENC_NUM_REF_FRAMES; + +/** + * Encoder capabilities enumeration. + */ +typedef enum _NV_ENC_CAPS +{ + /** + * Maximum number of B-Frames supported. + */ + NV_ENC_CAPS_NUM_MAX_BFRAMES, + + /** + * Rate control modes supported. + * \n The API return value is a bitmask of the values in NV_ENC_PARAMS_RC_MODE. + */ + NV_ENC_CAPS_SUPPORTED_RATECONTROL_MODES, + + /** + * Indicates HW support for field mode encoding. + * \n 0 : Interlaced mode encoding is not supported. + * \n 1 : Interlaced field mode encoding is supported. + * \n 2 : Interlaced frame encoding and field mode encoding are both supported. + */ + NV_ENC_CAPS_SUPPORT_FIELD_ENCODING, + + /** + * Indicates HW support for monochrome mode encoding. + * \n 0 : Monochrome mode not supported. + * \n 1 : Monochrome mode supported. + */ + NV_ENC_CAPS_SUPPORT_MONOCHROME, + + /** + * Indicates HW support for FMO. + * \n 0 : FMO not supported. + * \n 1 : FMO supported. + */ + NV_ENC_CAPS_SUPPORT_FMO, + + /** + * Indicates HW capability for Quarter pel motion estimation. + * \n 0 : Quarter-Pel Motion Estimation not supported. + * \n 1 : Quarter-Pel Motion Estimation supported. + */ + NV_ENC_CAPS_SUPPORT_QPELMV, + + /** + * H.264 specific. Indicates HW support for BDirect modes. + * \n 0 : BDirect mode encoding not supported. + * \n 1 : BDirect mode encoding supported. + */ + NV_ENC_CAPS_SUPPORT_BDIRECT_MODE, + + /** + * H264 specific. Indicates HW support for CABAC entropy coding mode. + * \n 0 : CABAC entropy coding not supported. + * \n 1 : CABAC entropy coding supported. + */ + NV_ENC_CAPS_SUPPORT_CABAC, + + /** + * Indicates HW support for Adaptive Transform. + * \n 0 : Adaptive Transform not supported. + * \n 1 : Adaptive Transform supported. + */ + NV_ENC_CAPS_SUPPORT_ADAPTIVE_TRANSFORM, + + /** + * Indicates HW support for Multi View Coding. + * \n 0 : Multi View Coding not supported. + * \n 1 : Multi View Coding supported. + */ + NV_ENC_CAPS_SUPPORT_STEREO_MVC, + + /** + * Indicates HW support for encoding Temporal layers. + * \n 0 : Encoding Temporal layers not supported. + * \n 1 : Encoding Temporal layers supported. + */ + NV_ENC_CAPS_NUM_MAX_TEMPORAL_LAYERS, + + /** + * Indicates HW support for Hierarchical P frames. + * \n 0 : Hierarchical P frames not supported. + * \n 1 : Hierarchical P frames supported. + */ + NV_ENC_CAPS_SUPPORT_HIERARCHICAL_PFRAMES, + + /** + * Indicates HW support for Hierarchical B frames. + * \n 0 : Hierarchical B frames not supported. + * \n 1 : Hierarchical B frames supported. + */ + NV_ENC_CAPS_SUPPORT_HIERARCHICAL_BFRAMES, + + /** + * Maximum Encoding level supported (See ::NV_ENC_LEVEL for details). + */ + NV_ENC_CAPS_LEVEL_MAX, + + /** + * Minimum Encoding level supported (See ::NV_ENC_LEVEL for details). + */ + NV_ENC_CAPS_LEVEL_MIN, + + /** + * Indicates HW support for separate colour plane encoding. + * \n 0 : Separate colour plane encoding not supported. + * \n 1 : Separate colour plane encoding supported. + */ + NV_ENC_CAPS_SEPARATE_COLOUR_PLANE, + + /** + * Maximum output width supported. + */ + NV_ENC_CAPS_WIDTH_MAX, + + /** + * Maximum output height supported. + */ + NV_ENC_CAPS_HEIGHT_MAX, + + /** + * Indicates Temporal Scalability Support. + * \n 0 : Temporal SVC encoding not supported. + * \n 1 : Temporal SVC encoding supported. + */ + NV_ENC_CAPS_SUPPORT_TEMPORAL_SVC, + + /** + * Indicates Dynamic Encode Resolution Change Support. + * Support added from NvEncodeAPI version 2.0. + * \n 0 : Dynamic Encode Resolution Change not supported. + * \n 1 : Dynamic Encode Resolution Change supported. + */ + NV_ENC_CAPS_SUPPORT_DYN_RES_CHANGE, + + /** + * Indicates Dynamic Encode Bitrate Change Support. + * Support added from NvEncodeAPI version 2.0. + * \n 0 : Dynamic Encode bitrate change not supported. + * \n 1 : Dynamic Encode bitrate change supported. + */ + NV_ENC_CAPS_SUPPORT_DYN_BITRATE_CHANGE, + + /** + * Indicates Forcing Constant QP On The Fly Support. + * Support added from NvEncodeAPI version 2.0. + * \n 0 : Forcing constant QP on the fly not supported. + * \n 1 : Forcing constant QP on the fly supported. + */ + NV_ENC_CAPS_SUPPORT_DYN_FORCE_CONSTQP, + + /** + * Indicates Dynamic rate control mode Change Support. + * \n 0 : Dynamic rate control mode change not supported. + * \n 1 : Dynamic rate control mode change supported. + */ + NV_ENC_CAPS_SUPPORT_DYN_RCMODE_CHANGE, + + /** + * Indicates Subframe readback support for slice-based encoding. If this feature is supported, it can be enabled by setting enableSubFrameWrite = 1. + * \n 0 : Subframe readback not supported. + * \n 1 : Subframe readback supported. + */ + NV_ENC_CAPS_SUPPORT_SUBFRAME_READBACK, + + /** + * Indicates Constrained Encoding mode support. + * Support added from NvEncodeAPI version 2.0. + * \n 0 : Constrained encoding mode not supported. + * \n 1 : Constrained encoding mode supported. + * If this mode is supported client can enable this during initialization. + * Client can then force a picture to be coded as constrained picture where + * in-loop filtering is disabled across slice boundaries and prediction vectors for inter + * macroblocks in each slice will be restricted to the slice region. + */ + NV_ENC_CAPS_SUPPORT_CONSTRAINED_ENCODING, + + /** + * Indicates Intra Refresh Mode Support. + * Support added from NvEncodeAPI version 2.0. + * \n 0 : Intra Refresh Mode not supported. + * \n 1 : Intra Refresh Mode supported. + */ + NV_ENC_CAPS_SUPPORT_INTRA_REFRESH, + + /** + * Indicates Custom VBV Buffer Size support. It can be used for capping frame size. + * Support added from NvEncodeAPI version 2.0. + * \n 0 : Custom VBV buffer size specification from client, not supported. + * \n 1 : Custom VBV buffer size specification from client, supported. + */ + NV_ENC_CAPS_SUPPORT_CUSTOM_VBV_BUF_SIZE, + + /** + * Indicates Dynamic Slice Mode Support. + * Support added from NvEncodeAPI version 2.0. + * \n 0 : Dynamic Slice Mode not supported. + * \n 1 : Dynamic Slice Mode supported. + */ + NV_ENC_CAPS_SUPPORT_DYNAMIC_SLICE_MODE, + + /** + * Indicates Reference Picture Invalidation Support. + * Support added from NvEncodeAPI version 2.0. + * \n 0 : Reference Picture Invalidation not supported. + * \n 1 : Reference Picture Invalidation supported. + */ + NV_ENC_CAPS_SUPPORT_REF_PIC_INVALIDATION, + + /** + * Indicates support for Pre-Processing. + * The API return value is a bitmask of the values defined in ::NV_ENC_PREPROC_FLAGS + */ + NV_ENC_CAPS_PREPROC_SUPPORT, + + /** + * Indicates support Async mode. + * \n 0 : Async Encode mode not supported. + * \n 1 : Async Encode mode supported. + */ + NV_ENC_CAPS_ASYNC_ENCODE_SUPPORT, + + /** + * Maximum MBs per frame supported. + */ + NV_ENC_CAPS_MB_NUM_MAX, + + /** + * Maximum aggregate throughput in MBs per sec. + */ + NV_ENC_CAPS_MB_PER_SEC_MAX, + + /** + * Indicates HW support for YUV444 mode encoding. + * \n 0 : YUV444 mode encoding not supported. + * \n 1 : YUV444 mode encoding supported. + */ + NV_ENC_CAPS_SUPPORT_YUV444_ENCODE, + + /** + * Indicates HW support for lossless encoding. + * \n 0 : lossless encoding not supported. + * \n 1 : lossless encoding supported. + */ + NV_ENC_CAPS_SUPPORT_LOSSLESS_ENCODE, + + /** + * Indicates HW support for Sample Adaptive Offset. + * \n 0 : SAO not supported. + * \n 1 : SAO encoding supported. + */ + NV_ENC_CAPS_SUPPORT_SAO, + + /** + * Indicates HW support for Motion Estimation Only Mode. + * \n 0 : MEOnly Mode not supported. + * \n 1 : MEOnly Mode supported for I and P frames. + * \n 2 : MEOnly Mode supported for I, P and B frames. + */ + NV_ENC_CAPS_SUPPORT_MEONLY_MODE, + + /** + * Indicates HW support for lookahead encoding (enableLookahead=1). + * \n 0 : Lookahead not supported. + * \n 1 : Lookahead supported. + */ + NV_ENC_CAPS_SUPPORT_LOOKAHEAD, + + /** + * Indicates HW support for temporal AQ encoding (enableTemporalAQ=1). + * \n 0 : Temporal AQ not supported. + * \n 1 : Temporal AQ supported. + */ + NV_ENC_CAPS_SUPPORT_TEMPORAL_AQ, + /** + * Indicates HW support for 10 bit encoding. + * \n 0 : 10 bit encoding not supported. + * \n 1 : 10 bit encoding supported. + */ + NV_ENC_CAPS_SUPPORT_10BIT_ENCODE, + /** + * Maximum number of Long Term Reference frames supported + */ + NV_ENC_CAPS_NUM_MAX_LTR_FRAMES, + + /** + * Indicates HW support for Weighted Prediction. + * \n 0 : Weighted Prediction not supported. + * \n 1 : Weighted Prediction supported. + */ + NV_ENC_CAPS_SUPPORT_WEIGHTED_PREDICTION, + + + /** + * On managed (vGPU) platforms (Windows only), this API, in conjunction with other GRID Management APIs, can be used + * to estimate the residual capacity of the hardware encoder on the GPU as a percentage of the total available encoder capacity. + * This API can be called at any time; i.e. during the encode session or before opening the encode session. + * If the available encoder capacity is returned as zero, applications may choose to switch to software encoding + * and continue to call this API (e.g. polling once per second) until capacity becomes available. + * + * On bare metal (non-virtualized GPU) and linux platforms, this API always returns 100. + */ + NV_ENC_CAPS_DYNAMIC_QUERY_ENCODER_CAPACITY, + + /** + * Indicates B as reference support. + * \n 0 : B as reference is not supported. + * \n 1 : each B-Frame as reference is supported. + * \n 2 : only Middle B-frame as reference is supported. + */ + NV_ENC_CAPS_SUPPORT_BFRAME_REF_MODE, + + /** + * Indicates HW support for Emphasis Level Map based delta QP computation. + * \n 0 : Emphasis Level Map based delta QP not supported. + * \n 1 : Emphasis Level Map based delta QP is supported. + */ + NV_ENC_CAPS_SUPPORT_EMPHASIS_LEVEL_MAP, + + /** + * Minimum input width supported. + */ + NV_ENC_CAPS_WIDTH_MIN, + + /** + * Minimum input height supported. + */ + NV_ENC_CAPS_HEIGHT_MIN, + + /** + * Indicates HW support for multiple reference frames. + */ + NV_ENC_CAPS_SUPPORT_MULTIPLE_REF_FRAMES, + + /** + * Indicates HW support for HEVC with alpha encoding. + * \n 0 : HEVC with alpha encoding not supported. + * \n 1 : HEVC with alpha encoding is supported. + */ + NV_ENC_CAPS_SUPPORT_ALPHA_LAYER_ENCODING, + + /** + * Indicates number of Encoding engines present on GPU. + */ + NV_ENC_CAPS_NUM_ENCODER_ENGINES, + + /** + * Indicates single slice intra refresh support. + */ + NV_ENC_CAPS_SINGLE_SLICE_INTRA_REFRESH, + + /** + * Indicates encoding without advancing the state support. + */ + NV_ENC_CAPS_DISABLE_ENC_STATE_ADVANCE, + + /** + * Indicates reconstructed output support. + */ + NV_ENC_CAPS_OUTPUT_RECON_SURFACE, + + /** + * Indicates encoded frame output stats support for every block. Block represents a CTB for HEVC, macroblock for H.264 and super block for AV1. + */ + NV_ENC_CAPS_OUTPUT_BLOCK_STATS, + + /** + * Indicates encoded frame output stats support for every row. Row represents a CTB row for HEVC, macroblock row for H.264 and super block row for AV1. + */ + NV_ENC_CAPS_OUTPUT_ROW_STATS, + + /** + * Reserved - Not to be used by clients. + */ + NV_ENC_CAPS_EXPOSED_COUNT + +} NV_ENC_CAPS; + +/** + * HEVC CU SIZE + */ +typedef enum _NV_ENC_HEVC_CUSIZE +{ + NV_ENC_HEVC_CUSIZE_AUTOSELECT = 0, + NV_ENC_HEVC_CUSIZE_8x8 = 1, + NV_ENC_HEVC_CUSIZE_16x16 = 2, + NV_ENC_HEVC_CUSIZE_32x32 = 3, + NV_ENC_HEVC_CUSIZE_64x64 = 4, +} NV_ENC_HEVC_CUSIZE; + +/** +* AV1 PART SIZE +*/ +typedef enum _NV_ENC_AV1_PART_SIZE +{ + NV_ENC_AV1_PART_SIZE_AUTOSELECT = 0, + NV_ENC_AV1_PART_SIZE_4x4 = 1, + NV_ENC_AV1_PART_SIZE_8x8 = 2, + NV_ENC_AV1_PART_SIZE_16x16 = 3, + NV_ENC_AV1_PART_SIZE_32x32 = 4, + NV_ENC_AV1_PART_SIZE_64x64 = 5, +} NV_ENC_AV1_PART_SIZE; + +/** +* Enums related to fields in VUI parameters. +*/ +typedef enum _NV_ENC_VUI_VIDEO_FORMAT +{ + NV_ENC_VUI_VIDEO_FORMAT_COMPONENT = 0, + NV_ENC_VUI_VIDEO_FORMAT_PAL = 1, + NV_ENC_VUI_VIDEO_FORMAT_NTSC = 2, + NV_ENC_VUI_VIDEO_FORMAT_SECAM = 3, + NV_ENC_VUI_VIDEO_FORMAT_MAC = 4, + NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED = 5, +} NV_ENC_VUI_VIDEO_FORMAT; + +typedef enum _NV_ENC_VUI_COLOR_PRIMARIES +{ + NV_ENC_VUI_COLOR_PRIMARIES_UNDEFINED = 0, + NV_ENC_VUI_COLOR_PRIMARIES_BT709 = 1, + NV_ENC_VUI_COLOR_PRIMARIES_UNSPECIFIED = 2, + NV_ENC_VUI_COLOR_PRIMARIES_RESERVED = 3, + NV_ENC_VUI_COLOR_PRIMARIES_BT470M = 4, + NV_ENC_VUI_COLOR_PRIMARIES_BT470BG = 5, + NV_ENC_VUI_COLOR_PRIMARIES_SMPTE170M = 6, + NV_ENC_VUI_COLOR_PRIMARIES_SMPTE240M = 7, + NV_ENC_VUI_COLOR_PRIMARIES_FILM = 8, + NV_ENC_VUI_COLOR_PRIMARIES_BT2020 = 9, + NV_ENC_VUI_COLOR_PRIMARIES_SMPTE428 = 10, + NV_ENC_VUI_COLOR_PRIMARIES_SMPTE431 = 11, + NV_ENC_VUI_COLOR_PRIMARIES_SMPTE432 = 12, + NV_ENC_VUI_COLOR_PRIMARIES_JEDEC_P22 = 22, +} NV_ENC_VUI_COLOR_PRIMARIES; + +typedef enum _NV_ENC_VUI_TRANSFER_CHARACTERISTIC +{ + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_UNDEFINED = 0, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709 = 1, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_UNSPECIFIED = 2, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_RESERVED = 3, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT470M = 4, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT470BG = 5, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE170M = 6, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE240M = 7, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_LINEAR = 8, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_LOG = 9, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_LOG_SQRT = 10, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_IEC61966_2_4 = 11, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT1361_ECG = 12, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SRGB = 13, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT2020_10 = 14, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT2020_12 = 15, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE2084 = 16, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_SMPTE428 = 17, + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_ARIB_STD_B67 = 18, +} NV_ENC_VUI_TRANSFER_CHARACTERISTIC; + +typedef enum _NV_ENC_VUI_MATRIX_COEFFS +{ + NV_ENC_VUI_MATRIX_COEFFS_RGB = 0, + NV_ENC_VUI_MATRIX_COEFFS_BT709 = 1, + NV_ENC_VUI_MATRIX_COEFFS_UNSPECIFIED = 2, + NV_ENC_VUI_MATRIX_COEFFS_RESERVED = 3, + NV_ENC_VUI_MATRIX_COEFFS_FCC = 4, + NV_ENC_VUI_MATRIX_COEFFS_BT470BG = 5, + NV_ENC_VUI_MATRIX_COEFFS_SMPTE170M = 6, + NV_ENC_VUI_MATRIX_COEFFS_SMPTE240M = 7, + NV_ENC_VUI_MATRIX_COEFFS_YCGCO = 8, + NV_ENC_VUI_MATRIX_COEFFS_BT2020_NCL = 9, + NV_ENC_VUI_MATRIX_COEFFS_BT2020_CL = 10, + NV_ENC_VUI_MATRIX_COEFFS_SMPTE2085 = 11, +} NV_ENC_VUI_MATRIX_COEFFS; + +/** + * Input struct for querying Encoding capabilities. + */ +typedef struct _NV_ENC_CAPS_PARAM +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_CAPS_PARAM_VER */ + NV_ENC_CAPS capsToQuery; /**< [in]: Specifies the encode capability to be queried. Client should pass a member for ::NV_ENC_CAPS enum. */ + uint32_t reserved[62]; /**< [in]: Reserved and must be set to 0 */ +} NV_ENC_CAPS_PARAM; + +/** NV_ENC_CAPS_PARAM struct version. */ +#define NV_ENC_CAPS_PARAM_VER NVENCAPI_STRUCT_VERSION(1) + + +/** + * Restore encoder state parameters + */ +typedef struct _NV_ENC_RESTORE_ENCODER_STATE_PARAMS +{ + uint32_t version; /**< [in]: Struct version. */ + uint32_t bufferIdx; /**< [in]: State buffer index to which the encoder state will be restored */ + NV_ENC_STATE_RESTORE_TYPE state; /**< [in]: State type to restore */ + NV_ENC_OUTPUT_PTR outputBitstream; /**< [in]: Specifies the output buffer pointer, for AV1 encode only. + Application must call NvEncRestoreEncoderState() API with _NV_ENC_RESTORE_ENCODER_STATE_PARAMS::outputBitstream and + _NV_ENC_RESTORE_ENCODER_STATE_PARAMS::completionEvent as input when an earlier call to this API returned "NV_ENC_ERR_NEED_MORE_OUTPUT", for AV1 encode. */ + void *completionEvent; /**< [in]: Specifies the completion event when asynchronous mode of encoding is enabled. Used for AV1 encode only. */ + uint32_t reserved1[64]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_RESTORE_ENCODER_STATE_PARAMS; + +/** NV_ENC_RESTORE_ENCODER_STATE_PARAMS struct version. */ +#define NV_ENC_RESTORE_ENCODER_STATE_PARAMS_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * Encoded frame information parameters for every block. + */ +typedef struct _NV_ENC_OUTPUT_STATS_BLOCK +{ + uint32_t version; /**< [in]: Struct version */ + uint8_t QP; /**< [out]: QP of the block */ + uint8_t reserved[3]; /**< [in]: Reserved and must be set to 0 */ + uint32_t bitcount; /**< [out]: Bitcount of the block */ + uint32_t reserved1[13]; /**< [in]: Reserved and must be set to 0 */ +} NV_ENC_OUTPUT_STATS_BLOCK; + +/** NV_ENC_OUTPUT_STATS_BLOCK struct version. */ +#define NV_ENC_OUTPUT_STATS_BLOCK_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * Encoded frame information parameters for every row. + */ +typedef struct _NV_ENC_OUTPUT_STATS_ROW +{ + uint32_t version; /**< [in]: Struct version */ + uint8_t QP; /**< [out]: QP of the row */ + uint8_t reserved[3]; /**< [in]: Reserved and must be set to 0 */ + uint32_t bitcount; /**< [out]: Bitcount of the row */ + uint32_t reserved1[13]; /**< [in]: Reserved and must be set to 0 */ +} NV_ENC_OUTPUT_STATS_ROW; + +/** NV_ENC_OUTPUT_STATS_ROW struct version. */ +#define NV_ENC_OUTPUT_STATS_ROW_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * Encoder Output parameters + */ +typedef struct _NV_ENC_ENCODE_OUT_PARAMS +{ + uint32_t version; /**< [out]: Struct version. */ + uint32_t bitstreamSizeInBytes; /**< [out]: Encoded bitstream size in bytes */ + uint32_t reserved[62]; /**< [out]: Reserved and must be set to 0 */ +} NV_ENC_ENCODE_OUT_PARAMS; + +/** NV_ENC_ENCODE_OUT_PARAMS struct version. */ +#define NV_ENC_ENCODE_OUT_PARAMS_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * Lookahead picture parameters + */ +typedef struct _NV_ENC_LOOKAHEAD_PIC_PARAMS +{ + uint32_t version; /**< [in]: Struct version. */ + NV_ENC_INPUT_PTR inputBuffer; /**< [in]: Specifies the input buffer pointer. Client must use a pointer obtained from ::NvEncCreateInputBuffer() or ::NvEncMapInputResource() APIs.*/ + NV_ENC_PIC_TYPE pictureType; /**< [in]: Specifies input picture type. Client required to be set explicitly by the client if the client has not set NV_ENC_INITALIZE_PARAMS::enablePTD to 1 while calling NvInitializeEncoder. */ + uint32_t reserved[64]; /**< [in]: Reserved and must be set to 0 */ + void *reserved1[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_LOOKAHEAD_PIC_PARAMS; + +/** NV_ENC_LOOKAHEAD_PIC_PARAMS struct version. */ +#define NV_ENC_LOOKAHEAD_PIC_PARAMS_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * Creation parameters for input buffer. + */ +typedef struct _NV_ENC_CREATE_INPUT_BUFFER +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_CREATE_INPUT_BUFFER_VER */ + uint32_t width; /**< [in]: Input frame width */ + uint32_t height; /**< [in]: Input frame height */ + NV_ENC_MEMORY_HEAP memoryHeap; /**< [in]: Deprecated. Do not use */ + NV_ENC_BUFFER_FORMAT bufferFmt; /**< [in]: Input buffer format */ + uint32_t reserved; /**< [in]: Reserved and must be set to 0 */ + NV_ENC_INPUT_PTR inputBuffer; /**< [out]: Pointer to input buffer */ + void *pSysMemBuffer; /**< [in]: Pointer to existing system memory buffer */ + uint32_t reserved1[57]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[63]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_CREATE_INPUT_BUFFER; + +/** NV_ENC_CREATE_INPUT_BUFFER struct version. */ +#define NV_ENC_CREATE_INPUT_BUFFER_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * Creation parameters for output bitstream buffer. + */ +typedef struct _NV_ENC_CREATE_BITSTREAM_BUFFER +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_CREATE_BITSTREAM_BUFFER_VER */ + uint32_t size; /**< [in]: Deprecated. Do not use */ + NV_ENC_MEMORY_HEAP memoryHeap; /**< [in]: Deprecated. Do not use */ + uint32_t reserved; /**< [in]: Reserved and must be set to 0 */ + NV_ENC_OUTPUT_PTR bitstreamBuffer; /**< [out]: Pointer to the output bitstream buffer */ + void *bitstreamBufferPtr; /**< [out]: Reserved and should not be used */ + uint32_t reserved1[58]; /**< [in]: Reserved and should be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and should be set to NULL */ +} NV_ENC_CREATE_BITSTREAM_BUFFER; + +/** NV_ENC_CREATE_BITSTREAM_BUFFER struct version. */ +#define NV_ENC_CREATE_BITSTREAM_BUFFER_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * Structs needed for ME only mode. + */ +typedef struct _NV_ENC_MVECTOR +{ + int16_t mvx; /**< the x component of MV in quarter-pel units */ + int16_t mvy; /**< the y component of MV in quarter-pel units */ +} NV_ENC_MVECTOR; + +/** + * Motion vector structure per macroblock for H264 motion estimation. + */ +typedef struct _NV_ENC_H264_MV_DATA +{ + NV_ENC_MVECTOR mv[4]; /**< up to 4 vectors for 8x8 partition */ + uint8_t mbType; /**< 0 (I), 1 (P), 2 (IPCM), 3 (B) */ + uint8_t partitionType; /**< Specifies the block partition type. 0:16x16, 1:8x8, 2:16x8, 3:8x16 */ + uint16_t reserved; /**< reserved padding for alignment */ + uint32_t mbCost; +} NV_ENC_H264_MV_DATA; + +/** + * Motion vector structure per CU for HEVC motion estimation. + */ +typedef struct _NV_ENC_HEVC_MV_DATA +{ + NV_ENC_MVECTOR mv[4]; /**< up to 4 vectors within a CU */ + uint8_t cuType; /**< 0 (I), 1(P) */ + uint8_t cuSize; /**< 0: 8x8, 1: 16x16, 2: 32x32, 3: 64x64 */ + uint8_t partitionMode; /**< The CU partition mode + 0 (2Nx2N), 1 (2NxN), 2(Nx2N), 3 (NxN), + 4 (2NxnU), 5 (2NxnD), 6(nLx2N), 7 (nRx2N) */ + uint8_t lastCUInCTB; /**< Marker to separate CUs in the current CTB from CUs in the next CTB */ +} NV_ENC_HEVC_MV_DATA; + +/** + * Creation parameters for output motion vector buffer for ME only mode. + */ +typedef struct _NV_ENC_CREATE_MV_BUFFER +{ + uint32_t version; /**< [in]: Struct version. Must be set to NV_ENC_CREATE_MV_BUFFER_VER */ + NV_ENC_OUTPUT_PTR mvBuffer; /**< [out]: Pointer to the output motion vector buffer */ + uint32_t reserved1[255]; /**< [in]: Reserved and should be set to 0 */ + void *reserved2[63]; /**< [in]: Reserved and should be set to NULL */ +} NV_ENC_CREATE_MV_BUFFER; + +/** NV_ENC_CREATE_MV_BUFFER struct version*/ +#define NV_ENC_CREATE_MV_BUFFER_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * QP value for frames + */ +typedef struct _NV_ENC_QP +{ + uint32_t qpInterP; /**< [in]: Specifies QP value for P-frame. Even though this field is uint32_t for legacy reasons, the client should treat this as a signed parameter(int32_t) for cases in which negative QP values are to be specified. */ + uint32_t qpInterB; /**< [in]: Specifies QP value for B-frame. Even though this field is uint32_t for legacy reasons, the client should treat this as a signed parameter(int32_t) for cases in which negative QP values are to be specified. */ + uint32_t qpIntra; /**< [in]: Specifies QP value for Intra Frame. Even though this field is uint32_t for legacy reasons, the client should treat this as a signed parameter(int32_t) for cases in which negative QP values are to be specified. */ +} NV_ENC_QP; + +/** + * Rate Control Configuration Parameters + */ +typedef struct _NV_ENC_RC_PARAMS +{ + uint32_t version; + NV_ENC_PARAMS_RC_MODE rateControlMode; /**< [in]: Specifies the rate control mode. Check support for various rate control modes using ::NV_ENC_CAPS_SUPPORTED_RATECONTROL_MODES caps. */ + NV_ENC_QP constQP; /**< [in]: Specifies the initial QP to be used for encoding, these values would be used for all frames if in Constant QP mode. */ + uint32_t averageBitRate; /**< [in]: Specifies the average bitrate(in bits/sec) used for encoding. */ + uint32_t maxBitRate; /**< [in]: Specifies the maximum bitrate for the encoded output. This is used for VBR and ignored for CBR mode. */ + uint32_t vbvBufferSize; /**< [in]: Specifies the VBV(HRD) buffer size. in bits. Set 0 to use the default VBV buffer size. */ + uint32_t vbvInitialDelay; /**< [in]: Specifies the VBV(HRD) initial delay in bits. Set 0 to use the default VBV initial delay .*/ + uint32_t enableMinQP : 1; /**< [in]: Set this to 1 if minimum QP used for rate control. */ + uint32_t enableMaxQP : 1; /**< [in]: Set this to 1 if maximum QP used for rate control. */ + uint32_t enableInitialRCQP : 1; /**< [in]: Set this to 1 if user supplied initial QP is used for rate control. */ + uint32_t enableAQ : 1; /**< [in]: Set this to 1 to enable adaptive quantization (Spatial). */ + uint32_t reservedBitField1 : 1; /**< [in]: Reserved bitfields and must be set to 0. */ + uint32_t enableLookahead : 1; /**< [in]: Set this to 1 to enable lookahead with depth (if lookahead is enabled, input frames must remain available to the encoder until encode completion) */ + uint32_t disableIadapt : 1; /**< [in]: Set this to 1 to disable adaptive I-frame insertion at scene cuts (only has an effect when lookahead is enabled) */ + uint32_t disableBadapt : 1; /**< [in]: Set this to 1 to disable adaptive B-frame decision (only has an effect when lookahead is enabled) */ + uint32_t enableTemporalAQ : 1; /**< [in]: Set this to 1 to enable temporal AQ */ + uint32_t zeroReorderDelay : 1; /**< [in]: Set this to 1 to indicate zero latency operation (no reordering delay, num_reorder_frames=0) */ + uint32_t enableNonRefP : 1; /**< [in]: Set this to 1 to enable automatic insertion of non-reference P-frames (no effect if enablePTD=0) */ + uint32_t strictGOPTarget : 1; /**< [in]: Set this to 1 to minimize GOP-to-GOP rate fluctuations */ + uint32_t aqStrength : 4; /**< [in]: When AQ (Spatial) is enabled (i.e. NV_ENC_RC_PARAMS::enableAQ is set), this field is used to specify AQ strength. AQ strength scale is from 1 (low) - 15 (aggressive). If not set, strength is auto selected by driver. */ + uint32_t enableExtLookahead : 1; /**< [in]: Set this to 1 to enable lookahead externally. + Application must call NvEncLookahead() for NV_ENC_RC_PARAMS::lookaheadDepth number of frames, + before calling NvEncEncodePicture() for the first frame */ + uint32_t reservedBitFields : 15; /**< [in]: Reserved bitfields and must be set to 0 */ + NV_ENC_QP minQP; /**< [in]: Specifies the minimum QP used for rate control. Client must set NV_ENC_CONFIG::enableMinQP to 1. */ + NV_ENC_QP maxQP; /**< [in]: Specifies the maximum QP used for rate control. Client must set NV_ENC_CONFIG::enableMaxQP to 1. */ + NV_ENC_QP initialRCQP; /**< [in]: Specifies the initial QP hint used for rate control. The parameter is just used as hint to influence the QP difference between I,P and B frames. + Client must set NV_ENC_CONFIG::enableInitialRCQP to 1. */ + uint32_t temporallayerIdxMask; /**< [in]: Specifies the temporal layers (as a bitmask) whose QPs have changed. Valid max bitmask is [2^NV_ENC_CAPS_NUM_MAX_TEMPORAL_LAYERS - 1]. + Applicable only for constant QP mode (NV_ENC_RC_PARAMS::rateControlMode = NV_ENC_PARAMS_RC_CONSTQP). */ + uint8_t temporalLayerQP[8]; /**< [in]: Specifies the temporal layer QPs used for rate control. Temporal layer index is used as the array index. + Applicable only for constant QP mode (NV_ENC_RC_PARAMS::rateControlMode = NV_ENC_PARAMS_RC_CONSTQP). */ + uint8_t targetQuality; /**< [in]: Target CQ (Constant Quality) level for VBR mode (range 0-51 with 0-automatic) */ + uint8_t targetQualityLSB; /**< [in]: Fractional part of target quality (as 8.8 fixed point format) */ + uint16_t lookaheadDepth; /**< [in]: Maximum depth of lookahead with range 0-(31 - number of B frames). + lookaheadDepth is only used if enableLookahead=1.*/ + uint8_t lowDelayKeyFrameScale; /**< [in]: Specifies the ratio of I frame bits to P frame bits in case of single frame VBV and CBR rate control mode, + is set to 2 by default for low latency tuning info and 1 by default for ultra low latency tuning info */ + int8_t yDcQPIndexOffset; /**< [in]: Specifies the value of 'deltaQ_y_dc' in AV1.*/ + int8_t uDcQPIndexOffset; /**< [in]: Specifies the value of 'deltaQ_u_dc' in AV1.*/ + int8_t vDcQPIndexOffset; /**< [in]: Specifies the value of 'deltaQ_v_dc' in AV1 (for future use only - deltaQ_v_dc is currently always internally set to same value as deltaQ_u_dc). */ + NV_ENC_QP_MAP_MODE qpMapMode; /**< [in]: This flag is used to interpret values in array specified by NV_ENC_PIC_PARAMS::qpDeltaMap. + Set this to NV_ENC_QP_MAP_EMPHASIS to treat values specified by NV_ENC_PIC_PARAMS::qpDeltaMap as Emphasis Level Map. + Emphasis Level can be assigned any value specified in enum NV_ENC_EMPHASIS_MAP_LEVEL. + Emphasis Level Map is used to specify regions to be encoded at varying levels of quality. + The hardware encoder adjusts the quantization within the image as per the provided emphasis map, + by adjusting the quantization parameter (QP) assigned to each macroblock. This adjustment is commonly called "Delta QP". + The adjustment depends on the absolute QP decided by the rate control algorithm, and is applied after the rate control has decided each macroblock's QP. + Since the Delta QP overrides rate control, enabling Emphasis Level Map may violate bitrate and VBV buffer size constraints. + Emphasis Level Map is useful in situations where client has a priori knowledge of the image complexity (e.g. via use of NVFBC's Classification feature) and encoding those high-complexity areas at higher quality (lower QP) is important, even at the possible cost of violating bitrate/VBV buffer size constraints + This feature is not supported when AQ( Spatial/Temporal) is enabled. + This feature is only supported for H264 codec currently. + + Set this to NV_ENC_QP_MAP_DELTA to treat values specified by NV_ENC_PIC_PARAMS::qpDeltaMap as QP Delta. This specifies QP modifier to be applied on top of the QP chosen by rate control + + Set this to NV_ENC_QP_MAP_DISABLED to ignore NV_ENC_PIC_PARAMS::qpDeltaMap values. In this case, qpDeltaMap should be set to NULL. + + Other values are reserved for future use.*/ + NV_ENC_MULTI_PASS multiPass; /**< [in]: This flag is used to enable multi-pass encoding for a given ::NV_ENC_PARAMS_RC_MODE. This flag is not valid for H264 and HEVC MEOnly mode */ + uint32_t alphaLayerBitrateRatio; /**< [in]: Specifies the ratio in which bitrate should be split between base and alpha layer. A value 'x' for this field will split the target bitrate in a ratio of x : 1 between base and alpha layer. + The default split ratio is 15.*/ + int8_t cbQPIndexOffset; /**< [in]: Specifies the value of 'chroma_qp_index_offset' in H264 / 'pps_cb_qp_offset' in HEVC / 'deltaQ_u_ac' in AV1.*/ + int8_t crQPIndexOffset; /**< [in]: Specifies the value of 'second_chroma_qp_index_offset' in H264 / 'pps_cr_qp_offset' in HEVC / 'deltaQ_v_ac' in AV1 (for future use only - deltaQ_v_ac is currently always internally set to same value as deltaQ_u_ac). */ + uint16_t reserved2; + uint32_t reserved[4]; +} NV_ENC_RC_PARAMS; + +/** macro for constructing the version field of ::_NV_ENC_RC_PARAMS */ +#define NV_ENC_RC_PARAMS_VER NVENCAPI_STRUCT_VERSION(1) + +#define MAX_NUM_CLOCK_TS 3 + +/** +* Clock Timestamp set parameters +* For H264, this structure is used to populate Picture Timing SEI when NV_ENC_CONFIG_H264::enableTimeCode is set to 1. +* For HEVC, this structure is used to populate Time Code SEI when NV_ENC_CONFIG_HEVC::enableTimeCodeSEI is set to 1. +* For more details, refer to Annex D of ITU-T Specification. +*/ + +typedef struct _NV_ENC_CLOCK_TIMESTAMP_SET +{ + uint32_t countingType : 1; /**< [in] Specifies the 'counting_type' */ + uint32_t discontinuityFlag : 1; /**< [in] Specifies the 'discontinuity_flag' */ + uint32_t cntDroppedFrames : 1; /**< [in] Specifies the 'cnt_dropped_flag' */ + uint32_t nFrames : 8; /**< [in] Specifies the value of 'n_frames' */ + uint32_t secondsValue : 6; /**< [in] Specifies the 'seconds_value' */ + uint32_t minutesValue : 6; /**< [in] Specifies the 'minutes_value' */ + uint32_t hoursValue : 5; /**< [in] Specifies the 'hours_value' */ + uint32_t reserved2 : 4; /**< [in] Reserved and must be set to 0 */ + uint32_t timeOffset; /**< [in] Specifies the 'time_offset_value' */ +} NV_ENC_CLOCK_TIMESTAMP_SET; + +typedef struct _NV_ENC_TIME_CODE +{ + NV_ENC_DISPLAY_PIC_STRUCT displayPicStruct; /**< [in] Display picStruct */ + NV_ENC_CLOCK_TIMESTAMP_SET clockTimestamp[MAX_NUM_CLOCK_TS]; /**< [in] Clock Timestamp set */ +} NV_ENC_TIME_CODE; + + +/** + * \struct _NV_ENC_CONFIG_H264_VUI_PARAMETERS + * H264 Video Usability Info parameters + */ +typedef struct _NV_ENC_CONFIG_H264_VUI_PARAMETERS +{ + uint32_t overscanInfoPresentFlag; /**< [in]: If set to 1 , it specifies that the overscanInfo is present */ + uint32_t overscanInfo; /**< [in]: Specifies the overscan info(as defined in Annex E of the ITU-T Specification). */ + uint32_t videoSignalTypePresentFlag; /**< [in]: If set to 1, it specifies that the videoFormat, videoFullRangeFlag and colourDescriptionPresentFlag are present. */ + NV_ENC_VUI_VIDEO_FORMAT videoFormat; /**< [in]: Specifies the source video format(as defined in Annex E of the ITU-T Specification).*/ + uint32_t videoFullRangeFlag; /**< [in]: Specifies the output range of the luma and chroma samples(as defined in Annex E of the ITU-T Specification). */ + uint32_t colourDescriptionPresentFlag; /**< [in]: If set to 1, it specifies that the colourPrimaries, transferCharacteristics and colourMatrix are present. */ + NV_ENC_VUI_COLOR_PRIMARIES colourPrimaries; /**< [in]: Specifies color primaries for converting to RGB(as defined in Annex E of the ITU-T Specification) */ + NV_ENC_VUI_TRANSFER_CHARACTERISTIC transferCharacteristics; /**< [in]: Specifies the opto-electronic transfer characteristics to use (as defined in Annex E of the ITU-T Specification) */ + NV_ENC_VUI_MATRIX_COEFFS colourMatrix; /**< [in]: Specifies the matrix coefficients used in deriving the luma and chroma from the RGB primaries (as defined in Annex E of the ITU-T Specification). */ + uint32_t chromaSampleLocationFlag; /**< [in]: If set to 1 , it specifies that the chromaSampleLocationTop and chromaSampleLocationBot are present.*/ + uint32_t chromaSampleLocationTop; /**< [in]: Specifies the chroma sample location for top field(as defined in Annex E of the ITU-T Specification) */ + uint32_t chromaSampleLocationBot; /**< [in]: Specifies the chroma sample location for bottom field(as defined in Annex E of the ITU-T Specification) */ + uint32_t bitstreamRestrictionFlag; /**< [in]: If set to 1, it specifies the bitstream restriction parameters are present in the bitstream.*/ + uint32_t timingInfoPresentFlag; /**< [in]: If set to 1, it specifies that the timingInfo is present and the 'numUnitInTicks' and 'timeScale' fields are specified by the application. */ + /**< [in]: If not set, the timingInfo may still be present with timing related fields calculated internally basedon the frame rate specified by the application. */ + uint32_t numUnitInTicks; /**< [in]: Specifies the number of time units of the clock(as defined in Annex E of the ITU-T Specification). */ + uint32_t timeScale; /**< [in]: Specifies the frquency of the clock(as defined in Annex E of the ITU-T Specification). */ + uint32_t reserved[12]; /**< [in]: Reserved and must be set to 0 */ +} NV_ENC_CONFIG_H264_VUI_PARAMETERS; + +typedef NV_ENC_CONFIG_H264_VUI_PARAMETERS NV_ENC_CONFIG_HEVC_VUI_PARAMETERS; + +/** + * \struct _NVENC_EXTERNAL_ME_HINT_COUNTS_PER_BLOCKTYPE + * External motion vector hint counts per block type. + * H264 and AV1 support multiple hint while HEVC supports one hint for each valid candidate. + */ +typedef struct _NVENC_EXTERNAL_ME_HINT_COUNTS_PER_BLOCKTYPE +{ + uint32_t numCandsPerBlk16x16 : 4; /**< [in]: Supported for H264, HEVC. It Specifies the number of candidates per 16x16 block. */ + uint32_t numCandsPerBlk16x8 : 4; /**< [in]: Supported for H264 only. Specifies the number of candidates per 16x8 block. */ + uint32_t numCandsPerBlk8x16 : 4; /**< [in]: Supported for H264 only. Specifies the number of candidates per 8x16 block. */ + uint32_t numCandsPerBlk8x8 : 4; /**< [in]: Supported for H264, HEVC. Specifies the number of candidates per 8x8 block. */ + uint32_t numCandsPerSb : 8; /**< [in]: Supported for AV1 only. Specifies the number of candidates per SB. */ + uint32_t reserved : 8; /**< [in]: Reserved for padding. */ + uint32_t reserved1[3]; /**< [in]: Reserved for future use. */ +} NVENC_EXTERNAL_ME_HINT_COUNTS_PER_BLOCKTYPE; + + +/** + * \struct _NVENC_EXTERNAL_ME_HINT + * External Motion Vector hint structure for H264 and HEVC. + */ +typedef struct _NVENC_EXTERNAL_ME_HINT +{ + int32_t mvx : 12; /**< [in]: Specifies the x component of integer pixel MV (relative to current MB) S12.0. */ + int32_t mvy : 10; /**< [in]: Specifies the y component of integer pixel MV (relative to current MB) S10.0 .*/ + int32_t refidx : 5; /**< [in]: Specifies the reference index (31=invalid). Current we support only 1 reference frame per direction for external hints, so \p refidx must be 0. */ + int32_t dir : 1; /**< [in]: Specifies the direction of motion estimation . 0=L0 1=L1.*/ + int32_t partType : 2; /**< [in]: Specifies the block partition type.0=16x16 1=16x8 2=8x16 3=8x8 (blocks in partition must be consecutive).*/ + int32_t lastofPart : 1; /**< [in]: Set to 1 for the last MV of (sub) partition */ + int32_t lastOfMB : 1; /**< [in]: Set to 1 for the last MV of macroblock. */ +} NVENC_EXTERNAL_ME_HINT; + +/** + * \struct _NVENC_EXTERNAL_ME_SB_HINT + * External Motion Vector SB hint structure for AV1 + */ +typedef struct _NVENC_EXTERNAL_ME_SB_HINT +{ + int16_t refidx : 5; /**< [in]: Specifies the reference index (31=invalid) */ + int16_t direction : 1; /**< [in]: Specifies the direction of motion estimation . 0=L0 1=L1.*/ + int16_t bi : 1; /**< [in]: Specifies reference mode 0=single mv, 1=compound mv */ + int16_t partition_type : 3; /**< [in]: Specifies the partition type: 0: 2NX2N, 1:2NxN, 2:Nx2N. reserved 3bits for future modes */ + int16_t x8 : 3; /**< [in]: Specifies the current partition's top left x position in 8 pixel unit */ + int16_t last_of_cu : 1; /**< [in]: Set to 1 for the last MV current CU */ + int16_t last_of_sb : 1; /**< [in]: Set to 1 for the last MV of current SB */ + int16_t reserved0 : 1; /**< [in]: Reserved and must be set to 0 */ + int16_t mvx : 14; /**< [in]: Specifies the x component of integer pixel MV (relative to current MB) S12.2. */ + int16_t cu_size : 2; /**< [in]: Specifies the CU size: 0: 8x8, 1: 16x16, 2:32x32, 3:64x64 */ + int16_t mvy : 12; /**< [in]: Specifies the y component of integer pixel MV (relative to current MB) S10.2 .*/ + int16_t y8 : 3; /**< [in]: Specifies the current partition's top left y position in 8 pixel unit */ + int16_t reserved1 : 1; /**< [in]: Reserved and must be set to 0 */ +} NVENC_EXTERNAL_ME_SB_HINT; + +/** + * \struct _NV_ENC_CONFIG_H264 + * H264 encoder configuration parameters + */ +typedef struct _NV_ENC_CONFIG_H264 +{ + uint32_t enableTemporalSVC : 1; /**< [in]: Set to 1 to enable SVC temporal*/ + uint32_t enableStereoMVC : 1; /**< [in]: Set to 1 to enable stereo MVC*/ + uint32_t hierarchicalPFrames : 1; /**< [in]: Set to 1 to enable hierarchical P Frames */ + uint32_t hierarchicalBFrames : 1; /**< [in]: Set to 1 to enable hierarchical B Frames */ + uint32_t outputBufferingPeriodSEI : 1; /**< [in]: Set to 1 to write SEI buffering period syntax in the bitstream */ + uint32_t outputPictureTimingSEI : 1; /**< [in]: Set to 1 to write SEI picture timing syntax in the bitstream. */ + uint32_t outputAUD : 1; /**< [in]: Set to 1 to write access unit delimiter syntax in bitstream */ + uint32_t disableSPSPPS : 1; /**< [in]: Set to 1 to disable writing of Sequence and Picture parameter info in bitstream */ + uint32_t outputFramePackingSEI : 1; /**< [in]: Set to 1 to enable writing of frame packing arrangement SEI messages to bitstream */ + uint32_t outputRecoveryPointSEI : 1; /**< [in]: Set to 1 to enable writing of recovery point SEI message */ + uint32_t enableIntraRefresh : 1; /**< [in]: Set to 1 to enable gradual decoder refresh or intra refresh. If the GOP structure uses B frames this will be ignored */ + uint32_t enableConstrainedEncoding : 1; /**< [in]: Set this to 1 to enable constrainedFrame encoding where each slice in the constrained picture is independent of other slices. + Constrained encoding works only with rectangular slices. + Check support for constrained encoding using ::NV_ENC_CAPS_SUPPORT_CONSTRAINED_ENCODING caps. */ + uint32_t repeatSPSPPS : 1; /**< [in]: Set to 1 to enable writing of Sequence and Picture parameter for every IDR frame */ + uint32_t enableVFR : 1; /**< [in]: Setting enableVFR=1 currently only sets the fixed_frame_rate_flag=0 in the VUI but otherwise + has no impact on the encoder behavior. For more details please refer to E.1 VUI syntax of H.264 standard. Note, however, that NVENC does not support VFR encoding and rate control. */ + uint32_t enableLTR : 1; /**< [in]: Set to 1 to enable LTR (Long Term Reference) frame support. LTR can be used in two modes: "LTR Trust" mode and "LTR Per Picture" mode. + LTR Trust mode: In this mode, ltrNumFrames pictures after IDR are automatically marked as LTR. This mode is enabled by setting ltrTrustMode = 1. + Use of LTR Trust mode is strongly discouraged as this mode may be deprecated in future. + LTR Per Picture mode: In this mode, client can control whether the current picture should be marked as LTR. Enable this mode by setting + ltrTrustMode = 0 and ltrMarkFrame = 1 for the picture to be marked as LTR. This is the preferred mode + for using LTR. + Note that LTRs are not supported if encoding session is configured with B-frames */ + uint32_t qpPrimeYZeroTransformBypassFlag : 1; /**< [in]: To enable lossless encode set this to 1, set QP to 0 and RC_mode to NV_ENC_PARAMS_RC_CONSTQP and profile to HIGH_444_PREDICTIVE_PROFILE. + Check support for lossless encoding using ::NV_ENC_CAPS_SUPPORT_LOSSLESS_ENCODE caps. */ + uint32_t useConstrainedIntraPred : 1; /**< [in]: Set 1 to enable constrained intra prediction. */ + uint32_t enableFillerDataInsertion : 1; /**< [in]: Set to 1 to enable insertion of filler data in the bitstream. + This flag will take effect only when CBR rate control mode is in use and both + NV_ENC_INITIALIZE_PARAMS::frameRateNum and + NV_ENC_INITIALIZE_PARAMS::frameRateDen are set to non-zero + values. Setting this field when + NV_ENC_INITIALIZE_PARAMS::enableOutputInVidmem is also set + is currently not supported and will make ::NvEncInitializeEncoder() + return an error. */ + uint32_t disableSVCPrefixNalu : 1; /**< [in]: Set to 1 to disable writing of SVC Prefix NALU preceding each slice in bitstream. + Applicable only when temporal SVC is enabled (NV_ENC_CONFIG_H264::enableTemporalSVC = 1). */ + uint32_t enableScalabilityInfoSEI : 1; /**< [in]: Set to 1 to enable writing of Scalability Information SEI message preceding each IDR picture in bitstream + Applicable only when temporal SVC is enabled (NV_ENC_CONFIG_H264::enableTemporalSVC = 1). */ + uint32_t singleSliceIntraRefresh : 1; /**< [in]: Set to 1 to maintain single slice in frames during intra refresh. + Check support for single slice intra refresh using ::NV_ENC_CAPS_SINGLE_SLICE_INTRA_REFRESH caps. + This flag will be ignored if the value returned for ::NV_ENC_CAPS_SINGLE_SLICE_INTRA_REFRESH caps is false. */ + uint32_t enableTimeCode : 1; /**< [in]: Set to 1 to enable writing of clock timestamp sets in picture timing SEI. Note that this flag will be ignored for D3D12 interface. */ + uint32_t reservedBitFields : 10; /**< [in]: Reserved bitfields and must be set to 0 */ + uint32_t level; /**< [in]: Specifies the encoding level. Client is recommended to set this to NV_ENC_LEVEL_AUTOSELECT in order to enable the NvEncodeAPI interface to select the correct level. */ + uint32_t idrPeriod; /**< [in]: Specifies the IDR interval. If not set, this is made equal to gopLength in NV_ENC_CONFIG.Low latency application client can set IDR interval to NVENC_INFINITE_GOPLENGTH so that IDR frames are not inserted automatically. */ + uint32_t separateColourPlaneFlag; /**< [in]: Set to 1 to enable 4:4:4 separate colour planes */ + uint32_t disableDeblockingFilterIDC; /**< [in]: Specifies the deblocking filter mode. Permissible value range: [0,2]. This flag corresponds + to the flag disable_deblocking_filter_idc specified in section 7.4.3 of H.264 specification, + which specifies whether the operation of the deblocking filter shall be disabled across some + block edges of the slice and specifies for which edges the filtering is disabled. See section + 7.4.3 of H.264 specification for more details.*/ + uint32_t numTemporalLayers; /**< [in]: Specifies number of temporal layers to be used for hierarchical coding / temporal SVC. Valid value range is [1,::NV_ENC_CAPS_NUM_MAX_TEMPORAL_LAYERS] */ + uint32_t spsId; /**< [in]: Specifies the SPS id of the sequence header */ + uint32_t ppsId; /**< [in]: Specifies the PPS id of the picture header */ + NV_ENC_H264_ADAPTIVE_TRANSFORM_MODE adaptiveTransformMode; /**< [in]: Specifies the AdaptiveTransform Mode. Check support for AdaptiveTransform mode using ::NV_ENC_CAPS_SUPPORT_ADAPTIVE_TRANSFORM caps. */ + NV_ENC_H264_FMO_MODE fmoMode; /**< [in]: Specified the FMO Mode. Check support for FMO using ::NV_ENC_CAPS_SUPPORT_FMO caps. */ + NV_ENC_H264_BDIRECT_MODE bdirectMode; /**< [in]: Specifies the BDirect mode. Check support for BDirect mode using ::NV_ENC_CAPS_SUPPORT_BDIRECT_MODE caps.*/ + NV_ENC_H264_ENTROPY_CODING_MODE entropyCodingMode; /**< [in]: Specifies the entropy coding mode. Check support for CABAC mode using ::NV_ENC_CAPS_SUPPORT_CABAC caps. */ + NV_ENC_STEREO_PACKING_MODE stereoMode; /**< [in]: Specifies the stereo frame packing mode which is to be signaled in frame packing arrangement SEI */ + uint32_t intraRefreshPeriod; /**< [in]: Specifies the interval between successive intra refresh if enableIntrarefresh is set. Requires enableIntraRefresh to be set. + Will be disabled if NV_ENC_CONFIG::gopLength is not set to NVENC_INFINITE_GOPLENGTH. */ + uint32_t intraRefreshCnt; /**< [in]: Specifies the length of intra refresh in number of frames for periodic intra refresh. This value should be smaller than intraRefreshPeriod */ + uint32_t maxNumRefFrames; /**< [in]: Specifies the DPB size used for encoding. Setting it to 0 will let driver use the default DPB size. + The low latency application which wants to invalidate reference frame as an error resilience tool + is recommended to use a large DPB size so that the encoder can keep old reference frames which can be used if recent + frames are invalidated. */ + uint32_t sliceMode; /**< [in]: This parameter in conjunction with sliceModeData specifies the way in which the picture is divided into slices + sliceMode = 0 MB based slices, sliceMode = 1 Byte based slices, sliceMode = 2 MB row based slices, sliceMode = 3 numSlices in Picture. + When forceIntraRefreshWithFrameCnt is set it will have priority over sliceMode setting + When sliceMode == 0 and sliceModeData == 0 whole picture will be coded with one slice */ + uint32_t sliceModeData; /**< [in]: Specifies the parameter needed for sliceMode. For: + sliceMode = 0, sliceModeData specifies # of MBs in each slice (except last slice) + sliceMode = 1, sliceModeData specifies maximum # of bytes in each slice (except last slice) + sliceMode = 2, sliceModeData specifies # of MB rows in each slice (except last slice) + sliceMode = 3, sliceModeData specifies number of slices in the picture. Driver will divide picture into slices optimally */ + NV_ENC_CONFIG_H264_VUI_PARAMETERS h264VUIParameters; /**< [in]: Specifies the H264 video usability info parameters */ + uint32_t ltrNumFrames; /**< [in]: Specifies the number of LTR frames. This parameter has different meaning in two LTR modes. + In "LTR Trust" mode (ltrTrustMode = 1), encoder will mark the first ltrNumFrames base layer reference frames within each IDR interval as LTR. + In "LTR Per Picture" mode (ltrTrustMode = 0 and ltrMarkFrame = 1), ltrNumFrames specifies maximum number of LTR frames in DPB. */ + uint32_t ltrTrustMode; /**< [in]: Specifies the LTR operating mode. See comments near NV_ENC_CONFIG_H264::enableLTR for description of the two modes. + Set to 1 to use "LTR Trust" mode of LTR operation. Clients are discouraged to use "LTR Trust" mode as this mode may + be deprecated in future releases. + Set to 0 when using "LTR Per Picture" mode of LTR operation. */ + uint32_t chromaFormatIDC; /**< [in]: Specifies the chroma format. Should be set to 1 for yuv420 input, 3 for yuv444 input. + Check support for YUV444 encoding using ::NV_ENC_CAPS_SUPPORT_YUV444_ENCODE caps.*/ + uint32_t maxTemporalLayers; /**< [in]: Specifies the max temporal layer used for temporal SVC / hierarchical coding. + Defaut value of this field is NV_ENC_CAPS::NV_ENC_CAPS_NUM_MAX_TEMPORAL_LAYERS. Note that the value NV_ENC_CONFIG_H264::maxNumRefFrames should + be greater than or equal to (NV_ENC_CONFIG_H264::maxTemporalLayers - 2) * 2, for NV_ENC_CONFIG_H264::maxTemporalLayers >= 2.*/ + NV_ENC_BFRAME_REF_MODE useBFramesAsRef; /**< [in]: Specifies the B-Frame as reference mode. Check support for useBFramesAsRef mode using ::NV_ENC_CAPS_SUPPORT_BFRAME_REF_MODE caps.*/ + NV_ENC_NUM_REF_FRAMES numRefL0; /**< [in]: Specifies max number of reference frames in reference picture list L0, that can be used by hardware for prediction of a frame. + Check support for numRefL0 using ::NV_ENC_CAPS_SUPPORT_MULTIPLE_REF_FRAMES caps. */ + NV_ENC_NUM_REF_FRAMES numRefL1; /**< [in]: Specifies max number of reference frames in reference picture list L1, that can be used by hardware for prediction of a frame. + Check support for numRefL1 using ::NV_ENC_CAPS_SUPPORT_MULTIPLE_REF_FRAMES caps. */ + + uint32_t reserved1[267]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_CONFIG_H264; + +/** + * \struct _NV_ENC_CONFIG_HEVC + * HEVC encoder configuration parameters to be set during initialization. + */ +typedef struct _NV_ENC_CONFIG_HEVC +{ + uint32_t level; /**< [in]: Specifies the level of the encoded bitstream.*/ + uint32_t tier; /**< [in]: Specifies the level tier of the encoded bitstream.*/ + NV_ENC_HEVC_CUSIZE minCUSize; /**< [in]: Specifies the minimum size of luma coding unit.*/ + NV_ENC_HEVC_CUSIZE maxCUSize; /**< [in]: Specifies the maximum size of luma coding unit. Currently NVENC SDK only supports maxCUSize equal to NV_ENC_HEVC_CUSIZE_32x32.*/ + uint32_t useConstrainedIntraPred : 1; /**< [in]: Set 1 to enable constrained intra prediction. */ + uint32_t disableDeblockAcrossSliceBoundary : 1; /**< [in]: Set 1 to disable in loop filtering across slice boundary.*/ + uint32_t outputBufferingPeriodSEI : 1; /**< [in]: Set 1 to write SEI buffering period syntax in the bitstream */ + uint32_t outputPictureTimingSEI : 1; /**< [in]: Set 1 to write SEI picture timing syntax in the bitstream */ + uint32_t outputAUD : 1; /**< [in]: Set 1 to write Access Unit Delimiter syntax. */ + uint32_t enableLTR : 1; /**< [in]: Set to 1 to enable LTR (Long Term Reference) frame support. LTR can be used in two modes: "LTR Trust" mode and "LTR Per Picture" mode. + LTR Trust mode: In this mode, ltrNumFrames pictures after IDR are automatically marked as LTR. This mode is enabled by setting ltrTrustMode = 1. + Use of LTR Trust mode is strongly discouraged as this mode may be deprecated in future releases. + LTR Per Picture mode: In this mode, client can control whether the current picture should be marked as LTR. Enable this mode by setting + ltrTrustMode = 0 and ltrMarkFrame = 1 for the picture to be marked as LTR. This is the preferred mode + for using LTR. + Note that LTRs are not supported if encoding session is configured with B-frames */ + uint32_t disableSPSPPS : 1; /**< [in]: Set 1 to disable VPS, SPS and PPS signaling in the bitstream. */ + uint32_t repeatSPSPPS : 1; /**< [in]: Set 1 to output VPS,SPS and PPS for every IDR frame.*/ + uint32_t enableIntraRefresh : 1; /**< [in]: Set 1 to enable gradual decoder refresh or intra refresh. If the GOP structure uses B frames this will be ignored */ + uint32_t chromaFormatIDC : 2; /**< [in]: Specifies the chroma format. Should be set to 1 for yuv420 input, 3 for yuv444 input.*/ + uint32_t pixelBitDepthMinus8 : 3; /**< [in]: Specifies pixel bit depth minus 8. Should be set to 0 for 8 bit input, 2 for 10 bit input.*/ + uint32_t enableFillerDataInsertion : 1; /**< [in]: Set to 1 to enable insertion of filler data in the bitstream. + This flag will take effect only when CBR rate control mode is in use and both + NV_ENC_INITIALIZE_PARAMS::frameRateNum and + NV_ENC_INITIALIZE_PARAMS::frameRateDen are set to non-zero + values. Setting this field when + NV_ENC_INITIALIZE_PARAMS::enableOutputInVidmem is also set + is currently not supported and will make ::NvEncInitializeEncoder() + return an error. */ + uint32_t enableConstrainedEncoding : 1; /**< [in]: Set this to 1 to enable constrainedFrame encoding where each slice in the constrained picture is independent of other slices. + Constrained encoding works only with rectangular slices. + Check support for constrained encoding using ::NV_ENC_CAPS_SUPPORT_CONSTRAINED_ENCODING caps. */ + uint32_t enableAlphaLayerEncoding : 1; /**< [in]: Set this to 1 to enable HEVC encode with alpha layer. */ + uint32_t singleSliceIntraRefresh : 1; /**< [in]: Set this to 1 to maintain single slice frames during intra refresh. + Check support for single slice intra refresh using ::NV_ENC_CAPS_SINGLE_SLICE_INTRA_REFRESH caps. + This flag will be ignored if the value returned for ::NV_ENC_CAPS_SINGLE_SLICE_INTRA_REFRESH caps is false. */ + uint32_t outputRecoveryPointSEI : 1; /**< [in]: Set to 1 to enable writing of recovery point SEI message */ + uint32_t outputTimeCodeSEI : 1; /**< [in]: Set 1 to write SEI time code syntax in the bitstream. Note that this flag will be ignored for D3D12 interface.*/ + uint32_t reserved : 12; /**< [in]: Reserved bitfields.*/ + uint32_t idrPeriod; /**< [in]: Specifies the IDR interval. If not set, this is made equal to gopLength in NV_ENC_CONFIG. Low latency application client can set IDR interval to NVENC_INFINITE_GOPLENGTH so that IDR frames are not inserted automatically. */ + uint32_t intraRefreshPeriod; /**< [in]: Specifies the interval between successive intra refresh if enableIntrarefresh is set. Requires enableIntraRefresh to be set. + Will be disabled if NV_ENC_CONFIG::gopLength is not set to NVENC_INFINITE_GOPLENGTH. */ + uint32_t intraRefreshCnt; /**< [in]: Specifies the length of intra refresh in number of frames for periodic intra refresh. This value should be smaller than intraRefreshPeriod */ + uint32_t maxNumRefFramesInDPB; /**< [in]: Specifies the maximum number of references frames in the DPB.*/ + uint32_t ltrNumFrames; /**< [in]: This parameter has different meaning in two LTR modes. + In "LTR Trust" mode (ltrTrustMode = 1), encoder will mark the first ltrNumFrames base layer reference frames within each IDR interval as LTR. + In "LTR Per Picture" mode (ltrTrustMode = 0 and ltrMarkFrame = 1), ltrNumFrames specifies maximum number of LTR frames in DPB. + These ltrNumFrames acts as a guidance to the encoder and are not necessarily honored. To achieve a right balance between the encoding + quality and keeping LTR frames in the DPB queue, the encoder can internally limit the number of LTR frames. + The number of LTR frames actually used depends upon the encoding preset being used; Faster encoding presets will use fewer LTR frames.*/ + uint32_t vpsId; /**< [in]: Specifies the VPS id of the video parameter set */ + uint32_t spsId; /**< [in]: Specifies the SPS id of the sequence header */ + uint32_t ppsId; /**< [in]: Specifies the PPS id of the picture header */ + uint32_t sliceMode; /**< [in]: This parameter in conjunction with sliceModeData specifies the way in which the picture is divided into slices + sliceMode = 0 CTU based slices, sliceMode = 1 Byte based slices, sliceMode = 2 CTU row based slices, sliceMode = 3, numSlices in Picture + When sliceMode == 0 and sliceModeData == 0 whole picture will be coded with one slice */ + uint32_t sliceModeData; /**< [in]: Specifies the parameter needed for sliceMode. For: + sliceMode = 0, sliceModeData specifies # of CTUs in each slice (except last slice) + sliceMode = 1, sliceModeData specifies maximum # of bytes in each slice (except last slice) + sliceMode = 2, sliceModeData specifies # of CTU rows in each slice (except last slice) + sliceMode = 3, sliceModeData specifies number of slices in the picture. Driver will divide picture into slices optimally */ + uint32_t maxTemporalLayersMinus1; /**< [in]: Specifies the max temporal layer used for hierarchical coding. */ + NV_ENC_CONFIG_HEVC_VUI_PARAMETERS hevcVUIParameters; /**< [in]: Specifies the HEVC video usability info parameters */ + uint32_t ltrTrustMode; /**< [in]: Specifies the LTR operating mode. See comments near NV_ENC_CONFIG_HEVC::enableLTR for description of the two modes. + Set to 1 to use "LTR Trust" mode of LTR operation. Clients are discouraged to use "LTR Trust" mode as this mode may + be deprecated in future releases. + Set to 0 when using "LTR Per Picture" mode of LTR operation. */ + NV_ENC_BFRAME_REF_MODE useBFramesAsRef; /**< [in]: Specifies the B-Frame as reference mode. Check support for useBFramesAsRef mode using ::NV_ENC_CAPS_SUPPORT_BFRAME_REF_MODE caps.*/ + NV_ENC_NUM_REF_FRAMES numRefL0; /**< [in]: Specifies max number of reference frames in reference picture list L0, that can be used by hardware for prediction of a frame. + Check support for numRefL0 using ::NV_ENC_CAPS_SUPPORT_MULTIPLE_REF_FRAMES caps. */ + NV_ENC_NUM_REF_FRAMES numRefL1; /**< [in]: Specifies max number of reference frames in reference picture list L1, that can be used by hardware for prediction of a frame. + Check support for numRefL1 using ::NV_ENC_CAPS_SUPPORT_MULTIPLE_REF_FRAMES caps. */ + uint32_t reserved1[214]; /**< [in]: Reserved and must be set to 0.*/ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_CONFIG_HEVC; + +#define NV_MAX_TILE_COLS_AV1 64 +#define NV_MAX_TILE_ROWS_AV1 64 + +/** + * \struct _NV_ENC_FILM_GRAIN_PARAMS_AV1 + * AV1 Film Grain Parameters structure + */ + +typedef struct _NV_ENC_FILM_GRAIN_PARAMS_AV1 +{ + uint32_t applyGrain : 1; /**< [in]: Set to 1 to specify film grain should be added to frame */ + uint32_t chromaScalingFromLuma : 1; /**< [in]: Set to 1 to specify the chroma scaling is inferred from luma scaling */ + uint32_t overlapFlag : 1; /**< [in]: Set to 1 to indicate that overlap between film grain blocks should be applied*/ + uint32_t clipToRestrictedRange : 1; /**< [in]: Set to 1 to clip values to restricted (studio) range after adding film grain */ + uint32_t grainScalingMinus8 : 2; /**< [in]: Represents the shift - 8 applied to the values of the chroma component */ + uint32_t arCoeffLag : 2; /**< [in]: Specifies the number of auto-regressive coefficients for luma and chroma */ + uint32_t numYPoints : 4; /**< [in]: Specifies the number of points for the piecewise linear scaling function of the luma component */ + uint32_t numCbPoints : 4; /**< [in]: Specifies the number of points for the piecewise linear scaling function of the cb component */ + uint32_t numCrPoints : 4; /**< [in]: Specifies the number of points for the piecewise linear scaling function of the cr component */ + uint32_t arCoeffShiftMinus6 : 2; /**< [in]: specifies the range of the auto-regressive coefficients */ + uint32_t grainScaleShift : 2; /**< [in]: Specifies how much the Gaussian random numbers should be scaled down during the grain synthesi process */ + uint32_t reserved1 : 8; /**< [in]: Reserved bits field - should be set to 0 */ + uint8_t pointYValue[14]; /**< [in]: pointYValue[i]: x coordinate for i-th point of luma piecewise linear scaling function. Values on a scale of 0...255 */ + uint8_t pointYScaling[14]; /**< [in]: pointYScaling[i]: i-th point output value of luma piecewise linear scaling function */ + uint8_t pointCbValue[10]; /**< [in]: pointCbValue[i]: x coordinate for i-th point of cb piecewise linear scaling function. Values on a scale of 0...255 */ + uint8_t pointCbScaling[10]; /**< [in]: pointCbScaling[i]: i-th point output value of cb piecewise linear scaling function */ + uint8_t pointCrValue[10]; /**< [in]: pointCrValue[i]: x coordinate for i-th point of cr piecewise linear scaling function. Values on a scale of 0...255 */ + uint8_t pointCrScaling[10]; /**< [in]: pointCrScaling[i]: i-th point output value of cr piecewise linear scaling function */ + uint8_t arCoeffsYPlus128[24]; /**< [in]: Specifies auto-regressive coefficients used for the Y plane */ + uint8_t arCoeffsCbPlus128[25]; /**< [in]: Specifies auto-regressive coefficients used for the U plane */ + uint8_t arCoeffsCrPlus128[25]; /**< [in]: Specifies auto-regressive coefficients used for the V plane */ + uint8_t reserved2[2]; /**< [in]: Reserved bytes - should be set to 0 */ + uint8_t cbMult; /**< [in]: Represents a multiplier for the cb component used in derivation of the input index to the cb component scaling function */ + uint8_t cbLumaMult; /**< [in]: represents a multiplier for the average luma component used in derivation of the input index to the cb component scaling function. */ + uint16_t cbOffset; /**< [in]: Represents an offset used in derivation of the input index to the cb component scaling function */ + uint8_t crMult; /**< [in]: Represents a multiplier for the cr component used in derivation of the input index to the cr component scaling function */ + uint8_t crLumaMult; /**< [in]: represents a multiplier for the average luma component used in derivation of the input index to the cr component scaling function. */ + uint16_t crOffset; /**< [in]: Represents an offset used in derivation of the input index to the cr component scaling function */ +} NV_ENC_FILM_GRAIN_PARAMS_AV1; + +/** +* \struct _NV_ENC_CONFIG_AV1 +* AV1 encoder configuration parameters to be set during initialization. +*/ +typedef struct _NV_ENC_CONFIG_AV1 +{ + uint32_t level; /**< [in]: Specifies the level of the encoded bitstream.*/ + uint32_t tier; /**< [in]: Specifies the level tier of the encoded bitstream.*/ + NV_ENC_AV1_PART_SIZE minPartSize; /**< [in]: Specifies the minimum size of luma coding block partition.*/ + NV_ENC_AV1_PART_SIZE maxPartSize; /**< [in]: Specifies the maximum size of luma coding block partition.*/ + uint32_t outputAnnexBFormat : 1; /**< [in]: Set 1 to use Annex B format for bitstream output.*/ + uint32_t enableTimingInfo : 1; /**< [in]: Set 1 to write Timing Info into sequence/frame headers */ + uint32_t enableDecoderModelInfo : 1; /**< [in]: Set 1 to write Decoder Model Info into sequence/frame headers */ + uint32_t enableFrameIdNumbers : 1; /**< [in]: Set 1 to write Frame id numbers in bitstream */ + uint32_t disableSeqHdr : 1; /**< [in]: Set 1 to disable Sequence Header signaling in the bitstream. */ + uint32_t repeatSeqHdr : 1; /**< [in]: Set 1 to output Sequence Header for every Key frame.*/ + uint32_t enableIntraRefresh : 1; /**< [in]: Set 1 to enable gradual decoder refresh or intra refresh. If the GOP structure uses B frames this will be ignored */ + uint32_t chromaFormatIDC : 2; /**< [in]: Specifies the chroma format. Should be set to 1 for yuv420 input (yuv444 input currently not supported).*/ + uint32_t enableBitstreamPadding : 1; /**< [in]: Set 1 to enable bitstream padding. */ + uint32_t enableCustomTileConfig : 1; /**< [in]: Set 1 to enable custom tile configuration: numTileColumns and numTileRows must have non zero values and tileWidths and tileHeights must point to a valid address */ + uint32_t enableFilmGrainParams : 1; /**< [in]: Set 1 to enable custom film grain parameters: filmGrainParams must point to a valid address */ + uint32_t inputPixelBitDepthMinus8 : 3; /**< [in]: Specifies pixel bit depth minus 8 of video input. Should be set to 0 for 8 bit input, 2 for 10 bit input.*/ + uint32_t pixelBitDepthMinus8 : 3; /**< [in]: Specifies pixel bit depth minus 8 of encoded video. Should be set to 0 for 8 bit, 2 for 10 bit. + HW will do the bitdepth conversion internally from inputPixelBitDepthMinus8 -> pixelBitDepthMinus8 if bit dpeths differ + Support for 8 bit input to 10 bit encode conversion only */ + uint32_t reserved : 14; /**< [in]: Reserved bitfields.*/ + uint32_t idrPeriod; /**< [in]: Specifies the IDR/Key frame interval. If not set, this is made equal to gopLength in NV_ENC_CONFIG.Low latency application client can set IDR interval to NVENC_INFINITE_GOPLENGTH so that IDR frames are not inserted automatically. */ + uint32_t intraRefreshPeriod; /**< [in]: Specifies the interval between successive intra refresh if enableIntrarefresh is set. Requires enableIntraRefresh to be set. + Will be disabled if NV_ENC_CONFIG::gopLength is not set to NVENC_INFINITE_GOPLENGTH. */ + uint32_t intraRefreshCnt; /**< [in]: Specifies the length of intra refresh in number of frames for periodic intra refresh. This value should be smaller than intraRefreshPeriod */ + uint32_t maxNumRefFramesInDPB; /**< [in]: Specifies the maximum number of references frames in the DPB.*/ + uint32_t numTileColumns; /**< [in]: This parameter in conjunction with the flag enableCustomTileConfig and the array tileWidths[] specifies the way in which the picture is divided into tile columns. + When enableCustomTileConfig == 0, the picture will be uniformly divided into numTileColumns tile columns. If numTileColumns is not a power of 2, + it will be rounded down to the next power of 2 value. If numTileColumns == 0, the picture will be coded with the smallest number of vertical tiles as allowed by standard. + When enableCustomTileConfig == 1, numTileColumns must be > 0 and <= NV_MAX_TILE_COLS_AV1 and tileWidths must point to a valid array of numTileColumns entries. + Entry i specifies the width in 64x64 CTU unit of tile colum i. The sum of all the entries should be equal to the picture width in 64x64 CTU units. */ + uint32_t numTileRows; /**< [in]: This parameter in conjunction with the flag enableCustomTileConfig and the array tileHeights[] specifies the way in which the picture is divided into tiles rows + When enableCustomTileConfig == 0, the picture will be uniformly divided into numTileRows tile rows. If numTileRows is not a power of 2, + it will be rounded down to the next power of 2 value. If numTileRows == 0, the picture will be coded with the smallest number of horizontal tiles as allowed by standard. + When enableCustomTileConfig == 1, numTileRows must be > 0 and <= NV_MAX_TILE_ROWS_AV1 and tileHeights must point to a valid array of numTileRows entries. + Entry i specifies the height in 64x64 CTU unit of tile row i. The sum of all the entries should be equal to the picture hieght in 64x64 CTU units. */ + uint32_t *tileWidths; /**< [in]: If enableCustomTileConfig == 1, tileWidths[i] specifies the width of tile column i in 64x64 CTU unit, with 0 <= i <= numTileColumns -1. */ + uint32_t *tileHeights; /**< [in]: If enableCustomTileConfig == 1, tileHeights[i] specifies the height of tile row i in 64x64 CTU unit, with 0 <= i <= numTileRows -1. */ + uint32_t maxTemporalLayersMinus1; /**< [in]: Specifies the max temporal layer used for hierarchical coding. */ + NV_ENC_VUI_COLOR_PRIMARIES colorPrimaries; /**< [in]: as defined in section of ISO/IEC 23091-4/ITU-T H.273 */ + NV_ENC_VUI_TRANSFER_CHARACTERISTIC transferCharacteristics; /**< [in]: as defined in section of ISO/IEC 23091-4/ITU-T H.273 */ + NV_ENC_VUI_MATRIX_COEFFS matrixCoefficients; /**< [in]: as defined in section of ISO/IEC 23091-4/ITU-T H.273 */ + uint32_t colorRange; /**< [in]: 0: studio swing representation - 1: full swing representation */ + uint32_t chromaSamplePosition; /**< [in]: 0: unknown + 1: Horizontally collocated with luma (0,0) sample, between two vertical samples + 2: Co-located with luma (0,0) sample */ + NV_ENC_BFRAME_REF_MODE useBFramesAsRef; /**< [in]: Specifies the B-Frame as reference mode. Check support for useBFramesAsRef mode using ::NV_ENC_CAPS_SUPPORT_BFRAME_REF_MODE caps.*/ + NV_ENC_FILM_GRAIN_PARAMS_AV1 *filmGrainParams; /**< [in]: If enableFilmGrainParams == 1, filmGrainParams must point to a valid NV_ENC_FILM_GRAIN_PARAMS_AV1 structure */ + NV_ENC_NUM_REF_FRAMES numFwdRefs; /**< [in]: Specifies max number of forward reference frame used for prediction of a frame. It must be in range 1-4 (Last, Last2, last3 and Golden). It's a suggestive value not necessarily be honored always. */ + NV_ENC_NUM_REF_FRAMES numBwdRefs; /**< [in]: Specifies max number of L1 list reference frame used for prediction of a frame. It must be in range 1-3 (Backward, Altref2, Altref). It's a suggestive value not necessarily be honored always. */ + uint32_t reserved1[235]; /**< [in]: Reserved and must be set to 0.*/ + void *reserved2[62]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_CONFIG_AV1; + +/** + * \struct _NV_ENC_CONFIG_H264_MEONLY + * H264 encoder configuration parameters for ME only Mode + * + */ +typedef struct _NV_ENC_CONFIG_H264_MEONLY +{ + uint32_t disablePartition16x16 : 1; /**< [in]: Disable Motion Estimation on 16x16 blocks*/ + uint32_t disablePartition8x16 : 1; /**< [in]: Disable Motion Estimation on 8x16 blocks*/ + uint32_t disablePartition16x8 : 1; /**< [in]: Disable Motion Estimation on 16x8 blocks*/ + uint32_t disablePartition8x8 : 1; /**< [in]: Disable Motion Estimation on 8x8 blocks*/ + uint32_t disableIntraSearch : 1; /**< [in]: Disable Intra search during Motion Estimation*/ + uint32_t bStereoEnable : 1; /**< [in]: Enable Stereo Mode for Motion Estimation where each view is independently executed*/ + uint32_t reserved : 26; /**< [in]: Reserved and must be set to 0 */ + uint32_t reserved1 [255]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_CONFIG_H264_MEONLY; + + +/** + * \struct _NV_ENC_CONFIG_HEVC_MEONLY + * HEVC encoder configuration parameters for ME only Mode + * + */ +typedef struct _NV_ENC_CONFIG_HEVC_MEONLY +{ + uint32_t reserved [256]; /**< [in]: Reserved and must be set to 0 */ + void *reserved1[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_CONFIG_HEVC_MEONLY; + +/** + * \struct _NV_ENC_CODEC_CONFIG + * Codec-specific encoder configuration parameters to be set during initialization. + */ +typedef union _NV_ENC_CODEC_CONFIG +{ + NV_ENC_CONFIG_H264 h264Config; /**< [in]: Specifies the H.264-specific encoder configuration. */ + NV_ENC_CONFIG_HEVC hevcConfig; /**< [in]: Specifies the HEVC-specific encoder configuration. */ + NV_ENC_CONFIG_AV1 av1Config; /**< [in]: Specifies the AV1-specific encoder configuration. */ + NV_ENC_CONFIG_H264_MEONLY h264MeOnlyConfig; /**< [in]: Specifies the H.264-specific ME only encoder configuration. */ + NV_ENC_CONFIG_HEVC_MEONLY hevcMeOnlyConfig; /**< [in]: Specifies the HEVC-specific ME only encoder configuration. */ + uint32_t reserved[320]; /**< [in]: Reserved and must be set to 0 */ +} NV_ENC_CODEC_CONFIG; + + +/** + * \struct _NV_ENC_CONFIG + * Encoder configuration parameters to be set during initialization. + */ +typedef struct _NV_ENC_CONFIG +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_CONFIG_VER. */ + GUID profileGUID; /**< [in]: Specifies the codec profile GUID. If client specifies \p NV_ENC_CODEC_PROFILE_AUTOSELECT_GUID the NvEncodeAPI interface will select the appropriate codec profile. */ + uint32_t gopLength; /**< [in]: Specifies the number of pictures in one GOP. Low latency application client can set goplength to NVENC_INFINITE_GOPLENGTH so that keyframes are not inserted automatically. */ + int32_t frameIntervalP; /**< [in]: Specifies the GOP pattern as follows: \p frameIntervalP = 0: I, 1: IPP, 2: IBP, 3: IBBP If goplength is set to NVENC_INFINITE_GOPLENGTH \p frameIntervalP should be set to 1. */ + uint32_t monoChromeEncoding; /**< [in]: Set this to 1 to enable monochrome encoding for this session. */ + NV_ENC_PARAMS_FRAME_FIELD_MODE frameFieldMode; /**< [in]: Specifies the frame/field mode. + Check support for field encoding using ::NV_ENC_CAPS_SUPPORT_FIELD_ENCODING caps. + Using a frameFieldMode other than NV_ENC_PARAMS_FRAME_FIELD_MODE_FRAME for RGB input is not supported. */ + NV_ENC_MV_PRECISION mvPrecision; /**< [in]: Specifies the desired motion vector prediction precision. */ + NV_ENC_RC_PARAMS rcParams; /**< [in]: Specifies the rate control parameters for the current encoding session. */ + NV_ENC_CODEC_CONFIG encodeCodecConfig; /**< [in]: Specifies the codec specific config parameters through this union. */ + uint32_t reserved [278]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_CONFIG; + +/** macro for constructing the version field of ::_NV_ENC_CONFIG */ +#define NV_ENC_CONFIG_VER (NVENCAPI_STRUCT_VERSION(8) | ( 1U<<31 )) + +/** + * Tuning information of NVENC encoding (TuningInfo is not applicable to H264 and HEVC MEOnly mode). + */ +typedef enum NV_ENC_TUNING_INFO +{ + NV_ENC_TUNING_INFO_UNDEFINED = 0, /**< Undefined tuningInfo. Invalid value for encoding. */ + NV_ENC_TUNING_INFO_HIGH_QUALITY = 1, /**< Tune presets for latency tolerant encoding.*/ + NV_ENC_TUNING_INFO_LOW_LATENCY = 2, /**< Tune presets for low latency streaming.*/ + NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY = 3, /**< Tune presets for ultra low latency streaming.*/ + NV_ENC_TUNING_INFO_LOSSLESS = 4, /**< Tune presets for lossless encoding.*/ + NV_ENC_TUNING_INFO_COUNT /**< Count number of tuningInfos. Invalid value. */ +} NV_ENC_TUNING_INFO; + +/** + * Split Encoding Modes (Split Encoding is not applicable to H264). + */ +typedef enum _NV_ENC_SPLIT_ENCODE_MODE +{ + NV_ENC_SPLIT_AUTO_MODE = 0, /**< Default value, split frame forced mode disabled, split frame auto mode enabled */ + NV_ENC_SPLIT_AUTO_FORCED_MODE = 1, /**< Split frame forced mode enabled with number of strips automatically selected by driver to best fit configuration */ + NV_ENC_SPLIT_TWO_FORCED_MODE = 2, /**< Forced 2-strip split frame encoding (if NVENC number > 1, 1-strip encode otherwise) */ + NV_ENC_SPLIT_THREE_FORCED_MODE = 3, /**< Forced 3-strip split frame encoding (if NVENC number > 2, NVENC number of strips otherwise) */ + NV_ENC_SPLIT_DISABLE_MODE = 15, /**< Both split frame auto mode and forced mode are disabled */ +} NV_ENC_SPLIT_ENCODE_MODE; + +/** + * \struct _NV_ENC_INITIALIZE_PARAMS + * Encode Session Initialization parameters. + */ +typedef struct _NV_ENC_INITIALIZE_PARAMS +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_INITIALIZE_PARAMS_VER. */ + GUID encodeGUID; /**< [in]: Specifies the Encode GUID for which the encoder is being created. ::NvEncInitializeEncoder() API will fail if this is not set, or set to unsupported value. */ + GUID presetGUID; /**< [in]: Specifies the preset for encoding. If the preset GUID is set then , the preset configuration will be applied before any other parameter. */ + uint32_t encodeWidth; /**< [in]: Specifies the encode width. If not set ::NvEncInitializeEncoder() API will fail. */ + uint32_t encodeHeight; /**< [in]: Specifies the encode height. If not set ::NvEncInitializeEncoder() API will fail. */ + uint32_t darWidth; /**< [in]: Specifies the display aspect ratio width (H264/HEVC) or the render width (AV1). */ + uint32_t darHeight; /**< [in]: Specifies the display aspect ratio height (H264/HEVC) or the render height (AV1). */ + uint32_t frameRateNum; /**< [in]: Specifies the numerator for frame rate used for encoding in frames per second ( Frame rate = frameRateNum / frameRateDen ). */ + uint32_t frameRateDen; /**< [in]: Specifies the denominator for frame rate used for encoding in frames per second ( Frame rate = frameRateNum / frameRateDen ). */ + uint32_t enableEncodeAsync; /**< [in]: Set this to 1 to enable asynchronous mode and is expected to use events to get picture completion notification. */ + uint32_t enablePTD; /**< [in]: Set this to 1 to enable the Picture Type Decision is be taken by the NvEncodeAPI interface. */ + uint32_t reportSliceOffsets : 1; /**< [in]: Set this to 1 to enable reporting slice offsets in ::_NV_ENC_LOCK_BITSTREAM. NV_ENC_INITIALIZE_PARAMS::enableEncodeAsync must be set to 0 to use this feature. Client must set this to 0 if NV_ENC_CONFIG_H264::sliceMode is 1 on Kepler GPUs */ + uint32_t enableSubFrameWrite : 1; /**< [in]: Set this to 1 to write out available bitstream to memory at subframe intervals. + If enableSubFrameWrite = 1, then the hardware encoder returns data as soon as a slice (H264/HEVC) or tile (AV1) has completed encoding. + This results in better encoding latency, but the downside is that the application has to keep polling via a call to nvEncLockBitstream API continuously to see if any encoded slice/tile data is available. + Use this mode if you feel that the marginal reduction in latency from sub-frame encoding is worth the increase in complexity due to CPU-based polling. */ + uint32_t enableExternalMEHints : 1; /**< [in]: Set to 1 to enable external ME hints for the current frame. For NV_ENC_INITIALIZE_PARAMS::enablePTD=1 with B frames, programming L1 hints is optional for B frames since Client doesn't know internal GOP structure. + NV_ENC_PIC_PARAMS::meHintRefPicDist should preferably be set with enablePTD=1. */ + uint32_t enableMEOnlyMode : 1; /**< [in]: Set to 1 to enable ME Only Mode .*/ + uint32_t enableWeightedPrediction : 1; /**< [in]: Set this to 1 to enable weighted prediction. Not supported if encode session is configured for B-Frames (i.e. NV_ENC_CONFIG::frameIntervalP > 1 or preset >=P3 when tuningInfo = ::NV_ENC_TUNING_INFO_HIGH_QUALITY or + tuningInfo = ::NV_ENC_TUNING_INFO_LOSSLESS. This is because preset >=p3 internally enables B frames when tuningInfo = ::NV_ENC_TUNING_INFO_HIGH_QUALITY or ::NV_ENC_TUNING_INFO_LOSSLESS). */ + uint32_t splitEncodeMode : 4; /**< [in]: Split Encoding mode in NVENC (Split Encoding is not applicable to H264). + Not supported if any of the following features: weighted prediction, alpha layer encoding, + subframe mode, output into video memory buffer, picture timing/buffering period SEI message + insertion with DX12 interface are enabled in case of HEVC. + For AV1, split encoding is not supported when output into video memory buffer is enabled. */ + uint32_t enableOutputInVidmem : 1; /**< [in]: Set this to 1 to enable output of NVENC in video memory buffer created by application. This feature is not supported for HEVC ME only mode. */ + uint32_t enableReconFrameOutput : 1; /**< [in]: Set this to 1 to enable reconstructed frame output. */ + uint32_t enableOutputStats : 1; /**< [in]: Set this to 1 to enable encoded frame output stats. Client must allocate buffer of size equal to number of blocks multiplied by the size of + NV_ENC_OUTPUT_STATS_BLOCK struct in system memory and assign to NV_ENC_LOCK_BITSTREAM::encodedOutputStatsPtr to receive the encoded frame output stats.*/ + uint32_t reservedBitFields : 20; /**< [in]: Reserved bitfields and must be set to 0 */ + uint32_t privDataSize; /**< [in]: Reserved private data buffer size and must be set to 0 */ + void *privData; /**< [in]: Reserved private data buffer and must be set to NULL */ + NV_ENC_CONFIG *encodeConfig; /**< [in]: Specifies the advanced codec specific structure. If client has sent a valid codec config structure, it will override parameters set by the NV_ENC_INITIALIZE_PARAMS::presetGUID parameter. If set to NULL the NvEncodeAPI interface will use the NV_ENC_INITIALIZE_PARAMS::presetGUID to set the codec specific parameters. + Client can also optionally query the NvEncodeAPI interface to get codec specific parameters for a presetGUID using ::NvEncGetEncodePresetConfigEx() API. It can then modify (if required) some of the codec config parameters and send down a custom config structure as part of ::_NV_ENC_INITIALIZE_PARAMS. + Even in this case client is recommended to pass the same preset guid it has used in ::NvEncGetEncodePresetConfigEx() API to query the config structure; as NV_ENC_INITIALIZE_PARAMS::presetGUID. This will not override the custom config structure but will be used to determine other Encoder HW specific parameters not exposed in the API. */ + uint32_t maxEncodeWidth; /**< [in]: Maximum encode width to be used for current Encode session. + Client should allocate output buffers according to this dimension for dynamic resolution change. If set to 0, Encoder will not allow dynamic resolution change. */ + uint32_t maxEncodeHeight; /**< [in]: Maximum encode height to be allowed for current Encode session. + Client should allocate output buffers according to this dimension for dynamic resolution change. If set to 0, Encode will not allow dynamic resolution change. */ + NVENC_EXTERNAL_ME_HINT_COUNTS_PER_BLOCKTYPE maxMEHintCountsPerBlock[2]; /**< [in]: If Client wants to pass external motion vectors in NV_ENC_PIC_PARAMS::meExternalHints buffer it must specify the maximum number of hint candidates per block per direction for the encode session. + The NV_ENC_INITIALIZE_PARAMS::maxMEHintCountsPerBlock[0] is for L0 predictors and NV_ENC_INITIALIZE_PARAMS::maxMEHintCountsPerBlock[1] is for L1 predictors. + This client must also set NV_ENC_INITIALIZE_PARAMS::enableExternalMEHints to 1. */ + NV_ENC_TUNING_INFO tuningInfo; /**< [in]: Tuning Info of NVENC encoding(TuningInfo is not applicable to H264 and HEVC meonly mode). */ + NV_ENC_BUFFER_FORMAT bufferFormat; /**< [in]: Input buffer format. Used only when DX12 interface type is used */ + uint32_t numStateBuffers; /**< [in]: Number of state buffers to allocate to save encoder state. Set this to value greater than zero to enable encoding without advancing the encoder state. */ + NV_ENC_OUTPUT_STATS_LEVEL outputStatsLevel; /**< [in]: Specifies the level for encoded frame output stats, when NV_ENC_INITIALIZE_PARAMS::enableOutputStats is set to 1. + Client should allocate buffer of size equal to number of blocks multiplied by the size of NV_ENC_OUTPUT_STATS_BLOCK struct + if NV_ENC_INITIALIZE_PARAMS::outputStatsLevel is set to NV_ENC_OUTPUT_STATS_BLOCK or number of rows multiplied by the size of + NV_ENC_OUTPUT_STATS_ROW struct if NV_ENC_INITIALIZE_PARAMS::outputStatsLevel is set to NV_ENC_OUTPUT_STATS_ROW + in system memory and assign to NV_ENC_LOCK_BITSTREAM::encodedOutputStatsPtr to receive the encoded frame output stats. */ + uint32_t reserved [285]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_INITIALIZE_PARAMS; + +/** macro for constructing the version field of ::_NV_ENC_INITIALIZE_PARAMS */ +#define NV_ENC_INITIALIZE_PARAMS_VER (NVENCAPI_STRUCT_VERSION(6) | ( 1U<<31 )) + + +/** + * \struct _NV_ENC_RECONFIGURE_PARAMS + * Encode Session Reconfigured parameters. + */ +typedef struct _NV_ENC_RECONFIGURE_PARAMS +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_RECONFIGURE_PARAMS_VER. */ + NV_ENC_INITIALIZE_PARAMS reInitEncodeParams; /**< [in]: Encoder session re-initialization parameters. + If reInitEncodeParams.encodeConfig is NULL and + reInitEncodeParams.presetGUID is the same as the preset + GUID specified on the call to NvEncInitializeEncoder(), + EncodeAPI will continue to use the existing encode + configuration. + If reInitEncodeParams.encodeConfig is NULL and + reInitEncodeParams.presetGUID is different from the preset + GUID specified on the call to NvEncInitializeEncoder(), + EncodeAPI will try to use the default configuration for + the preset specified by reInitEncodeParams.presetGUID. + In this case, reconfiguration may fail if the new + configuration is incompatible with the existing + configuration (e.g. the new configuration results in + a change in the GOP structure). */ + uint32_t resetEncoder : 1; /**< [in]: This resets the rate control states and other internal encoder states. This should be used only with an IDR frame. + If NV_ENC_INITIALIZE_PARAMS::enablePTD is set to 1, encoder will force the frame type to IDR */ + uint32_t forceIDR : 1; /**< [in]: Encode the current picture as an IDR picture. This flag is only valid when Picture type decision is taken by the Encoder + [_NV_ENC_INITIALIZE_PARAMS::enablePTD == 1]. */ + uint32_t reserved : 30; + +} NV_ENC_RECONFIGURE_PARAMS; + +/** macro for constructing the version field of ::_NV_ENC_RECONFIGURE_PARAMS */ +#define NV_ENC_RECONFIGURE_PARAMS_VER (NVENCAPI_STRUCT_VERSION(1) | ( 1<<31 )) + +/** + * \struct _NV_ENC_PRESET_CONFIG + * Encoder preset config + */ +typedef struct _NV_ENC_PRESET_CONFIG +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_PRESET_CONFIG_VER. */ + NV_ENC_CONFIG presetCfg; /**< [out]: preset config returned by the Nvidia Video Encoder interface. */ + uint32_t reserved1[255]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_PRESET_CONFIG; + +/** macro for constructing the version field of ::_NV_ENC_PRESET_CONFIG */ +#define NV_ENC_PRESET_CONFIG_VER (NVENCAPI_STRUCT_VERSION(4) | ( 1U<<31 )) + + +/** + * \struct _NV_ENC_PIC_PARAMS_MVC + * MVC-specific parameters to be sent on a per-frame basis. + */ +typedef struct _NV_ENC_PIC_PARAMS_MVC +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_PIC_PARAMS_MVC_VER. */ + uint32_t viewID; /**< [in]: Specifies the view ID associated with the current input view. */ + uint32_t temporalID; /**< [in]: Specifies the temporal ID associated with the current input view. */ + uint32_t priorityID; /**< [in]: Specifies the priority ID associated with the current input view. Reserved and ignored by the NvEncodeAPI interface. */ + uint32_t reserved1[12]; /**< [in]: Reserved and must be set to 0. */ + void *reserved2[8]; /**< [in]: Reserved and must be set to NULL. */ +} NV_ENC_PIC_PARAMS_MVC; + +/** macro for constructing the version field of ::_NV_ENC_PIC_PARAMS_MVC */ +#define NV_ENC_PIC_PARAMS_MVC_VER NVENCAPI_STRUCT_VERSION(1) + + +/** + * \union _NV_ENC_PIC_PARAMS_H264_EXT + * H264 extension picture parameters + */ +typedef union _NV_ENC_PIC_PARAMS_H264_EXT +{ + NV_ENC_PIC_PARAMS_MVC mvcPicParams; /**< [in]: Specifies the MVC picture parameters. */ + uint32_t reserved1[32]; /**< [in]: Reserved and must be set to 0. */ +} NV_ENC_PIC_PARAMS_H264_EXT; + +/** + * \struct _NV_ENC_SEI_PAYLOAD + * User SEI message + */ +typedef struct _NV_ENC_SEI_PAYLOAD +{ + uint32_t payloadSize; /**< [in] SEI payload size in bytes. SEI payload must be byte aligned, as described in Annex D */ + uint32_t payloadType; /**< [in] SEI payload types and syntax can be found in Annex D of the H.264 Specification. */ + uint8_t *payload; /**< [in] pointer to user data */ +} NV_ENC_SEI_PAYLOAD; + +#define NV_ENC_H264_SEI_PAYLOAD NV_ENC_SEI_PAYLOAD + +/** + * \struct _NV_ENC_PIC_PARAMS_H264 + * H264 specific enc pic params. sent on a per frame basis. + */ +typedef struct _NV_ENC_PIC_PARAMS_H264 +{ + uint32_t displayPOCSyntax; /**< [in]: Specifies the display POC syntax This is required to be set if client is handling the picture type decision. */ + uint32_t reserved3; /**< [in]: Reserved and must be set to 0 */ + uint32_t refPicFlag; /**< [in]: Set to 1 for a reference picture. This is ignored if NV_ENC_INITIALIZE_PARAMS::enablePTD is set to 1. */ + uint32_t colourPlaneId; /**< [in]: Specifies the colour plane ID associated with the current input. */ + uint32_t forceIntraRefreshWithFrameCnt; /**< [in]: Forces an intra refresh with duration equal to intraRefreshFrameCnt. + When outputRecoveryPointSEI is set this is value is used for recovery_frame_cnt in recovery point SEI message + forceIntraRefreshWithFrameCnt cannot be used if B frames are used in the GOP structure specified */ + uint32_t constrainedFrame : 1; /**< [in]: Set to 1 if client wants to encode this frame with each slice completely independent of other slices in the frame. + NV_ENC_INITIALIZE_PARAMS::enableConstrainedEncoding should be set to 1 */ + uint32_t sliceModeDataUpdate : 1; /**< [in]: Set to 1 if client wants to change the sliceModeData field to specify new sliceSize Parameter + When forceIntraRefreshWithFrameCnt is set it will have priority over sliceMode setting */ + uint32_t ltrMarkFrame : 1; /**< [in]: Set to 1 if client wants to mark this frame as LTR */ + uint32_t ltrUseFrames : 1; /**< [in]: Set to 1 if client allows encoding this frame using the LTR frames specified in ltrFrameBitmap */ + uint32_t reservedBitFields : 28; /**< [in]: Reserved bit fields and must be set to 0 */ + uint8_t *sliceTypeData; /**< [in]: Deprecated. */ + uint32_t sliceTypeArrayCnt; /**< [in]: Deprecated. */ + uint32_t seiPayloadArrayCnt; /**< [in]: Specifies the number of elements allocated in seiPayloadArray array. */ + NV_ENC_SEI_PAYLOAD *seiPayloadArray; /**< [in]: Array of SEI payloads which will be inserted for this frame. */ + uint32_t sliceMode; /**< [in]: This parameter in conjunction with sliceModeData specifies the way in which the picture is divided into slices + sliceMode = 0 MB based slices, sliceMode = 1 Byte based slices, sliceMode = 2 MB row based slices, sliceMode = 3, numSlices in Picture + When forceIntraRefreshWithFrameCnt is set it will have priority over sliceMode setting + When sliceMode == 0 and sliceModeData == 0 whole picture will be coded with one slice */ + uint32_t sliceModeData; /**< [in]: Specifies the parameter needed for sliceMode. For: + sliceMode = 0, sliceModeData specifies # of MBs in each slice (except last slice) + sliceMode = 1, sliceModeData specifies maximum # of bytes in each slice (except last slice) + sliceMode = 2, sliceModeData specifies # of MB rows in each slice (except last slice) + sliceMode = 3, sliceModeData specifies number of slices in the picture. Driver will divide picture into slices optimally */ + uint32_t ltrMarkFrameIdx; /**< [in]: Specifies the long term referenceframe index to use for marking this frame as LTR.*/ + uint32_t ltrUseFrameBitmap; /**< [in]: Specifies the associated bitmap of LTR frame indices to use when encoding this frame. */ + uint32_t ltrUsageMode; /**< [in]: Not supported. Reserved for future use and must be set to 0. */ + uint32_t forceIntraSliceCount; /**< [in]: Specifies the number of slices to be forced to Intra in the current picture. + This option along with forceIntraSliceIdx[] array needs to be used with sliceMode = 3 only */ + uint32_t *forceIntraSliceIdx; /**< [in]: Slice indices to be forced to intra in the current picture. Each slice index should be <= num_slices_in_picture -1. Index starts from 0 for first slice. + The number of entries in this array should be equal to forceIntraSliceCount */ + NV_ENC_PIC_PARAMS_H264_EXT h264ExtPicParams; /**< [in]: Specifies the H264 extension config parameters using this config. */ + NV_ENC_TIME_CODE timeCode; /**< [in]: Specifies the clock timestamp sets used in picture timing SEI. Applicable only when NV_ENC_CONFIG_H264::enableTimeCode is set to 1. */ + uint32_t reserved [203]; /**< [in]: Reserved and must be set to 0. */ + void *reserved2[61]; /**< [in]: Reserved and must be set to NULL. */ +} NV_ENC_PIC_PARAMS_H264; + +/** + * \struct _NV_ENC_PIC_PARAMS_HEVC + * HEVC specific enc pic params. sent on a per frame basis. + */ +typedef struct _NV_ENC_PIC_PARAMS_HEVC +{ + uint32_t displayPOCSyntax; /**< [in]: Specifies the display POC syntax This is required to be set if client is handling the picture type decision. */ + uint32_t refPicFlag; /**< [in]: Set to 1 for a reference picture. This is ignored if NV_ENC_INITIALIZE_PARAMS::enablePTD is set to 1. */ + uint32_t temporalId; /**< [in]: Specifies the temporal id of the picture */ + uint32_t forceIntraRefreshWithFrameCnt; /**< [in]: Forces an intra refresh with duration equal to intraRefreshFrameCnt. + When outputRecoveryPointSEI is set this is value is used for recovery_frame_cnt in recovery point SEI message + forceIntraRefreshWithFrameCnt cannot be used if B frames are used in the GOP structure specified */ + uint32_t constrainedFrame : 1; /**< [in]: Set to 1 if client wants to encode this frame with each slice completely independent of other slices in the frame. + NV_ENC_INITIALIZE_PARAMS::enableConstrainedEncoding should be set to 1 */ + uint32_t sliceModeDataUpdate : 1; /**< [in]: Set to 1 if client wants to change the sliceModeData field to specify new sliceSize Parameter + When forceIntraRefreshWithFrameCnt is set it will have priority over sliceMode setting */ + uint32_t ltrMarkFrame : 1; /**< [in]: Set to 1 if client wants to mark this frame as LTR */ + uint32_t ltrUseFrames : 1; /**< [in]: Set to 1 if client allows encoding this frame using the LTR frames specified in ltrFrameBitmap */ + uint32_t reservedBitFields : 28; /**< [in]: Reserved bit fields and must be set to 0 */ + uint8_t *sliceTypeData; /**< [in]: Array which specifies the slice type used to force intra slice for a particular slice. Currently supported only for NV_ENC_CONFIG_H264::sliceMode == 3. + Client should allocate array of size sliceModeData where sliceModeData is specified in field of ::_NV_ENC_CONFIG_H264 + Array element with index n corresponds to nth slice. To force a particular slice to intra client should set corresponding array element to NV_ENC_SLICE_TYPE_I + all other array elements should be set to NV_ENC_SLICE_TYPE_DEFAULT */ + uint32_t sliceTypeArrayCnt; /**< [in]: Client should set this to the number of elements allocated in sliceTypeData array. If sliceTypeData is NULL then this should be set to 0 */ + uint32_t sliceMode; /**< [in]: This parameter in conjunction with sliceModeData specifies the way in which the picture is divided into slices + sliceMode = 0 CTU based slices, sliceMode = 1 Byte based slices, sliceMode = 2 CTU row based slices, sliceMode = 3, numSlices in Picture + When forceIntraRefreshWithFrameCnt is set it will have priority over sliceMode setting + When sliceMode == 0 and sliceModeData == 0 whole picture will be coded with one slice */ + uint32_t sliceModeData; /**< [in]: Specifies the parameter needed for sliceMode. For: + sliceMode = 0, sliceModeData specifies # of CTUs in each slice (except last slice) + sliceMode = 1, sliceModeData specifies maximum # of bytes in each slice (except last slice) + sliceMode = 2, sliceModeData specifies # of CTU rows in each slice (except last slice) + sliceMode = 3, sliceModeData specifies number of slices in the picture. Driver will divide picture into slices optimally */ + uint32_t ltrMarkFrameIdx; /**< [in]: Specifies the long term reference frame index to use for marking this frame as LTR.*/ + uint32_t ltrUseFrameBitmap; /**< [in]: Specifies the associated bitmap of LTR frame indices to use when encoding this frame. */ + uint32_t ltrUsageMode; /**< [in]: Not supported. Reserved for future use and must be set to 0. */ + uint32_t seiPayloadArrayCnt; /**< [in]: Specifies the number of elements allocated in seiPayloadArray array. */ + uint32_t reserved; /**< [in]: Reserved and must be set to 0. */ + NV_ENC_SEI_PAYLOAD *seiPayloadArray; /**< [in]: Array of SEI payloads which will be inserted for this frame. */ + NV_ENC_TIME_CODE timeCode; /**< [in]: Specifies the clock timestamp sets used in time code SEI. Applicable only when NV_ENC_CONFIG_HEVC::enableTimeCodeSEI is set to 1. */ + uint32_t reserved2 [237]; /**< [in]: Reserved and must be set to 0. */ + void *reserved3[61]; /**< [in]: Reserved and must be set to NULL. */ +} NV_ENC_PIC_PARAMS_HEVC; + +#define NV_ENC_AV1_OBU_PAYLOAD NV_ENC_SEI_PAYLOAD + +/** +* \struct _NV_ENC_PIC_PARAMS_AV1 +* AV1 specific enc pic params. sent on a per frame basis. +*/ +typedef struct _NV_ENC_PIC_PARAMS_AV1 +{ + uint32_t displayPOCSyntax; /**< [in]: Specifies the display POC syntax This is required to be set if client is handling the picture type decision. */ + uint32_t refPicFlag; /**< [in]: Set to 1 for a reference picture. This is ignored if NV_ENC_INITIALIZE_PARAMS::enablePTD is set to 1. */ + uint32_t temporalId; /**< [in]: Specifies the temporal id of the picture */ + uint32_t forceIntraRefreshWithFrameCnt; /**< [in]: Forces an intra refresh with duration equal to intraRefreshFrameCnt. + forceIntraRefreshWithFrameCnt cannot be used if B frames are used in the GOP structure specified */ + uint32_t goldenFrameFlag : 1; /**< [in]: Encode frame as Golden Frame. This is ignored if NV_ENC_INITIALIZE_PARAMS::enablePTD is set to 1. */ + uint32_t arfFrameFlag : 1; /**< [in]: Encode frame as Alternate Reference Frame. This is ignored if NV_ENC_INITIALIZE_PARAMS::enablePTD is set to 1. */ + uint32_t arf2FrameFlag : 1; /**< [in]: Encode frame as Alternate Reference 2 Frame. This is ignored if NV_ENC_INITIALIZE_PARAMS::enablePTD is set to 1. */ + uint32_t bwdFrameFlag : 1; /**< [in]: Encode frame as Backward Reference Frame. This is ignored if NV_ENC_INITIALIZE_PARAMS::enablePTD is set to 1. */ + uint32_t overlayFrameFlag : 1; /**< [in]: Encode frame as overlay frame. A previously encoded frame with the same displayPOCSyntax value should be present in reference frame buffer. + This is ignored if NV_ENC_INITIALIZE_PARAMS::enablePTD is set to 1. */ + uint32_t showExistingFrameFlag : 1; /**< [in]: When ovelayFrameFlag is set to 1, this flag controls the value of the show_existing_frame syntax element associated with the overlay frame. + This flag is added to the interface as a placeholder. Its value is ignored for now and always assumed to be set to 1. + This is ignored if NV_ENC_INITIALIZE_PARAMS::enablePTD is set to 1. */ + uint32_t errorResilientModeFlag : 1; /**< [in]: encode frame independently from previously encoded frames */ + + uint32_t tileConfigUpdate : 1; /**< [in]: Set to 1 if client wants to overwrite the default tile configuration with the tile parameters specified below + When forceIntraRefreshWithFrameCnt is set it will have priority over tileConfigUpdate setting */ + uint32_t enableCustomTileConfig : 1; /**< [in]: Set 1 to enable custom tile configuration: numTileColumns and numTileRows must have non zero values and tileWidths and tileHeights must point to a valid address */ + uint32_t filmGrainParamsUpdate : 1; /**< [in]: Set to 1 if client wants to update previous film grain parameters: filmGrainParams must point to a valid address and encoder must have been configured with film grain enabled */ + uint32_t reservedBitFields : 22; /**< [in]: Reserved bitfields and must be set to 0 */ + uint32_t numTileColumns; /**< [in]: This parameter in conjunction with the flag enableCustomTileConfig and the array tileWidths[] specifies the way in which the picture is divided into tile columns. + When enableCustomTileConfig == 0, the picture will be uniformly divided into numTileColumns tile columns. If numTileColumns is not a power of 2, + it will be rounded down to the next power of 2 value. If numTileColumns == 0, the picture will be coded with the smallest number of vertical tiles as allowed by standard. + When enableCustomTileConfig == 1, numTileColumns must be > 0 and <= NV_MAX_TILE_COLS_AV1 and tileWidths must point to a valid array of numTileColumns entries. + Entry i specifies the width in 64x64 CTU unit of tile colum i. The sum of all the entries should be equal to the picture width in 64x64 CTU units. */ + uint32_t numTileRows; /**< [in]: This parameter in conjunction with the flag enableCustomTileConfig and the array tileHeights[] specifies the way in which the picture is divided into tiles rows + When enableCustomTileConfig == 0, the picture will be uniformly divided into numTileRows tile rows. If numTileRows is not a power of 2, + it will be rounded down to the next power of 2 value. If numTileRows == 0, the picture will be coded with the smallest number of horizontal tiles as allowed by standard. + When enableCustomTileConfig == 1, numTileRows must be > 0 and <= NV_MAX_TILE_ROWS_AV1 and tileHeights must point to a valid array of numTileRows entries. + Entry i specifies the height in 64x64 CTU unit of tile row i. The sum of all the entries should be equal to the picture hieght in 64x64 CTU units. */ + uint32_t *tileWidths; /**< [in]: If enableCustomTileConfig == 1, tileWidths[i] specifies the width of tile column i in 64x64 CTU unit, with 0 <= i <= numTileColumns -1. */ + uint32_t *tileHeights; /**< [in]: If enableCustomTileConfig == 1, tileHeights[i] specifies the height of tile row i in 64x64 CTU unit, with 0 <= i <= numTileRows -1. */ + uint32_t obuPayloadArrayCnt; /**< [in]: Specifies the number of elements allocated in obuPayloadArray array. */ + uint32_t reserved; /**< [in]: Reserved and must be set to 0. */ + NV_ENC_AV1_OBU_PAYLOAD *obuPayloadArray; /**< [in]: Array of OBU payloads which will be inserted for this frame. */ + NV_ENC_FILM_GRAIN_PARAMS_AV1 *filmGrainParams; /**< [in]: If filmGrainParamsUpdate == 1, filmGrainParams must point to a valid NV_ENC_FILM_GRAIN_PARAMS_AV1 structure */ + uint32_t reserved2[247]; /**< [in]: Reserved and must be set to 0. */ + void *reserved3[61]; /**< [in]: Reserved and must be set to NULL. */ +} NV_ENC_PIC_PARAMS_AV1; + +/** + * Codec specific per-picture encoding parameters. + */ +typedef union _NV_ENC_CODEC_PIC_PARAMS +{ + NV_ENC_PIC_PARAMS_H264 h264PicParams; /**< [in]: H264 encode picture params. */ + NV_ENC_PIC_PARAMS_HEVC hevcPicParams; /**< [in]: HEVC encode picture params. */ + NV_ENC_PIC_PARAMS_AV1 av1PicParams; /**< [in]: AV1 encode picture params. */ + uint32_t reserved[256]; /**< [in]: Reserved and must be set to 0. */ +} NV_ENC_CODEC_PIC_PARAMS; + + +/** + * \struct _NV_ENC_PIC_PARAMS + * Encoding parameters that need to be sent on a per frame basis. + */ +typedef struct _NV_ENC_PIC_PARAMS +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_PIC_PARAMS_VER. */ + uint32_t inputWidth; /**< [in]: Specifies the input frame width */ + uint32_t inputHeight; /**< [in]: Specifies the input frame height */ + uint32_t inputPitch; /**< [in]: Specifies the input buffer pitch. If pitch value is not known, set this to inputWidth. */ + uint32_t encodePicFlags; /**< [in]: Specifies bit-wise OR of encode picture flags. See ::NV_ENC_PIC_FLAGS enum. */ + uint32_t frameIdx; /**< [in]: Specifies the frame index associated with the input frame [optional]. */ + uint64_t inputTimeStamp; /**< [in]: Specifies opaque data which is associated with the encoded frame, but not actually encoded in the output bitstream. + This opaque data can be used later to uniquely refer to the corresponding encoded frame. For example, it can be used + for identifying the frame to be invalidated in the reference picture buffer, if lost at the client. */ + uint64_t inputDuration; /**< [in]: Specifies duration of the input picture */ + NV_ENC_INPUT_PTR inputBuffer; /**< [in]: Specifies the input buffer pointer. Client must use a pointer obtained from ::NvEncCreateInputBuffer() or ::NvEncMapInputResource() APIs.*/ + NV_ENC_OUTPUT_PTR outputBitstream; /**< [in]: Specifies the output buffer pointer. + If NV_ENC_INITIALIZE_PARAMS::enableOutputInVidmem is set to 0, specifies the pointer to output buffer. Client should use a pointer obtained from ::NvEncCreateBitstreamBuffer() API. + If NV_ENC_INITIALIZE_PARAMS::enableOutputInVidmem is set to 1, client should allocate buffer in video memory for NV_ENC_ENCODE_OUT_PARAMS struct and encoded bitstream data. Client + should use a pointer obtained from ::NvEncMapInputResource() API, when mapping this output buffer and assign it to NV_ENC_PIC_PARAMS::outputBitstream. + First 256 bytes of this buffer should be interpreted as NV_ENC_ENCODE_OUT_PARAMS struct followed by encoded bitstream data. Recommended size for output buffer is sum of size of + NV_ENC_ENCODE_OUT_PARAMS struct and twice the input frame size for lower resolution eg. CIF and 1.5 times the input frame size for higher resolutions. If encoded bitstream size is + greater than the allocated buffer size for encoded bitstream, then the output buffer will have encoded bitstream data equal to buffer size. All CUDA operations on this buffer must use + the default stream. */ + void *completionEvent; /**< [in]: Specifies an event to be signaled on completion of encoding of this Frame [only if operating in Asynchronous mode]. Each output buffer should be associated with a distinct event pointer. */ + NV_ENC_BUFFER_FORMAT bufferFmt; /**< [in]: Specifies the input buffer format. */ + NV_ENC_PIC_STRUCT pictureStruct; /**< [in]: Specifies structure of the input picture. */ + NV_ENC_PIC_TYPE pictureType; /**< [in]: Specifies input picture type. Client required to be set explicitly by the client if the client has not set NV_ENC_INITALIZE_PARAMS::enablePTD to 1 while calling NvInitializeEncoder. */ + NV_ENC_CODEC_PIC_PARAMS codecPicParams; /**< [in]: Specifies the codec specific per-picture encoding parameters. */ + NVENC_EXTERNAL_ME_HINT_COUNTS_PER_BLOCKTYPE meHintCountsPerBlock[2]; /**< [in]: For H264 and Hevc, specifies the number of hint candidates per block per direction for the current frame. meHintCountsPerBlock[0] is for L0 predictors and meHintCountsPerBlock[1] is for L1 predictors. + The candidate count in NV_ENC_PIC_PARAMS::meHintCountsPerBlock[lx] must never exceed NV_ENC_INITIALIZE_PARAMS::maxMEHintCountsPerBlock[lx] provided during encoder initialization. */ + NVENC_EXTERNAL_ME_HINT *meExternalHints; /**< [in]: For H264 and Hevc, Specifies the pointer to ME external hints for the current frame. The size of ME hint buffer should be equal to number of macroblocks * the total number of candidates per macroblock. + The total number of candidates per MB per direction = 1*meHintCountsPerBlock[Lx].numCandsPerBlk16x16 + 2*meHintCountsPerBlock[Lx].numCandsPerBlk16x8 + 2*meHintCountsPerBlock[Lx].numCandsPerBlk8x8 + + 4*meHintCountsPerBlock[Lx].numCandsPerBlk8x8. For frames using bidirectional ME , the total number of candidates for single macroblock is sum of total number of candidates per MB for each direction (L0 and L1) */ + uint32_t reserved1[6]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[2]; /**< [in]: Reserved and must be set to NULL */ + int8_t *qpDeltaMap; /**< [in]: Specifies the pointer to signed byte array containing value per MB for H264, per CTB for HEVC and per SB for AV1 in raster scan order for the current picture, which will be interpreted depending on NV_ENC_RC_PARAMS::qpMapMode. + If NV_ENC_RC_PARAMS::qpMapMode is NV_ENC_QP_MAP_DELTA, qpDeltaMap specifies QP modifier per MB for H264, per CTB for HEVC and per SB for AV1. This QP modifier will be applied on top of the QP chosen by rate control. + If NV_ENC_RC_PARAMS::qpMapMode is NV_ENC_QP_MAP_EMPHASIS, qpDeltaMap specifies Emphasis Level Map per MB for H264. This level value along with QP chosen by rate control is used to + compute the QP modifier, which in turn is applied on top of QP chosen by rate control. + If NV_ENC_RC_PARAMS::qpMapMode is NV_ENC_QP_MAP_DISABLED, value in qpDeltaMap will be ignored.*/ + uint32_t qpDeltaMapSize; /**< [in]: Specifies the size in bytes of qpDeltaMap surface allocated by client and pointed to by NV_ENC_PIC_PARAMS::qpDeltaMap. Surface (array) should be picWidthInMbs * picHeightInMbs for H264, picWidthInCtbs * picHeightInCtbs for HEVC and + picWidthInSbs * picHeightInSbs for AV1 */ + uint32_t reservedBitFields; /**< [in]: Reserved bitfields and must be set to 0 */ + uint16_t meHintRefPicDist[2]; /**< [in]: Specifies temporal distance for reference picture (NVENC_EXTERNAL_ME_HINT::refidx = 0) used during external ME with NV_ENC_INITALIZE_PARAMS::enablePTD = 1 . meHintRefPicDist[0] is for L0 hints and meHintRefPicDist[1] is for L1 hints. + If not set, will internally infer distance of 1. Ignored for NV_ENC_INITALIZE_PARAMS::enablePTD = 0 */ + NV_ENC_INPUT_PTR alphaBuffer; /**< [in]: Specifies the input alpha buffer pointer. Client must use a pointer obtained from ::NvEncCreateInputBuffer() or ::NvEncMapInputResource() APIs. + Applicable only when encoding hevc with alpha layer is enabled. */ + NVENC_EXTERNAL_ME_SB_HINT *meExternalSbHints; /**< [in]: For AV1,Specifies the pointer to ME external SB hints for the current frame. The size of ME hint buffer should be equal to meSbHintsCount. */ + uint32_t meSbHintsCount; /**< [in]: For AV1, specifies the total number of external ME SB hint candidates for the frame + NV_ENC_PIC_PARAMS::meSbHintsCount must never exceed the total number of SBs in frame * the max number of candidates per SB provided during encoder initialization. + The max number of candidates per SB is maxMeHintCountsPerBlock[0].numCandsPerSb + maxMeHintCountsPerBlock[1].numCandsPerSb */ + uint32_t stateBufferIdx; /**< [in]: Specifies the buffer index in which the encoder state will be saved for current frame encode. It must be in the + range 0 to NV_ENC_INITIALIZE_PARAMS::numStateBuffers - 1. */ + NV_ENC_OUTPUT_PTR outputReconBuffer; /**< [in]: Specifies the reconstructed frame buffer pointer to output reconstructed frame, if enabled by setting NV_ENC_INITIALIZE_PARAMS::enableReconFrameOutput. + Client must allocate buffers for writing the reconstructed frames and register them with the Nvidia Video Encoder Interface with NV_ENC_REGISTER_RESOURCE::bufferUsage + set to NV_ENC_OUTPUT_RECON. + Client must use the pointer obtained from ::NvEncMapInputResource() API and assign it to NV_ENC_PIC_PARAMS::outputReconBuffer. + Reconstructed output will be in NV_ENC_BUFFER_FORMAT_NV12 format when chromaFormatIDC is set to 1. + chromaFormatIDC = 3 is not supported. */ + uint32_t reserved3[284]; /**< [in]: Reserved and must be set to 0 */ + void *reserved4[57]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_PIC_PARAMS; + +/** Macro for constructing the version field of ::_NV_ENC_PIC_PARAMS */ +#define NV_ENC_PIC_PARAMS_VER (NVENCAPI_STRUCT_VERSION(6) | ( 1U<<31 )) + + +/** + * \struct _NV_ENC_MEONLY_PARAMS + * MEOnly parameters that need to be sent on a per motion estimation basis. + * NV_ENC_MEONLY_PARAMS::meExternalHints is supported for H264 only. + */ +typedef struct _NV_ENC_MEONLY_PARAMS +{ + uint32_t version; /**< [in]: Struct version. Must be set to NV_ENC_MEONLY_PARAMS_VER.*/ + uint32_t inputWidth; /**< [in]: Specifies the input frame width */ + uint32_t inputHeight; /**< [in]: Specifies the input frame height */ + NV_ENC_INPUT_PTR inputBuffer; /**< [in]: Specifies the input buffer pointer. Client must use a pointer obtained from NvEncCreateInputBuffer() or NvEncMapInputResource() APIs. */ + NV_ENC_INPUT_PTR referenceFrame; /**< [in]: Specifies the reference frame pointer */ + NV_ENC_OUTPUT_PTR mvBuffer; /**< [in]: Specifies the output buffer pointer. + If NV_ENC_INITIALIZE_PARAMS::enableOutputInVidmem is set to 0, specifies the pointer to motion vector data buffer allocated by NvEncCreateMVBuffer. + Client must lock mvBuffer using ::NvEncLockBitstream() API to get the motion vector data. + If NV_ENC_INITIALIZE_PARAMS::enableOutputInVidmem is set to 1, client should allocate buffer in video memory for storing the motion vector data. The size of this buffer must + be equal to total number of macroblocks multiplied by size of NV_ENC_H264_MV_DATA struct. Client should use a pointer obtained from ::NvEncMapInputResource() API, when mapping this + output buffer and assign it to NV_ENC_MEONLY_PARAMS::mvBuffer. All CUDA operations on this buffer must use the default stream. */ + NV_ENC_BUFFER_FORMAT bufferFmt; /**< [in]: Specifies the input buffer format. */ + void *completionEvent; /**< [in]: Specifies an event to be signaled on completion of motion estimation + of this Frame [only if operating in Asynchronous mode]. + Each output buffer should be associated with a distinct event pointer. */ + uint32_t viewID; /**< [in]: Specifies left or right viewID if NV_ENC_CONFIG_H264_MEONLY::bStereoEnable is set. + viewID can be 0,1 if bStereoEnable is set, 0 otherwise. */ + NVENC_EXTERNAL_ME_HINT_COUNTS_PER_BLOCKTYPE + meHintCountsPerBlock[2]; /**< [in]: Specifies the number of hint candidates per block for the current frame. meHintCountsPerBlock[0] is for L0 predictors. + The candidate count in NV_ENC_PIC_PARAMS::meHintCountsPerBlock[lx] must never exceed NV_ENC_INITIALIZE_PARAMS::maxMEHintCountsPerBlock[lx] provided during encoder initialization. */ + NVENC_EXTERNAL_ME_HINT *meExternalHints; /**< [in]: Specifies the pointer to ME external hints for the current frame. The size of ME hint buffer should be equal to number of macroblocks * the total number of candidates per macroblock. + The total number of candidates per MB per direction = 1*meHintCountsPerBlock[Lx].numCandsPerBlk16x16 + 2*meHintCountsPerBlock[Lx].numCandsPerBlk16x8 + 2*meHintCountsPerBlock[Lx].numCandsPerBlk8x8 + + 4*meHintCountsPerBlock[Lx].numCandsPerBlk8x8. For frames using bidirectional ME , the total number of candidates for single macroblock is sum of total number of candidates per MB for each direction (L0 and L1) */ + uint32_t reserved1[243]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[59]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_MEONLY_PARAMS; + +/** NV_ENC_MEONLY_PARAMS struct version*/ +#define NV_ENC_MEONLY_PARAMS_VER NVENCAPI_STRUCT_VERSION(3) + + +/** + * \struct _NV_ENC_LOCK_BITSTREAM + * Bitstream buffer lock parameters. + */ +typedef struct _NV_ENC_LOCK_BITSTREAM +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_LOCK_BITSTREAM_VER. */ + uint32_t doNotWait : 1; /**< [in]: If this flag is set, the NvEncodeAPI interface will return buffer pointer even if operation is not completed. If not set, the call will block until operation completes. */ + uint32_t ltrFrame : 1; /**< [out]: Flag indicating this frame is marked as LTR frame */ + uint32_t getRCStats : 1; /**< [in]: If this flag is set then lockBitstream call will add additional intra-inter MB count and average MVX, MVY */ + uint32_t reservedBitFields : 29; /**< [in]: Reserved bit fields and must be set to 0 */ + void *outputBitstream; /**< [in]: Pointer to the bitstream buffer being locked. */ + uint32_t *sliceOffsets; /**< [in, out]: Array which receives the slice (H264/HEVC) or tile (AV1) offsets. This is not supported if NV_ENC_CONFIG_H264::sliceMode is 1 on Kepler GPUs. Array size must be equal to size of frame in MBs. */ + uint32_t frameIdx; /**< [out]: Frame no. for which the bitstream is being retrieved. */ + uint32_t hwEncodeStatus; /**< [out]: The NvEncodeAPI interface status for the locked picture. */ + uint32_t numSlices; /**< [out]: Number of slices (H264/HEVC) or tiles (AV1) in the encoded picture. Will be reported only if NV_ENC_INITIALIZE_PARAMS::reportSliceOffsets set to 1. */ + uint32_t bitstreamSizeInBytes; /**< [out]: Actual number of bytes generated and copied to the memory pointed by bitstreamBufferPtr. + When HEVC alpha layer encoding is enabled, this field reports the total encoded size in bytes i.e it is the encoded size of the base plus the alpha layer. + For AV1 when enablePTD is set, this field reports the total encoded size in bytes of all the encoded frames packed into the current output surface i.e. show frame plus all preceding no-show frames */ + uint64_t outputTimeStamp; /**< [out]: Presentation timestamp associated with the encoded output. */ + uint64_t outputDuration; /**< [out]: Presentation duration associates with the encoded output. */ + void *bitstreamBufferPtr; /**< [out]: Pointer to the generated output bitstream. + For MEOnly mode _NV_ENC_LOCK_BITSTREAM::bitstreamBufferPtr should be typecast to + NV_ENC_H264_MV_DATA/NV_ENC_HEVC_MV_DATA pointer respectively for H264/HEVC */ + NV_ENC_PIC_TYPE pictureType; /**< [out]: Picture type of the encoded picture. */ + NV_ENC_PIC_STRUCT pictureStruct; /**< [out]: Structure of the generated output picture. */ + uint32_t frameAvgQP; /**< [out]: Average QP of the frame. */ + uint32_t frameSatd; /**< [out]: Total SATD cost for whole frame. */ + uint32_t ltrFrameIdx; /**< [out]: Frame index associated with this LTR frame. */ + uint32_t ltrFrameBitmap; /**< [out]: Bitmap of LTR frames indices which were used for encoding this frame. Value of 0 if no LTR frames were used. */ + uint32_t temporalId; /**< [out]: TemporalId value of the frame when using temporalSVC encoding */ + uint32_t intraMBCount; /**< [out]: For H264, Number of Intra MBs in the encoded frame. For HEVC, Number of Intra CTBs in the encoded frame. For AV1, Number of Intra SBs in the encoded show frame. Supported only if _NV_ENC_LOCK_BITSTREAM::getRCStats set to 1. */ + uint32_t interMBCount; /**< [out]: For H264, Number of Inter MBs in the encoded frame, includes skip MBs. For HEVC, Number of Inter CTBs in the encoded frame. For AV1, Number of Inter SBs in the encoded show frame. Supported only if _NV_ENC_LOCK_BITSTREAM::getRCStats set to 1. */ + int32_t averageMVX; /**< [out]: Average Motion Vector in X direction for the encoded frame. Supported only if _NV_ENC_LOCK_BITSTREAM::getRCStats set to 1. */ + int32_t averageMVY; /**< [out]: Average Motion Vector in y direction for the encoded frame. Supported only if _NV_ENC_LOCK_BITSTREAM::getRCStats set to 1. */ + uint32_t alphaLayerSizeInBytes; /**< [out]: Number of bytes generated for the alpha layer in the encoded output. Applicable only when HEVC with alpha encoding is enabled. */ + uint32_t outputStatsPtrSize; /**< [in]: Size of the buffer pointed by NV_ENC_LOCK_BITSTREAM::outputStatsPtr. */ + void *outputStatsPtr; /**< [in, out]: Buffer which receives the encoded frame output stats, if NV_ENC_INITIALIZE_PARAMS::enableOutputStats is set to 1. */ + uint32_t frameIdxDisplay; /**< [out]: Frame index in display order */ + uint32_t reserved1[220]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[63]; /**< [in]: Reserved and must be set to NULL */ + uint32_t reservedInternal[8]; /**< [in]: Reserved and must be set to 0 */ +} NV_ENC_LOCK_BITSTREAM; + +#define NV_ENC_LOCK_BITSTREAM_VER (NVENCAPI_STRUCT_VERSION(1) | ( 1U<<31 )) + + +/** + * \struct _NV_ENC_LOCK_INPUT_BUFFER + * Uncompressed Input Buffer lock parameters. + */ +typedef struct _NV_ENC_LOCK_INPUT_BUFFER +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_LOCK_INPUT_BUFFER_VER. */ + uint32_t doNotWait : 1; /**< [in]: Set to 1 to make ::NvEncLockInputBuffer() a unblocking call. If the encoding is not completed, driver will return ::NV_ENC_ERR_ENCODER_BUSY error code. */ + uint32_t reservedBitFields : 31; /**< [in]: Reserved bitfields and must be set to 0 */ + NV_ENC_INPUT_PTR inputBuffer; /**< [in]: Pointer to the input buffer to be locked, client should pass the pointer obtained from ::NvEncCreateInputBuffer() or ::NvEncMapInputResource API. */ + void *bufferDataPtr; /**< [out]: Pointed to the locked input buffer data. Client can only access input buffer using the \p bufferDataPtr. */ + uint32_t pitch; /**< [out]: Pitch of the locked input buffer. */ + uint32_t reserved1[251]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_LOCK_INPUT_BUFFER; + +/** Macro for constructing the version field of ::_NV_ENC_LOCK_INPUT_BUFFER */ +#define NV_ENC_LOCK_INPUT_BUFFER_VER NVENCAPI_STRUCT_VERSION(1) + + +/** + * \struct _NV_ENC_MAP_INPUT_RESOURCE + * Map an input resource to a Nvidia Encoder Input Buffer + */ +typedef struct _NV_ENC_MAP_INPUT_RESOURCE +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_MAP_INPUT_RESOURCE_VER. */ + uint32_t subResourceIndex; /**< [in]: Deprecated. Do not use. */ + void *inputResource; /**< [in]: Deprecated. Do not use. */ + NV_ENC_REGISTERED_PTR registeredResource; /**< [in]: The Registered resource handle obtained by calling NvEncRegisterInputResource. */ + NV_ENC_INPUT_PTR mappedResource; /**< [out]: Mapped pointer corresponding to the registeredResource. This pointer must be used in NV_ENC_PIC_PARAMS::inputBuffer parameter in ::NvEncEncodePicture() API. */ + NV_ENC_BUFFER_FORMAT mappedBufferFmt; /**< [out]: Buffer format of the outputResource. This buffer format must be used in NV_ENC_PIC_PARAMS::bufferFmt if client using the above mapped resource pointer. */ + uint32_t reserved1[251]; /**< [in]: Reserved and must be set to 0. */ + void *reserved2[63]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_MAP_INPUT_RESOURCE; + +/** Macro for constructing the version field of ::_NV_ENC_MAP_INPUT_RESOURCE */ +#define NV_ENC_MAP_INPUT_RESOURCE_VER NVENCAPI_STRUCT_VERSION(4) + +/** + * \struct _NV_ENC_INPUT_RESOURCE_OPENGL_TEX + * NV_ENC_REGISTER_RESOURCE::resourceToRegister must be a pointer to a variable of this type, + * when NV_ENC_REGISTER_RESOURCE::resourceType is NV_ENC_INPUT_RESOURCE_TYPE_OPENGL_TEX + */ +typedef struct _NV_ENC_INPUT_RESOURCE_OPENGL_TEX +{ + uint32_t texture; /**< [in]: The name of the texture to be used. */ + uint32_t target; /**< [in]: Accepted values are GL_TEXTURE_RECTANGLE and GL_TEXTURE_2D. */ +} NV_ENC_INPUT_RESOURCE_OPENGL_TEX; + +/** \struct NV_ENC_FENCE_POINT_D3D12 +* Fence and fence value for synchronization. +*/ +typedef struct _NV_ENC_FENCE_POINT_D3D12 +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_FENCE_POINT_D3D12_VER. */ + uint32_t reserved; /**< [in]: Reserved and must be set to 0. */ + void *pFence; /**< [in]: Pointer to ID3D12Fence. This fence object is used for synchronization. */ + uint64_t waitValue; /**< [in]: Fence value to reach or exceed before the GPU operation. */ + uint64_t signalValue; /**< [in]: Fence value to set the fence to, after the GPU operation. */ + uint32_t bWait: 1; /**< [in]: Wait on 'waitValue' if bWait is set to 1, before starting GPU operation. */ + uint32_t bSignal: 1; /**< [in]: Signal on 'signalValue' if bSignal is set to 1, after GPU operation is complete. */ + uint32_t reservedBitField: 30; /**< [in]: Reserved and must be set to 0. */ + uint32_t reserved1[7]; /**< [in]: Reserved and must be set to 0. */ +} NV_ENC_FENCE_POINT_D3D12; + +#define NV_ENC_FENCE_POINT_D3D12_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * \struct _NV_ENC_INPUT_RESOURCE_D3D12 + * NV_ENC_PIC_PARAMS::inputBuffer and NV_ENC_PIC_PARAMS::alphaBuffer must be a pointer to a struct of this type, + * when D3D12 interface is used + */ +typedef struct _NV_ENC_INPUT_RESOURCE_D3D12 +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_INPUT_RESOURCE_D3D12_VER. */ + uint32_t reserved; /**< [in]: Reserved and must be set to 0. */ + NV_ENC_INPUT_PTR pInputBuffer; /**< [in]: Specifies the input surface pointer. Client must use a pointer obtained from NvEncMapInputResource() in NV_ENC_MAP_INPUT_RESOURCE::mappedResource + when mapping the input surface. */ + NV_ENC_FENCE_POINT_D3D12 inputFencePoint; /**< [in]: Specifies the fence and corresponding fence values to do GPU wait and signal. */ + uint32_t reserved1[16]; /**< [in]: Reserved and must be set to 0. */ + void *reserved2[16]; /**< [in]: Reserved and must be set to NULL. */ +} NV_ENC_INPUT_RESOURCE_D3D12; + +#define NV_ENC_INPUT_RESOURCE_D3D12_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * \struct _NV_ENC_OUTPUT_RESOURCE_D3D12 + * NV_ENC_PIC_PARAMS::outputBitstream and NV_ENC_LOCK_BITSTREAM::outputBitstream must be a pointer to a struct of this type, + * when D3D12 interface is used + */ +typedef struct _NV_ENC_OUTPUT_RESOURCE_D3D12 +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_OUTPUT_RESOURCE_D3D12_VER. */ + uint32_t reserved; /**< [in]: Reserved and must be set to 0. */ + NV_ENC_INPUT_PTR pOutputBuffer; /**< [in]: Specifies the output buffer pointer. Client must use a pointer obtained from NvEncMapInputResource() in NV_ENC_MAP_INPUT_RESOURCE::mappedResource + when mapping output bitstream buffer */ + NV_ENC_FENCE_POINT_D3D12 outputFencePoint; /**< [in]: Specifies the fence and corresponding fence values to do GPU wait and signal.*/ + uint32_t reserved1[16]; /**< [in]: Reserved and must be set to 0. */ + void *reserved2[16]; /**< [in]: Reserved and must be set to NULL. */ +} NV_ENC_OUTPUT_RESOURCE_D3D12; + +#define NV_ENC_OUTPUT_RESOURCE_D3D12_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * \struct _NV_ENC_REGISTER_RESOURCE + * Register a resource for future use with the Nvidia Video Encoder Interface. + */ +typedef struct _NV_ENC_REGISTER_RESOURCE +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_REGISTER_RESOURCE_VER. */ + NV_ENC_INPUT_RESOURCE_TYPE resourceType; /**< [in]: Specifies the type of resource to be registered. + Supported values are + ::NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX, + ::NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR, + ::NV_ENC_INPUT_RESOURCE_TYPE_OPENGL_TEX */ + uint32_t width; /**< [in]: Input frame width. */ + uint32_t height; /**< [in]: Input frame height. */ + uint32_t pitch; /**< [in]: Input buffer pitch. + For ::NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX resources, set this to 0. + For ::NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR resources, set this to + the pitch as obtained from cuMemAllocPitch(), or to the width in + bytes (if this resource was created by using cuMemAlloc()). This + value must be a multiple of 4. + For ::NV_ENC_INPUT_RESOURCE_TYPE_CUDAARRAY resources, set this to the + width of the allocation in bytes (i.e. + CUDA_ARRAY3D_DESCRIPTOR::Width * CUDA_ARRAY3D_DESCRIPTOR::NumChannels). + For ::NV_ENC_INPUT_RESOURCE_TYPE_OPENGL_TEX resources, set this to the + texture width multiplied by the number of components in the texture + format. */ + uint32_t subResourceIndex; /**< [in]: Subresource Index of the DirectX resource to be registered. Should be set to 0 for other interfaces. */ + void *resourceToRegister; /**< [in]: Handle to the resource that is being registered. */ + NV_ENC_REGISTERED_PTR registeredResource; /**< [out]: Registered resource handle. This should be used in future interactions with the Nvidia Video Encoder Interface. */ + NV_ENC_BUFFER_FORMAT bufferFormat; /**< [in]: Buffer format of resource to be registered. */ + NV_ENC_BUFFER_USAGE bufferUsage; /**< [in]: Usage of resource to be registered. */ + NV_ENC_FENCE_POINT_D3D12 *pInputFencePoint; /**< [in]: Specifies the input fence and corresponding fence values to do GPU wait and signal. + To be used only when NV_ENC_REGISTER_RESOURCE::resourceToRegister represents D3D12 surface and + NV_ENC_BUFFER_USAGE::bufferUsage is NV_ENC_INPUT_IMAGE. + The fence NV_ENC_FENCE_POINT_D3D12::pFence and NV_ENC_FENCE_POINT_D3D12::waitValue will be used to do GPU wait + before starting GPU operation, if NV_ENC_FENCE_POINT_D3D12::bWait is set. + The fence NV_ENC_FENCE_POINT_D3D12::pFence and NV_ENC_FENCE_POINT_D3D12::signalValue will be used to do GPU signal + when GPU operation finishes, if NV_ENC_FENCE_POINT_D3D12::bSignal is set. */ + uint32_t chromaOffset[2]; /**< [out]: Chroma offset for the reconstructed output buffer when NV_ENC_BUFFER_USAGE::bufferUsage is set + to NV_ENC_OUTPUT_RECON and D3D11 interface is used. + When chroma components are interleaved, 'chromaOffset[0]' will contain chroma offset. + chromaOffset[1] is reserved for future use. */ + uint32_t reserved1[245]; /**< [in]: Reserved and must be set to 0. */ + void *reserved2[61]; /**< [in]: Reserved and must be set to NULL. */ +} NV_ENC_REGISTER_RESOURCE; + +/** Macro for constructing the version field of ::_NV_ENC_REGISTER_RESOURCE */ +#define NV_ENC_REGISTER_RESOURCE_VER NVENCAPI_STRUCT_VERSION(4) + +/** + * \struct _NV_ENC_STAT + * Encode Stats structure. + */ +typedef struct _NV_ENC_STAT +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_STAT_VER. */ + uint32_t reserved; /**< [in]: Reserved and must be set to 0 */ + NV_ENC_OUTPUT_PTR outputBitStream; /**< [out]: Specifies the pointer to output bitstream. */ + uint32_t bitStreamSize; /**< [out]: Size of generated bitstream in bytes. */ + uint32_t picType; /**< [out]: Picture type of encoded picture. See ::NV_ENC_PIC_TYPE. */ + uint32_t lastValidByteOffset; /**< [out]: Offset of last valid bytes of completed bitstream */ + uint32_t sliceOffsets[16]; /**< [out]: Offsets of each slice */ + uint32_t picIdx; /**< [out]: Picture number */ + uint32_t frameAvgQP; /**< [out]: Average QP of the frame. */ + uint32_t ltrFrame : 1; /**< [out]: Flag indicating this frame is marked as LTR frame */ + uint32_t reservedBitFields : 31; /**< [in]: Reserved bit fields and must be set to 0 */ + uint32_t ltrFrameIdx; /**< [out]: Frame index associated with this LTR frame. */ + uint32_t intraMBCount; /**< [out]: For H264, Number of Intra MBs in the encoded frame. For HEVC, Number of Intra CTBs in the encoded frame. */ + uint32_t interMBCount; /**< [out]: For H264, Number of Inter MBs in the encoded frame, includes skip MBs. For HEVC, Number of Inter CTBs in the encoded frame. */ + int32_t averageMVX; /**< [out]: Average Motion Vector in X direction for the encoded frame. */ + int32_t averageMVY; /**< [out]: Average Motion Vector in y direction for the encoded frame. */ + uint32_t reserved1[226]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_STAT; + +/** Macro for constructing the version field of ::_NV_ENC_STAT */ +#define NV_ENC_STAT_VER NVENCAPI_STRUCT_VERSION(1) + + +/** + * \struct _NV_ENC_SEQUENCE_PARAM_PAYLOAD + * Sequence and picture paramaters payload. + */ +typedef struct _NV_ENC_SEQUENCE_PARAM_PAYLOAD +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_INITIALIZE_PARAMS_VER. */ + uint32_t inBufferSize; /**< [in]: Specifies the size of the spsppsBuffer provided by the client */ + uint32_t spsId; /**< [in]: Specifies the SPS id to be used in sequence header. Default value is 0. */ + uint32_t ppsId; /**< [in]: Specifies the PPS id to be used in picture header. Default value is 0. */ + void *spsppsBuffer; /**< [in]: Specifies bitstream header pointer of size NV_ENC_SEQUENCE_PARAM_PAYLOAD::inBufferSize. + It is the client's responsibility to manage this memory. */ + uint32_t *outSPSPPSPayloadSize; /**< [out]: Size of the sequence and picture header in bytes. */ + uint32_t reserved [250]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_SEQUENCE_PARAM_PAYLOAD; + +/** Macro for constructing the version field of ::_NV_ENC_SEQUENCE_PARAM_PAYLOAD */ +#define NV_ENC_SEQUENCE_PARAM_PAYLOAD_VER NVENCAPI_STRUCT_VERSION(1) + + +/** + * Event registration/unregistration parameters. + */ +typedef struct _NV_ENC_EVENT_PARAMS +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_EVENT_PARAMS_VER. */ + uint32_t reserved; /**< [in]: Reserved and must be set to 0 */ + void *completionEvent; /**< [in]: Handle to event to be registered/unregistered with the NvEncodeAPI interface. */ + uint32_t reserved1[253]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_EVENT_PARAMS; + +/** Macro for constructing the version field of ::_NV_ENC_EVENT_PARAMS */ +#define NV_ENC_EVENT_PARAMS_VER NVENCAPI_STRUCT_VERSION(1) + +/** + * Encoder Session Creation parameters + */ +typedef struct _NV_ENC_OPEN_ENCODE_SESSIONEX_PARAMS +{ + uint32_t version; /**< [in]: Struct version. Must be set to ::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER. */ + NV_ENC_DEVICE_TYPE deviceType; /**< [in]: Specified the device Type */ + void *device; /**< [in]: Pointer to client device. */ + void *reserved; /**< [in]: Reserved and must be set to 0. */ + uint32_t apiVersion; /**< [in]: API version. Should be set to NVENCAPI_VERSION. */ + uint32_t reserved1[253]; /**< [in]: Reserved and must be set to 0 */ + void *reserved2[64]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS; +/** Macro for constructing the version field of ::_NV_ENC_OPEN_ENCODE_SESSIONEX_PARAMS */ +#define NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER NVENCAPI_STRUCT_VERSION(1) + +/** @} */ /* END ENCODER_STRUCTURE */ + + +/** + * \addtogroup ENCODE_FUNC NvEncodeAPI Functions + * @{ + */ + +// NvEncOpenEncodeSession +/** + * \brief Opens an encoding session. + * + * Deprecated. + * + * \return + * ::NV_ENC_ERR_INVALID_CALL\n + * + */ +NVENCSTATUS NVENCAPI NvEncOpenEncodeSession (void *device, uint32_t deviceType, void **encoder); + +// NvEncGetEncodeGuidCount +/** + * \brief Retrieves the number of supported encode GUIDs. + * + * The function returns the number of codec GUIDs supported by the NvEncodeAPI + * interface. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [out] encodeGUIDCount + * Number of supported encode GUIDs. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncGetEncodeGUIDCount (void *encoder, uint32_t *encodeGUIDCount); + + +// NvEncGetEncodeGUIDs +/** + * \brief Retrieves an array of supported encoder codec GUIDs. + * + * The function returns an array of codec GUIDs supported by the NvEncodeAPI interface. + * The client must allocate an array where the NvEncodeAPI interface can + * fill the supported GUIDs and pass the pointer in \p *GUIDs parameter. + * The size of the array can be determined by using ::NvEncGetEncodeGUIDCount() API. + * The Nvidia Encoding interface returns the number of codec GUIDs it has actually + * filled in the GUID array in the \p GUIDCount parameter. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] guidArraySize + * Number of GUIDs to retrieved. Should be set to the number retrieved using + * ::NvEncGetEncodeGUIDCount. + * \param [out] GUIDs + * Array of supported Encode GUIDs. + * \param [out] GUIDCount + * Number of supported Encode GUIDs. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncGetEncodeGUIDs (void *encoder, GUID *GUIDs, uint32_t guidArraySize, uint32_t *GUIDCount); + + +// NvEncGetEncodeProfileGuidCount +/** + * \brief Retrieves the number of supported profile GUIDs. + * + * The function returns the number of profile GUIDs supported for a given codec. + * The client must first enumerate the codec GUIDs supported by the NvEncodeAPI + * interface. After determining the codec GUID, it can query the NvEncodeAPI + * interface to determine the number of profile GUIDs supported for a particular + * codec GUID. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] encodeGUID + * The codec GUID for which the profile GUIDs are being enumerated. + * \param [out] encodeProfileGUIDCount + * Number of encode profiles supported for the given encodeGUID. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncGetEncodeProfileGUIDCount (void *encoder, GUID encodeGUID, uint32_t *encodeProfileGUIDCount); + + +// NvEncGetEncodeProfileGUIDs +/** + * \brief Retrieves an array of supported encode profile GUIDs. + * + * The function returns an array of supported profile GUIDs for a particular + * codec GUID. The client must allocate an array where the NvEncodeAPI interface + * can populate the profile GUIDs. The client can determine the array size using + * ::NvEncGetEncodeProfileGUIDCount() API. The client must also validiate that the + * NvEncodeAPI interface supports the GUID the client wants to pass as \p encodeGUID + * parameter. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] encodeGUID + * The encode GUID whose profile GUIDs are being enumerated. + * \param [in] guidArraySize + * Number of GUIDs to be retrieved. Should be set to the number retrieved using + * ::NvEncGetEncodeProfileGUIDCount. + * \param [out] profileGUIDs + * Array of supported Encode Profile GUIDs + * \param [out] GUIDCount + * Number of valid encode profile GUIDs in \p profileGUIDs array. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncGetEncodeProfileGUIDs (void *encoder, GUID encodeGUID, GUID *profileGUIDs, uint32_t guidArraySize, uint32_t *GUIDCount); + +// NvEncGetInputFormatCount +/** + * \brief Retrieve the number of supported Input formats. + * + * The function returns the number of supported input formats. The client must + * query the NvEncodeAPI interface to determine the supported input formats + * before creating the input surfaces. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] encodeGUID + * Encode GUID, corresponding to which the number of supported input formats + * is to be retrieved. + * \param [out] inputFmtCount + * Number of input formats supported for specified Encode GUID. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + */ +NVENCSTATUS NVENCAPI NvEncGetInputFormatCount (void *encoder, GUID encodeGUID, uint32_t *inputFmtCount); + + +// NvEncGetInputFormats +/** + * \brief Retrieves an array of supported Input formats + * + * Returns an array of supported input formats The client must use the input + * format to create input surface using ::NvEncCreateInputBuffer() API. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] encodeGUID + * Encode GUID, corresponding to which the number of supported input formats + * is to be retrieved. + *\param [in] inputFmtArraySize + * Size input format count array passed in \p inputFmts. + *\param [out] inputFmts + * Array of input formats supported for this Encode GUID. + *\param [out] inputFmtCount + * The number of valid input format types returned by the NvEncodeAPI + * interface in \p inputFmts array. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncGetInputFormats (void *encoder, GUID encodeGUID, NV_ENC_BUFFER_FORMAT *inputFmts, uint32_t inputFmtArraySize, uint32_t *inputFmtCount); + + +// NvEncGetEncodeCaps +/** + * \brief Retrieves the capability value for a specified encoder attribute. + * + * The function returns the capability value for a given encoder attribute. The + * client must validate the encodeGUID using ::NvEncGetEncodeGUIDs() API before + * calling this function. The encoder attribute being queried are enumerated in + * ::NV_ENC_CAPS_PARAM enum. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] encodeGUID + * Encode GUID, corresponding to which the capability attribute is to be retrieved. + * \param [in] capsParam + * Used to specify attribute being queried. Refer ::NV_ENC_CAPS_PARAM for more + * details. + * \param [out] capsVal + * The value corresponding to the capability attribute being queried. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + */ +NVENCSTATUS NVENCAPI NvEncGetEncodeCaps (void *encoder, GUID encodeGUID, NV_ENC_CAPS_PARAM *capsParam, int *capsVal); + + +// NvEncGetEncodePresetCount +/** + * \brief Retrieves the number of supported preset GUIDs. + * + * The function returns the number of preset GUIDs available for a given codec. + * The client must validate the codec GUID using ::NvEncGetEncodeGUIDs() API + * before calling this function. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] encodeGUID + * Encode GUID, corresponding to which the number of supported presets is to + * be retrieved. + * \param [out] encodePresetGUIDCount + * Receives the number of supported preset GUIDs. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncGetEncodePresetCount (void *encoder, GUID encodeGUID, uint32_t *encodePresetGUIDCount); + + +// NvEncGetEncodePresetGUIDs +/** + * \brief Receives an array of supported encoder preset GUIDs. + * + * The function returns an array of encode preset GUIDs available for a given codec. + * The client can directly use one of the preset GUIDs based upon the use case + * or target device. The preset GUID chosen can be directly used in + * NV_ENC_INITIALIZE_PARAMS::presetGUID parameter to ::NvEncEncodePicture() API. + * Alternately client can also use the preset GUID to retrieve the encoding config + * parameters being used by NvEncodeAPI interface for that given preset, using + * ::NvEncGetEncodePresetConfig() API. It can then modify preset config parameters + * as per its use case and send it to NvEncodeAPI interface as part of + * NV_ENC_INITIALIZE_PARAMS::encodeConfig parameter for NvEncInitializeEncoder() + * API. + * + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] encodeGUID + * Encode GUID, corresponding to which the list of supported presets is to be + * retrieved. + * \param [in] guidArraySize + * Size of array of preset GUIDs passed in \p preset GUIDs + * \param [out] presetGUIDs + * Array of supported Encode preset GUIDs from the NvEncodeAPI interface + * to client. + * \param [out] encodePresetGUIDCount + * Receives the number of preset GUIDs returned by the NvEncodeAPI + * interface. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncGetEncodePresetGUIDs (void *encoder, GUID encodeGUID, GUID *presetGUIDs, uint32_t guidArraySize, uint32_t *encodePresetGUIDCount); + + +// NvEncGetEncodePresetConfig +/** + * \brief Returns a preset config structure supported for given preset GUID. + * + * The function returns a preset config structure for a given preset GUID. + * NvEncGetEncodePresetConfig() API is not applicable to AV1. + * Before using this function the client must enumerate the preset GUIDs available for + * a given codec. The preset config structure can be modified by the client depending + * upon its use case and can be then used to initialize the encoder using + * ::NvEncInitializeEncoder() API. The client can use this function only if it + * wants to modify the NvEncodeAPI preset configuration, otherwise it can + * directly use the preset GUID. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] encodeGUID + * Encode GUID, corresponding to which the list of supported presets is to be + * retrieved. + * \param [in] presetGUID + * Preset GUID, corresponding to which the Encoding configurations is to be + * retrieved. + * \param [out] presetConfig + * The requested Preset Encoder Attribute set. Refer ::_NV_ENC_CONFIG for +* more details. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncGetEncodePresetConfig (void *encoder, GUID encodeGUID, GUID presetGUID, NV_ENC_PRESET_CONFIG *presetConfig); + +// NvEncGetEncodePresetConfigEx +/** + * \brief Returns a preset config structure supported for given preset GUID. + * + * The function returns a preset config structure for a given preset GUID and tuning info. + * NvEncGetEncodePresetConfigEx() API is not applicable to H264 and HEVC meonly mode. + * Before using this function the client must enumerate the preset GUIDs available for + * a given codec. The preset config structure can be modified by the client depending + * upon its use case and can be then used to initialize the encoder using + * ::NvEncInitializeEncoder() API. The client can use this function only if it + * wants to modify the NvEncodeAPI preset configuration, otherwise it can + * directly use the preset GUID. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] encodeGUID + * Encode GUID, corresponding to which the list of supported presets is to be + * retrieved. + * \param [in] presetGUID + * Preset GUID, corresponding to which the Encoding configurations is to be + * retrieved. + * \param [in] tuningInfo + * tuning info, corresponding to which the Encoding configurations is to be + * retrieved. + * \param [out] presetConfig + * The requested Preset Encoder Attribute set. Refer ::_NV_ENC_CONFIG for + * more details. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncGetEncodePresetConfigEx (void *encoder, GUID encodeGUID, GUID presetGUID, NV_ENC_TUNING_INFO tuningInfo, NV_ENC_PRESET_CONFIG *presetConfig); + +// NvEncInitializeEncoder +/** + * \brief Initialize the encoder. + * + * This API must be used to initialize the encoder. The initialization parameter + * is passed using \p *createEncodeParams The client must send the following + * fields of the _NV_ENC_INITIALIZE_PARAMS structure with a valid value. + * - NV_ENC_INITIALIZE_PARAMS::encodeGUID + * - NV_ENC_INITIALIZE_PARAMS::encodeWidth + * - NV_ENC_INITIALIZE_PARAMS::encodeHeight + * + * The client can pass a preset GUID directly to the NvEncodeAPI interface using + * NV_ENC_INITIALIZE_PARAMS::presetGUID field. If the client doesn't pass + * NV_ENC_INITIALIZE_PARAMS::encodeConfig structure, the codec specific parameters + * will be selected based on the preset GUID. The preset GUID must have been + * validated by the client using ::NvEncGetEncodePresetGUIDs() API. + * If the client passes a custom ::_NV_ENC_CONFIG structure through + * NV_ENC_INITIALIZE_PARAMS::encodeConfig , it will override the codec specific parameters + * based on the preset GUID. It is recommended that even if the client passes a custom config, + * it should also send a preset GUID. In this case, the preset GUID passed by the client + * will not override any of the custom config parameters programmed by the client, + * it is only used as a hint by the NvEncodeAPI interface to determine certain encoder parameters + * which are not exposed to the client. + * + * There are two modes of operation for the encoder namely: + * - Asynchronous mode + * - Synchronous mode + * + * The client can select asynchronous or synchronous mode by setting the \p + * enableEncodeAsync field in ::_NV_ENC_INITIALIZE_PARAMS to 1 or 0 respectively. + *\par Asynchronous mode of operation: + * The Asynchronous mode can be enabled by setting NV_ENC_INITIALIZE_PARAMS::enableEncodeAsync to 1. + * The client operating in asynchronous mode must allocate completion event object + * for each output buffer and pass the completion event object in the + * ::NvEncEncodePicture() API. The client can create another thread and wait on + * the event object to be signaled by NvEncodeAPI interface on completion of the + * encoding process for the output frame. This should unblock the main thread from + * submitting work to the encoder. When the event is signaled the client can call + * NvEncodeAPI interfaces to copy the bitstream data using ::NvEncLockBitstream() + * API. This is the preferred mode of operation. + * + * NOTE: Asynchronous mode is not supported on Linux. + * + *\par Synchronous mode of operation: + * The client can select synchronous mode by setting NV_ENC_INITIALIZE_PARAMS::enableEncodeAsync to 0. + * The client working in synchronous mode can work in a single threaded or multi + * threaded mode. The client need not allocate any event objects. The client can + * only lock the bitstream data after NvEncodeAPI interface has returned + * ::NV_ENC_SUCCESS from encode picture. The NvEncodeAPI interface can return + * ::NV_ENC_ERR_NEED_MORE_INPUT error code from ::NvEncEncodePicture() API. The + * client must not lock the output buffer in such case but should send the next + * frame for encoding. The client must keep on calling ::NvEncEncodePicture() API + * until it returns ::NV_ENC_SUCCESS. \n + * The client must always lock the bitstream data in order in which it has submitted. + * This is true for both asynchronous and synchronous mode. + * + *\par Picture type decision: + * If the client is taking the picture type decision and it must disable the picture + * type decision module in NvEncodeAPI by setting NV_ENC_INITIALIZE_PARAMS::enablePTD + * to 0. In this case the client is required to send the picture in encoding + * order to NvEncodeAPI by doing the re-ordering for B frames. \n + * If the client doesn't want to take the picture type decision it can enable + * picture type decision module in the NvEncodeAPI interface by setting + * NV_ENC_INITIALIZE_PARAMS::enablePTD to 1 and send the input pictures in display + * order. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] createEncodeParams + * Refer ::_NV_ENC_INITIALIZE_PARAMS for details. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncInitializeEncoder (void *encoder, NV_ENC_INITIALIZE_PARAMS *createEncodeParams); + + +// NvEncCreateInputBuffer +/** + * \brief Allocates Input buffer. + * + * This function is used to allocate an input buffer. The client must enumerate + * the input buffer format before allocating the input buffer resources. The + * NV_ENC_INPUT_PTR returned by the NvEncodeAPI interface in the + * NV_ENC_CREATE_INPUT_BUFFER::inputBuffer field can be directly used in + * ::NvEncEncodePicture() API. The number of input buffers to be allocated by the + * client must be at least 4 more than the number of B frames being used for encoding. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in,out] createInputBufferParams + * Pointer to the ::NV_ENC_CREATE_INPUT_BUFFER structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncCreateInputBuffer (void *encoder, NV_ENC_CREATE_INPUT_BUFFER *createInputBufferParams); + + +// NvEncDestroyInputBuffer +/** + * \brief Release an input buffers. + * + * This function is used to free an input buffer. If the client has allocated + * any input buffer using ::NvEncCreateInputBuffer() API, it must free those + * input buffers by calling this function. The client must release the input + * buffers before destroying the encoder using ::NvEncDestroyEncoder() API. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] inputBuffer + * Pointer to the input buffer to be released. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncDestroyInputBuffer (void *encoder, NV_ENC_INPUT_PTR inputBuffer); + +// NvEncSetIOCudaStreams +/** + * \brief Set input and output CUDA stream for specified encoder attribute. + * + * Encoding may involve CUDA pre-processing on the input and post-processing on encoded output. + * This function is used to set input and output CUDA streams to pipeline the CUDA pre-processing + * and post-processing tasks. Clients should call this function before the call to + * NvEncUnlockInputBuffer(). If this function is not called, the default CUDA stream is used for + * input and output processing. After a successful call to this function, the streams specified + * in that call will replace the previously-used streams. + * This API is supported for NVCUVID interface only. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] inputStream + * Pointer to CUstream which is used to process ::NV_ENC_PIC_PARAMS::inputFrame for encode. + * In case of ME-only mode, inputStream is used to process ::NV_ENC_MEONLY_PARAMS::inputBuffer and + * ::NV_ENC_MEONLY_PARAMS::referenceFrame + * \param [in] outputStream + * Pointer to CUstream which is used to process ::NV_ENC_PIC_PARAMS::outputBuffer for encode. + * In case of ME-only mode, outputStream is used to process ::NV_ENC_MEONLY_PARAMS::mvBuffer + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_GENERIC \n + */ +NVENCSTATUS NVENCAPI NvEncSetIOCudaStreams (void *encoder, NV_ENC_CUSTREAM_PTR inputStream, NV_ENC_CUSTREAM_PTR outputStream); + + +// NvEncCreateBitstreamBuffer +/** + * \brief Allocates an output bitstream buffer + * + * This function is used to allocate an output bitstream buffer and returns a + * NV_ENC_OUTPUT_PTR to bitstream buffer to the client in the + * NV_ENC_CREATE_BITSTREAM_BUFFER::bitstreamBuffer field. + * The client can only call this function after the encoder session has been + * initialized using ::NvEncInitializeEncoder() API. The minimum number of output + * buffers allocated by the client must be at least 4 more than the number of B + * B frames being used for encoding. The client can only access the output + * bitstream data by locking the \p bitstreamBuffer using the ::NvEncLockBitstream() + * function. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in,out] createBitstreamBufferParams + * Pointer ::NV_ENC_CREATE_BITSTREAM_BUFFER for details. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncCreateBitstreamBuffer (void *encoder, NV_ENC_CREATE_BITSTREAM_BUFFER *createBitstreamBufferParams); + + +// NvEncDestroyBitstreamBuffer +/** + * \brief Release a bitstream buffer. + * + * This function is used to release the output bitstream buffer allocated using + * the ::NvEncCreateBitstreamBuffer() function. The client must release the output + * bitstreamBuffer using this function before destroying the encoder session. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] bitstreamBuffer + * Pointer to the bitstream buffer being released. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncDestroyBitstreamBuffer (void *encoder, NV_ENC_OUTPUT_PTR bitstreamBuffer); + +// NvEncEncodePicture +/** + * \brief Submit an input picture for encoding. + * + * This function is used to submit an input picture buffer for encoding. The + * encoding parameters are passed using \p *encodePicParams which is a pointer + * to the ::_NV_ENC_PIC_PARAMS structure. + * + * If the client has set NV_ENC_INITIALIZE_PARAMS::enablePTD to 0, then it must + * send a valid value for the following fields. + * - NV_ENC_PIC_PARAMS::pictureType + * - NV_ENC_PIC_PARAMS_H264::displayPOCSyntax (H264 only) + * - NV_ENC_PIC_PARAMS_H264::frameNumSyntax(H264 only) + * - NV_ENC_PIC_PARAMS_H264::refPicFlag(H264 only) + * + *\par MVC Encoding: + * For MVC encoding the client must call encode picture API for each view separately + * and must pass valid view id in NV_ENC_PIC_PARAMS_MVC::viewID field. Currently + * NvEncodeAPI only support stereo MVC so client must send viewID as 0 for base + * view and view ID as 1 for dependent view. + * + *\par Asynchronous Encoding + * If the client has enabled asynchronous mode of encoding by setting + * NV_ENC_INITIALIZE_PARAMS::enableEncodeAsync to 1 in the ::NvEncInitializeEncoder() + * API ,then the client must send a valid NV_ENC_PIC_PARAMS::completionEvent. + * Incase of asynchronous mode of operation, client can queue the ::NvEncEncodePicture() + * API commands from the main thread and then queue output buffers to be processed + * to a secondary worker thread. Before the locking the output buffers in the + * secondary thread , the client must wait on NV_ENC_PIC_PARAMS::completionEvent + * it has queued in ::NvEncEncodePicture() API call. The client must always process + * completion event and the output buffer in the same order in which they have been + * submitted for encoding. The NvEncodeAPI interface is responsible for any + * re-ordering required for B frames and will always ensure that encoded bitstream + * data is written in the same order in which output buffer is submitted. + * The NvEncodeAPI interface may return ::NV_ENC_ERR_NEED_MORE_INPUT error code for + * some ::NvEncEncodePicture() API calls but the client must not treat it as a fatal error. + * The NvEncodeAPI interface might not be able to submit an input picture buffer for encoding + * immediately due to re-ordering for B frames. + *\code + The below example shows how asynchronous encoding in case of 1 B frames + ------------------------------------------------------------------------ + Suppose the client allocated 4 input buffers(I1,I2..), 4 output buffers(O1,O2..) + and 4 completion events(E1, E2, ...). The NvEncodeAPI interface will need to + keep a copy of the input buffers for re-ordering and it allocates following + internal buffers (NvI1, NvI2...). These internal buffers are managed by NvEncodeAPI + and the client is not responsible for the allocating or freeing the memory of + the internal buffers. + + a) The client main thread will queue the following encode frame calls. + Note the picture type is unknown to the client, the decision is being taken by + NvEncodeAPI interface. The client should pass ::_NV_ENC_PIC_PARAMS parameter + consisting of allocated input buffer, output buffer and output events in successive + ::NvEncEncodePicture() API calls along with other required encode picture params. + For example: + 1st EncodePicture parameters - (I1, O1, E1) + 2nd EncodePicture parameters - (I2, O2, E2) + 3rd EncodePicture parameters - (I3, O3, E3) + + b) NvEncodeAPI SW will receive the following encode Commands from the client. + The left side shows input from client in the form (Input buffer, Output Buffer, + Output Event). The right hand side shows a possible picture type decision take by + the NvEncodeAPI interface. + (I1, O1, E1) ---P1 Frame + (I2, O2, E2) ---B2 Frame + (I3, O3, E3) ---P3 Frame + + c) NvEncodeAPI interface will make a copy of the input buffers to its internal + buffers for re-ordering. These copies are done as part of nvEncEncodePicture + function call from the client and NvEncodeAPI interface is responsible for + synchronization of copy operation with the actual encoding operation. + I1 --> NvI1 + I2 --> NvI2 + I3 --> NvI3 + + d) The NvEncodeAPI encodes I1 as P frame and submits I1 to encoder HW and returns ::NV_ENC_SUCCESS. + The NvEncodeAPI tries to encode I2 as B frame and fails with ::NV_ENC_ERR_NEED_MORE_INPUT error code. + The error is not fatal and it notifies client that I2 is not submitted to encoder immediately. + The NvEncodeAPI encodes I3 as P frame and submits I3 for encoding which will be used as backward + reference frame for I2. The NvEncodeAPI then submits I2 for encoding and returns ::NV_ENC_SUCESS. + Both the submission are part of the same ::NvEncEncodePicture() function call. + + e) After returning from ::NvEncEncodePicture() call , the client must queue the output + bitstream processing work to the secondary thread. The output bitstream processing + for asynchronous mode consist of first waiting on completion event(E1, E2..) + and then locking the output bitstream buffer(O1, O2..) for reading the encoded + data. The work queued to the secondary thread by the client is in the following order + (I1, O1, E1) + (I2, O2, E2) + (I3, O3, E3) + Note they are in the same order in which client calls ::NvEncEncodePicture() API + in \p step a). + + f) NvEncodeAPI interface will do the re-ordering such that Encoder HW will receive + the following encode commands: + (NvI1, O1, E1) ---P1 Frame + (NvI3, O2, E2) ---P3 Frame + (NvI2, O3, E3) ---B2 frame + + g) After the encoding operations are completed, the events will be signaled + by NvEncodeAPI interface in the following order : + (O1, E1) ---P1 Frame ,output bitstream copied to O1 and event E1 signaled. + (O2, E2) ---P3 Frame ,output bitstream copied to O2 and event E2 signaled. + (O3, E3) ---B2 Frame ,output bitstream copied to O3 and event E3 signaled. + + h) The client must lock the bitstream data using ::NvEncLockBitstream() API in + the order O1,O2,O3 to read the encoded data, after waiting for the events + to be signaled in the same order i.e E1, E2 and E3.The output processing is + done in the secondary thread in the following order: + Waits on E1, copies encoded bitstream from O1 + Waits on E2, copies encoded bitstream from O2 + Waits on E3, copies encoded bitstream from O3 + + -Note the client will receive the events signaling and output buffer in the + same order in which they have submitted for encoding. + -Note the LockBitstream will have picture type field which will notify the + output picture type to the clients. + -Note the input, output buffer and the output completion event are free to be + reused once NvEncodeAPI interfaced has signaled the event and the client has + copied the data from the output buffer. + + * \endcode + * + *\par Synchronous Encoding + * The client can enable synchronous mode of encoding by setting + * NV_ENC_INITIALIZE_PARAMS::enableEncodeAsync to 0 in ::NvEncInitializeEncoder() API. + * The NvEncodeAPI interface may return ::NV_ENC_ERR_NEED_MORE_INPUT error code for + * some ::NvEncEncodePicture() API calls when NV_ENC_INITIALIZE_PARAMS::enablePTD + * is set to 1, but the client must not treat it as a fatal error. The NvEncodeAPI + * interface might not be able to submit an input picture buffer for encoding + * immediately due to re-ordering for B frames. The NvEncodeAPI interface cannot + * submit the input picture which is decided to be encoded as B frame as it waits + * for backward reference from temporally subsequent frames. This input picture + * is buffered internally and waits for more input picture to arrive. The client + * must not call ::NvEncLockBitstream() API on the output buffers whose + * ::NvEncEncodePicture() API returns ::NV_ENC_ERR_NEED_MORE_INPUT. The client must + * wait for the NvEncodeAPI interface to return ::NV_ENC_SUCCESS before locking the + * output bitstreams to read the encoded bitstream data. The following example + * explains the scenario with synchronous encoding with 2 B frames. + *\code + The below example shows how synchronous encoding works in case of 1 B frames + ----------------------------------------------------------------------------- + Suppose the client allocated 4 input buffers(I1,I2..), 4 output buffers(O1,O2..) + and 4 completion events(E1, E2, ...). The NvEncodeAPI interface will need to + keep a copy of the input buffers for re-ordering and it allocates following + internal buffers (NvI1, NvI2...). These internal buffers are managed by NvEncodeAPI + and the client is not responsible for the allocating or freeing the memory of + the internal buffers. + + The client calls ::NvEncEncodePicture() API with input buffer I1 and output buffer O1. + The NvEncodeAPI decides to encode I1 as P frame and submits it to encoder + HW and returns ::NV_ENC_SUCCESS. + The client can now read the encoded data by locking the output O1 by calling + NvEncLockBitstream API. + + The client calls ::NvEncEncodePicture() API with input buffer I2 and output buffer O2. + The NvEncodeAPI decides to encode I2 as B frame and buffers I2 by copying it + to internal buffer and returns ::NV_ENC_ERR_NEED_MORE_INPUT. + The error is not fatal and it notifies client that it cannot read the encoded + data by locking the output O2 by calling ::NvEncLockBitstream() API without submitting + more work to the NvEncodeAPI interface. + + The client calls ::NvEncEncodePicture() with input buffer I3 and output buffer O3. + The NvEncodeAPI decides to encode I3 as P frame and it first submits I3 for + encoding which will be used as backward reference frame for I2. + The NvEncodeAPI then submits I2 for encoding and returns ::NV_ENC_SUCESS. Both + the submission are part of the same ::NvEncEncodePicture() function call. + The client can now read the encoded data for both the frames by locking the output + O2 followed by O3 ,by calling ::NvEncLockBitstream() API. + + The client must always lock the output in the same order in which it has submitted + to receive the encoded bitstream in correct encoding order. + + * \endcode + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in,out] encodePicParams + * Pointer to the ::_NV_ENC_PIC_PARAMS structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_ENCODER_BUSY \n + * ::NV_ENC_ERR_NEED_MORE_INPUT \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncEncodePicture (void *encoder, NV_ENC_PIC_PARAMS *encodePicParams); + + +// NvEncLockBitstream +/** + * \brief Lock output bitstream buffer + * + * This function is used to lock the bitstream buffer to read the encoded data. + * The client can only access the encoded data by calling this function. + * The pointer to client accessible encoded data is returned in the + * NV_ENC_LOCK_BITSTREAM::bitstreamBufferPtr field. The size of the encoded data + * in the output buffer is returned in the NV_ENC_LOCK_BITSTREAM::bitstreamSizeInBytes + * The NvEncodeAPI interface also returns the output picture type and picture structure + * of the encoded frame in NV_ENC_LOCK_BITSTREAM::pictureType and + * NV_ENC_LOCK_BITSTREAM::pictureStruct fields respectively. If the client has + * set NV_ENC_LOCK_BITSTREAM::doNotWait to 1, the function might return + * ::NV_ENC_ERR_LOCK_BUSY if client is operating in synchronous mode. This is not + * a fatal failure if NV_ENC_LOCK_BITSTREAM::doNotWait is set to 1. In the above case the client can + * retry the function after few milliseconds. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in,out] lockBitstreamBufferParams + * Pointer to the ::_NV_ENC_LOCK_BITSTREAM structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_LOCK_BUSY \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncLockBitstream (void *encoder, NV_ENC_LOCK_BITSTREAM *lockBitstreamBufferParams); + + +// NvEncUnlockBitstream +/** + * \brief Unlock the output bitstream buffer + * + * This function is used to unlock the output bitstream buffer after the client + * has read the encoded data from output buffer. The client must call this function + * to unlock the output buffer which it has previously locked using ::NvEncLockBitstream() + * function. Using a locked bitstream buffer in ::NvEncEncodePicture() API will cause + * the function to fail. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in,out] bitstreamBuffer + * bitstream buffer pointer being unlocked + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncUnlockBitstream (void *encoder, NV_ENC_OUTPUT_PTR bitstreamBuffer); + +// NvEncRestoreEncoderState +/** + * \brief Restore state of encoder + * + * This function is used to restore the state of encoder with state saved internally in + * state buffer corresponding to index equal to 'NV_ENC_RESTORE_ENCODER_STATE_PARAMS::bfrIndex'. + * Client can specify the state type to be updated by specifying appropriate value in + * 'NV_ENC_RESTORE_ENCODER_STATE_PARAMS::state'. The client must call this + * function after all previous encodes have finished. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] restoreState + * Pointer to the ::_NV_ENC_RESTORE_ENCODER_STATE_PARAMS structure + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncRestoreEncoderState (void *encoder, NV_ENC_RESTORE_ENCODER_STATE_PARAMS *restoreState); + +// NvLockInputBuffer +/** + * \brief Locks an input buffer + * + * This function is used to lock the input buffer to load the uncompressed YUV + * pixel data into input buffer memory. The client must pass the NV_ENC_INPUT_PTR + * it had previously allocated using ::NvEncCreateInputBuffer()in the + * NV_ENC_LOCK_INPUT_BUFFER::inputBuffer field. + * The NvEncodeAPI interface returns pointer to client accessible input buffer + * memory in NV_ENC_LOCK_INPUT_BUFFER::bufferDataPtr field. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in,out] lockInputBufferParams + * Pointer to the ::_NV_ENC_LOCK_INPUT_BUFFER structure + * + * \return + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_LOCK_BUSY \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncLockInputBuffer (void *encoder, NV_ENC_LOCK_INPUT_BUFFER *lockInputBufferParams); + + +// NvUnlockInputBuffer +/** + * \brief Unlocks the input buffer + * + * This function is used to unlock the input buffer memory previously locked for + * uploading YUV pixel data. The input buffer must be unlocked before being used + * again for encoding, otherwise NvEncodeAPI will fail the ::NvEncEncodePicture() + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] inputBuffer + * Pointer to the input buffer that is being unlocked. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + * + */ +NVENCSTATUS NVENCAPI NvEncUnlockInputBuffer (void *encoder, NV_ENC_INPUT_PTR inputBuffer); + + +// NvEncGetEncodeStats +/** + * \brief Get encoding statistics. + * + * This function is used to retrieve the encoding statistics. + * This API is not supported when encode device type is CUDA. + * Note that this API will be removed in future Video Codec SDK release. + * Clients should use NvEncLockBitstream() API to retrieve the encoding statistics. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in,out] encodeStats + * Pointer to the ::_NV_ENC_STAT structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncGetEncodeStats (void *encoder, NV_ENC_STAT *encodeStats); + + +// NvEncGetSequenceParams +/** + * \brief Get encoded sequence and picture header. + * + * This function can be used to retrieve the sequence and picture header out of + * band. The client must call this function only after the encoder has been + * initialized using ::NvEncInitializeEncoder() function. The client must + * allocate the memory where the NvEncodeAPI interface can copy the bitstream + * header and pass the pointer to the memory in NV_ENC_SEQUENCE_PARAM_PAYLOAD::spsppsBuffer. + * The size of buffer is passed in the field NV_ENC_SEQUENCE_PARAM_PAYLOAD::inBufferSize. + * The NvEncodeAPI interface will copy the bitstream header payload and returns + * the actual size of the bitstream header in the field + * NV_ENC_SEQUENCE_PARAM_PAYLOAD::outSPSPPSPayloadSize. + * The client must call ::NvEncGetSequenceParams() function from the same thread which is + * being used to call ::NvEncEncodePicture() function. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in,out] sequenceParamPayload + * Pointer to the ::_NV_ENC_SEQUENCE_PARAM_PAYLOAD structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + */NVENCSTATUS NVENCAPI NvEncGetSequenceParams (void *encoder, NV_ENC_SEQUENCE_PARAM_PAYLOAD *sequenceParamPayload); + + +// NvEncGetSequenceParamEx +/** + * \brief Get sequence and picture header. + * + * This function can be used to retrieve the sequence and picture header out of band, even when + * encoder has not been initialized using ::NvEncInitializeEncoder() function. + * The client must allocate the memory where the NvEncodeAPI interface can copy the bitstream + * header and pass the pointer to the memory in NV_ENC_SEQUENCE_PARAM_PAYLOAD::spsppsBuffer. + * The size of buffer is passed in the field NV_ENC_SEQUENCE_PARAM_PAYLOAD::inBufferSize. + * If encoder has not been initialized using ::NvEncInitializeEncoder() function, client must + * send NV_ENC_INITIALIZE_PARAMS as input. The NV_ENC_INITIALIZE_PARAMS passed must be same as the + * one which will be used for initializing encoder using ::NvEncInitializeEncoder() function later. + * If encoder is already initialized using ::NvEncInitializeEncoder() function, the provided + * NV_ENC_INITIALIZE_PARAMS structure is ignored. The NvEncodeAPI interface will copy the bitstream + * header payload and returns the actual size of the bitstream header in the field + * NV_ENC_SEQUENCE_PARAM_PAYLOAD::outSPSPPSPayloadSize. The client must call ::NvEncGetSequenceParamsEx() + * function from the same thread which is being used to call ::NvEncEncodePicture() function. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] encInitParams + * Pointer to the _NV_ENC_INITIALIZE_PARAMS structure. + * \param [in,out] sequenceParamPayload + * Pointer to the ::_NV_ENC_SEQUENCE_PARAM_PAYLOAD structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncGetSequenceParamEx (void *encoder, NV_ENC_INITIALIZE_PARAMS *encInitParams, NV_ENC_SEQUENCE_PARAM_PAYLOAD *sequenceParamPayload); + +// NvEncRegisterAsyncEvent +/** + * \brief Register event for notification to encoding completion. + * + * This function is used to register the completion event with NvEncodeAPI + * interface. The event is required when the client has configured the encoder to + * work in asynchronous mode. In this mode the client needs to send a completion + * event with every output buffer. The NvEncodeAPI interface will signal the + * completion of the encoding process using this event. Only after the event is + * signaled the client can get the encoded data using ::NvEncLockBitstream() function. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] eventParams + * Pointer to the ::_NV_ENC_EVENT_PARAMS structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncRegisterAsyncEvent (void *encoder, NV_ENC_EVENT_PARAMS *eventParams); + + +// NvEncUnregisterAsyncEvent +/** + * \brief Unregister completion event. + * + * This function is used to unregister completion event which has been previously + * registered using ::NvEncRegisterAsyncEvent() function. The client must unregister + * all events before destroying the encoder using ::NvEncDestroyEncoder() function. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] eventParams + * Pointer to the ::_NV_ENC_EVENT_PARAMS structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncUnregisterAsyncEvent (void *encoder, NV_ENC_EVENT_PARAMS *eventParams); + + +// NvEncMapInputResource +/** + * \brief Map an externally created input resource pointer for encoding. + * + * Maps an externally allocated input resource [using and returns a NV_ENC_INPUT_PTR + * which can be used for encoding in the ::NvEncEncodePicture() function. The + * mapped resource is returned in the field NV_ENC_MAP_INPUT_RESOURCE::outputResourcePtr. + * The NvEncodeAPI interface also returns the buffer format of the mapped resource + * in the field NV_ENC_MAP_INPUT_RESOURCE::outbufferFmt. + * This function provides synchronization guarantee that any graphics work submitted + * on the input buffer is completed before the buffer is used for encoding. This is + * also true for compute (i.e. CUDA) work, provided that the previous workload using + * the input resource was submitted to the default stream. + * The client should not access any input buffer while they are mapped by the encoder. + * For D3D12 interface type, this function does not provide synchronization guarantee. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in,out] mapInputResParams + * Pointer to the ::_NV_ENC_MAP_INPUT_RESOURCE structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_RESOURCE_NOT_REGISTERED \n + * ::NV_ENC_ERR_MAP_FAILED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncMapInputResource (void *encoder, NV_ENC_MAP_INPUT_RESOURCE *mapInputResParams); + + +// NvEncUnmapInputResource +/** + * \brief UnMaps a NV_ENC_INPUT_PTR which was mapped for encoding + * + * + * UnMaps an input buffer which was previously mapped using ::NvEncMapInputResource() + * API. The mapping created using ::NvEncMapInputResource() should be invalidated + * using this API before the external resource is destroyed by the client. The client + * must unmap the buffer after ::NvEncLockBitstream() API returns successfully for encode + * work submitted using the mapped input buffer. + * + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] mappedInputBuffer + * Pointer to the NV_ENC_INPUT_PTR + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_RESOURCE_NOT_REGISTERED \n + * ::NV_ENC_ERR_RESOURCE_NOT_MAPPED \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncUnmapInputResource (void *encoder, NV_ENC_INPUT_PTR mappedInputBuffer); + +// NvEncDestroyEncoder +/** + * \brief Destroy Encoding Session + * + * Destroys the encoder session previously created using ::NvEncOpenEncodeSession() + * function. The client must flush the encoder before freeing any resources. In order + * to flush the encoder the client must pass a NULL encode picture packet and either + * wait for the ::NvEncEncodePicture() function to return in synchronous mode or wait + * for the flush event to be signaled by the encoder in asynchronous mode. + * The client must free all the input and output resources created using the + * NvEncodeAPI interface before destroying the encoder. If the client is operating + * in asynchronous mode, it must also unregister the completion events previously + * registered. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncDestroyEncoder (void *encoder); + +// NvEncInvalidateRefFrames +/** + * \brief Invalidate reference frames + * + * Invalidates reference frame based on the time stamp provided by the client. + * The encoder marks any reference frames or any frames which have been reconstructed + * using the corrupt frame as invalid for motion estimation and uses older reference + * frames for motion estimation. The encoder forces the current frame to be encoded + * as an intra frame if no reference frames are left after invalidation process. + * This is useful for low latency application for error resiliency. The client + * is recommended to set NV_ENC_CONFIG_H264::maxNumRefFrames to a large value so + * that encoder can keep a backup of older reference frames in the DPB and can use them + * for motion estimation when the newer reference frames have been invalidated. + * This API can be called multiple times. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] invalidRefFrameTimeStamp + * Timestamp of the invalid reference frames which needs to be invalidated. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncInvalidateRefFrames(void *encoder, uint64_t invalidRefFrameTimeStamp); + +// NvEncOpenEncodeSessionEx +/** + * \brief Opens an encoding session. + * + * Opens an encoding session and returns a pointer to the encoder interface in + * the \p **encoder parameter. The client should start encoding process by calling + * this API first. + * The client must pass a pointer to IDirect3DDevice9 device or CUDA context in the \p *device parameter. + * For the OpenGL interface, \p device must be NULL. An OpenGL context must be current when + * calling all NvEncodeAPI functions. + * If the creation of encoder session fails, the client must call ::NvEncDestroyEncoder API + * before exiting. + * + * \param [in] openSessionExParams + * Pointer to a ::NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS structure. + * \param [out] encoder + * Encode Session pointer to the NvEncodeAPI interface. + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_NO_ENCODE_DEVICE \n + * ::NV_ENC_ERR_UNSUPPORTED_DEVICE \n + * ::NV_ENC_ERR_INVALID_DEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncOpenEncodeSessionEx (NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS *openSessionExParams, void **encoder); + +// NvEncRegisterResource +/** + * \brief Registers a resource with the Nvidia Video Encoder Interface. + * + * Registers a resource with the Nvidia Video Encoder Interface for book keeping. + * The client is expected to pass the registered resource handle as well, while calling ::NvEncMapInputResource API. + * + * \param [in] encoder + * Pointer to the NVEncodeAPI interface. + * + * \param [in] registerResParams + * Pointer to a ::_NV_ENC_REGISTER_RESOURCE structure + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_RESOURCE_REGISTER_FAILED \n + * ::NV_ENC_ERR_GENERIC \n + * ::NV_ENC_ERR_UNIMPLEMENTED \n + * + */ +NVENCSTATUS NVENCAPI NvEncRegisterResource (void *encoder, NV_ENC_REGISTER_RESOURCE *registerResParams); + +// NvEncUnregisterResource +/** + * \brief Unregisters a resource previously registered with the Nvidia Video Encoder Interface. + * + * Unregisters a resource previously registered with the Nvidia Video Encoder Interface. + * The client is expected to unregister any resource that it has registered with the + * Nvidia Video Encoder Interface before destroying the resource. + * + * \param [in] encoder + * Pointer to the NVEncodeAPI interface. + * + * \param [in] registeredResource + * The registered resource pointer that was returned in ::NvEncRegisterResource. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_RESOURCE_NOT_REGISTERED \n + * ::NV_ENC_ERR_GENERIC \n + * ::NV_ENC_ERR_UNIMPLEMENTED \n + * + */ +NVENCSTATUS NVENCAPI NvEncUnregisterResource (void *encoder, NV_ENC_REGISTERED_PTR registeredResource); + +// NvEncReconfigureEncoder +/** + * \brief Reconfigure an existing encoding session. + * + * Reconfigure an existing encoding session. + * The client should call this API to change/reconfigure the parameter passed during + * NvEncInitializeEncoder API call. + * Currently Reconfiguration of following are not supported. + * Change in GOP structure. + * Change in sync-Async mode. + * Change in MaxWidth & MaxHeight. + * Change in PTD mode. + * + * Resolution change is possible only if maxEncodeWidth & maxEncodeHeight of NV_ENC_INITIALIZE_PARAMS + * is set while creating encoder session. + * + * \param [in] encoder + * Pointer to the NVEncodeAPI interface. + * + * \param [in] reInitEncodeParams + * Pointer to a ::NV_ENC_RECONFIGURE_PARAMS structure. + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_NO_ENCODE_DEVICE \n + * ::NV_ENC_ERR_UNSUPPORTED_DEVICE \n + * ::NV_ENC_ERR_INVALID_DEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_GENERIC \n + * + */ +NVENCSTATUS NVENCAPI NvEncReconfigureEncoder (void *encoder, NV_ENC_RECONFIGURE_PARAMS *reInitEncodeParams); + + + +// NvEncCreateMVBuffer +/** + * \brief Allocates output MV buffer for ME only mode. + * + * This function is used to allocate an output MV buffer. The size of the mvBuffer is + * dependent on the frame height and width of the last ::NvEncCreateInputBuffer() call. + * The NV_ENC_OUTPUT_PTR returned by the NvEncodeAPI interface in the + * ::NV_ENC_CREATE_MV_BUFFER::mvBuffer field should be used in + * ::NvEncRunMotionEstimationOnly() API. + * Client must lock ::NV_ENC_CREATE_MV_BUFFER::mvBuffer using ::NvEncLockBitstream() API to get the motion vector data. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in,out] createMVBufferParams + * Pointer to the ::NV_ENC_CREATE_MV_BUFFER structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_GENERIC \n + */ +NVENCSTATUS NVENCAPI NvEncCreateMVBuffer (void *encoder, NV_ENC_CREATE_MV_BUFFER *createMVBufferParams); + + +// NvEncDestroyMVBuffer +/** + * \brief Release an output MV buffer for ME only mode. + * + * This function is used to release the output MV buffer allocated using + * the ::NvEncCreateMVBuffer() function. The client must release the output + * mvBuffer using this function before destroying the encoder session. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] mvBuffer + * Pointer to the mvBuffer being released. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + */ +NVENCSTATUS NVENCAPI NvEncDestroyMVBuffer (void *encoder, NV_ENC_OUTPUT_PTR mvBuffer); + + +// NvEncRunMotionEstimationOnly +/** + * \brief Submit an input picture and reference frame for motion estimation in ME only mode. + * + * This function is used to submit the input frame and reference frame for motion + * estimation. The ME parameters are passed using *meOnlyParams which is a pointer + * to ::_NV_ENC_MEONLY_PARAMS structure. + * Client must lock ::NV_ENC_CREATE_MV_BUFFER::mvBuffer using ::NvEncLockBitstream() API to get the motion vector data. + * to get motion vector data. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] meOnlyParams + * Pointer to the ::_NV_ENC_MEONLY_PARAMS structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + * ::NV_ENC_ERR_NEED_MORE_INPUT \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + */ +NVENCSTATUS NVENCAPI NvEncRunMotionEstimationOnly (void *encoder, NV_ENC_MEONLY_PARAMS *meOnlyParams); + +// NvEncodeAPIGetMaxSupportedVersion +/** + * \brief Get the largest NvEncodeAPI version supported by the driver. + * + * This function can be used by clients to determine if the driver supports + * the NvEncodeAPI header the application was compiled with. + * + * \param [out] version + * Pointer to the requested value. The 4 least significant bits in the returned + * indicate the minor version and the rest of the bits indicate the major + * version of the largest supported version. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_ERR_INVALID_PTR \n + */ +NVENCSTATUS NVENCAPI NvEncodeAPIGetMaxSupportedVersion (uint32_t *version); + + +// NvEncGetLastErrorString +/** + * \brief Get the description of the last error reported by the API. + * + * This function returns a null-terminated string that can be used by clients to better understand the reason + * for failure of a previous API call. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * + * \return + * Pointer to buffer containing the details of the last error encountered by the API. + */ +const char *NVENCAPI NvEncGetLastErrorString (void *encoder); + +// NvEncLookaheadPicture +/** + * \brief Submit an input picture for lookahead. + * + * This function can be used by clients to submit input frame for lookahead. Client could call this function + * NV_ENC_INITIALIZE_PARAMS::lookaheadDepth plus one number of frames, before calling NvEncEncodePicture() for the first frame. + * + * \param [in] encoder + * Pointer to the NvEncodeAPI interface. + * \param [in] lookaheadParams + * Pointer to the ::_NV_ENC_LOOKAHEAD_PIC_PARAMS structure. + * + * \return + * ::NV_ENC_SUCCESS \n + * ::NV_ENC_NEED_MORE_INPUT \n should we return this error is lookahead queue is not full? + * ::NV_ENC_ERR_INVALID_PTR \n + * ::NV_ENC_ERR_ENCODER_NOT_INITIALIZED \n + * ::NV_ENC_ERR_GENERIC \n + * ::NV_ENC_ERR_INVALID_ENCODERDEVICE \n + * ::NV_ENC_ERR_DEVICE_NOT_EXIST \n + * ::NV_ENC_ERR_UNSUPPORTED_PARAM \n + * ::NV_ENC_ERR_OUT_OF_MEMORY \n + * ::NV_ENC_ERR_INVALID_PARAM \n + * ::NV_ENC_ERR_INVALID_VERSION \n + */ +NVENCSTATUS NVENCAPI NvEncLookaheadPicture (void *encoder, NV_ENC_LOOKAHEAD_PIC_PARAMS *lookaheadParamas); + +/// \cond API PFN +/* + * Defines API function pointers + */ +typedef NVENCSTATUS (NVENCAPI *PNVENCOPENENCODESESSION) (void *device, uint32_t deviceType, void **encoder); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETENCODEGUIDCOUNT) (void *encoder, uint32_t *encodeGUIDCount); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETENCODEGUIDS) (void *encoder, GUID *GUIDs, uint32_t guidArraySize, uint32_t *GUIDCount); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETENCODEPROFILEGUIDCOUNT) (void *encoder, GUID encodeGUID, uint32_t *encodeProfileGUIDCount); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETENCODEPROFILEGUIDS) (void *encoder, GUID encodeGUID, GUID *profileGUIDs, uint32_t guidArraySize, uint32_t *GUIDCount); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETINPUTFORMATCOUNT) (void *encoder, GUID encodeGUID, uint32_t *inputFmtCount); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETINPUTFORMATS) (void *encoder, GUID encodeGUID, NV_ENC_BUFFER_FORMAT *inputFmts, uint32_t inputFmtArraySize, uint32_t *inputFmtCount); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETENCODECAPS) (void *encoder, GUID encodeGUID, NV_ENC_CAPS_PARAM *capsParam, int *capsVal); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETENCODEPRESETCOUNT) (void *encoder, GUID encodeGUID, uint32_t *encodePresetGUIDCount); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETENCODEPRESETGUIDS) (void *encoder, GUID encodeGUID, GUID *presetGUIDs, uint32_t guidArraySize, uint32_t *encodePresetGUIDCount); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETENCODEPRESETCONFIG) (void *encoder, GUID encodeGUID, GUID presetGUID, NV_ENC_PRESET_CONFIG *presetConfig); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETENCODEPRESETCONFIGEX) (void *encoder, GUID encodeGUID, GUID presetGUID, NV_ENC_TUNING_INFO tuningInfo, NV_ENC_PRESET_CONFIG *presetConfig); +typedef NVENCSTATUS (NVENCAPI *PNVENCINITIALIZEENCODER) (void *encoder, NV_ENC_INITIALIZE_PARAMS *createEncodeParams); +typedef NVENCSTATUS (NVENCAPI *PNVENCCREATEINPUTBUFFER) (void *encoder, NV_ENC_CREATE_INPUT_BUFFER *createInputBufferParams); +typedef NVENCSTATUS (NVENCAPI *PNVENCDESTROYINPUTBUFFER) (void *encoder, NV_ENC_INPUT_PTR inputBuffer); +typedef NVENCSTATUS (NVENCAPI *PNVENCCREATEBITSTREAMBUFFER) (void *encoder, NV_ENC_CREATE_BITSTREAM_BUFFER *createBitstreamBufferParams); +typedef NVENCSTATUS (NVENCAPI *PNVENCDESTROYBITSTREAMBUFFER) (void *encoder, NV_ENC_OUTPUT_PTR bitstreamBuffer); +typedef NVENCSTATUS (NVENCAPI *PNVENCENCODEPICTURE) (void *encoder, NV_ENC_PIC_PARAMS *encodePicParams); +typedef NVENCSTATUS (NVENCAPI *PNVENCLOCKBITSTREAM) (void *encoder, NV_ENC_LOCK_BITSTREAM *lockBitstreamBufferParams); +typedef NVENCSTATUS (NVENCAPI *PNVENCUNLOCKBITSTREAM) (void *encoder, NV_ENC_OUTPUT_PTR bitstreamBuffer); +typedef NVENCSTATUS (NVENCAPI *PNVENCLOCKINPUTBUFFER) (void *encoder, NV_ENC_LOCK_INPUT_BUFFER *lockInputBufferParams); +typedef NVENCSTATUS (NVENCAPI *PNVENCUNLOCKINPUTBUFFER) (void *encoder, NV_ENC_INPUT_PTR inputBuffer); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETENCODESTATS) (void *encoder, NV_ENC_STAT *encodeStats); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETSEQUENCEPARAMS) (void *encoder, NV_ENC_SEQUENCE_PARAM_PAYLOAD *sequenceParamPayload); +typedef NVENCSTATUS (NVENCAPI *PNVENCREGISTERASYNCEVENT) (void *encoder, NV_ENC_EVENT_PARAMS *eventParams); +typedef NVENCSTATUS (NVENCAPI *PNVENCUNREGISTERASYNCEVENT) (void *encoder, NV_ENC_EVENT_PARAMS *eventParams); +typedef NVENCSTATUS (NVENCAPI *PNVENCMAPINPUTRESOURCE) (void *encoder, NV_ENC_MAP_INPUT_RESOURCE *mapInputResParams); +typedef NVENCSTATUS (NVENCAPI *PNVENCUNMAPINPUTRESOURCE) (void *encoder, NV_ENC_INPUT_PTR mappedInputBuffer); +typedef NVENCSTATUS (NVENCAPI *PNVENCDESTROYENCODER) (void *encoder); +typedef NVENCSTATUS (NVENCAPI *PNVENCINVALIDATEREFFRAMES) (void *encoder, uint64_t invalidRefFrameTimeStamp); +typedef NVENCSTATUS (NVENCAPI *PNVENCOPENENCODESESSIONEX) (NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS *openSessionExParams, void **encoder); +typedef NVENCSTATUS (NVENCAPI *PNVENCREGISTERRESOURCE) (void *encoder, NV_ENC_REGISTER_RESOURCE *registerResParams); +typedef NVENCSTATUS (NVENCAPI *PNVENCUNREGISTERRESOURCE) (void *encoder, NV_ENC_REGISTERED_PTR registeredRes); +typedef NVENCSTATUS (NVENCAPI *PNVENCRECONFIGUREENCODER) (void *encoder, NV_ENC_RECONFIGURE_PARAMS *reInitEncodeParams); + +typedef NVENCSTATUS (NVENCAPI *PNVENCCREATEMVBUFFER) (void *encoder, NV_ENC_CREATE_MV_BUFFER *createMVBufferParams); +typedef NVENCSTATUS (NVENCAPI *PNVENCDESTROYMVBUFFER) (void *encoder, NV_ENC_OUTPUT_PTR mvBuffer); +typedef NVENCSTATUS (NVENCAPI *PNVENCRUNMOTIONESTIMATIONONLY) (void *encoder, NV_ENC_MEONLY_PARAMS *meOnlyParams); +typedef const char *(NVENCAPI *PNVENCGETLASTERROR) (void *encoder); +typedef NVENCSTATUS (NVENCAPI *PNVENCSETIOCUDASTREAMS) (void *encoder, NV_ENC_CUSTREAM_PTR inputStream, NV_ENC_CUSTREAM_PTR outputStream); +typedef NVENCSTATUS (NVENCAPI *PNVENCGETSEQUENCEPARAMEX) (void *encoder, NV_ENC_INITIALIZE_PARAMS *encInitParams, NV_ENC_SEQUENCE_PARAM_PAYLOAD *sequenceParamPayload); +typedef NVENCSTATUS (NVENCAPI *PNVENCRESTOREENCODERSTATE) (void *encoder, NV_ENC_RESTORE_ENCODER_STATE_PARAMS *restoreState); +typedef NVENCSTATUS (NVENCAPI *PNVENCLOOKAHEADPICTURE) (void *encoder, NV_ENC_LOOKAHEAD_PIC_PARAMS *lookaheadParams); + +/// \endcond + + +/** @} */ /* END ENCODE_FUNC */ + +/** + * \ingroup ENCODER_STRUCTURE + * NV_ENCODE_API_FUNCTION_LIST + */ +typedef struct _NV_ENCODE_API_FUNCTION_LIST +{ + uint32_t version; /**< [in]: Client should pass NV_ENCODE_API_FUNCTION_LIST_VER. */ + uint32_t reserved; /**< [in]: Reserved and should be set to 0. */ + PNVENCOPENENCODESESSION nvEncOpenEncodeSession; /**< [out]: Client should access ::NvEncOpenEncodeSession() API through this pointer. */ + PNVENCGETENCODEGUIDCOUNT nvEncGetEncodeGUIDCount; /**< [out]: Client should access ::NvEncGetEncodeGUIDCount() API through this pointer. */ + PNVENCGETENCODEPRESETCOUNT nvEncGetEncodeProfileGUIDCount; /**< [out]: Client should access ::NvEncGetEncodeProfileGUIDCount() API through this pointer.*/ + PNVENCGETENCODEPRESETGUIDS nvEncGetEncodeProfileGUIDs; /**< [out]: Client should access ::NvEncGetEncodeProfileGUIDs() API through this pointer. */ + PNVENCGETENCODEGUIDS nvEncGetEncodeGUIDs; /**< [out]: Client should access ::NvEncGetEncodeGUIDs() API through this pointer. */ + PNVENCGETINPUTFORMATCOUNT nvEncGetInputFormatCount; /**< [out]: Client should access ::NvEncGetInputFormatCount() API through this pointer. */ + PNVENCGETINPUTFORMATS nvEncGetInputFormats; /**< [out]: Client should access ::NvEncGetInputFormats() API through this pointer. */ + PNVENCGETENCODECAPS nvEncGetEncodeCaps; /**< [out]: Client should access ::NvEncGetEncodeCaps() API through this pointer. */ + PNVENCGETENCODEPRESETCOUNT nvEncGetEncodePresetCount; /**< [out]: Client should access ::NvEncGetEncodePresetCount() API through this pointer. */ + PNVENCGETENCODEPRESETGUIDS nvEncGetEncodePresetGUIDs; /**< [out]: Client should access ::NvEncGetEncodePresetGUIDs() API through this pointer. */ + PNVENCGETENCODEPRESETCONFIG nvEncGetEncodePresetConfig; /**< [out]: Client should access ::NvEncGetEncodePresetConfig() API through this pointer. */ + PNVENCINITIALIZEENCODER nvEncInitializeEncoder; /**< [out]: Client should access ::NvEncInitializeEncoder() API through this pointer. */ + PNVENCCREATEINPUTBUFFER nvEncCreateInputBuffer; /**< [out]: Client should access ::NvEncCreateInputBuffer() API through this pointer. */ + PNVENCDESTROYINPUTBUFFER nvEncDestroyInputBuffer; /**< [out]: Client should access ::NvEncDestroyInputBuffer() API through this pointer. */ + PNVENCCREATEBITSTREAMBUFFER nvEncCreateBitstreamBuffer; /**< [out]: Client should access ::NvEncCreateBitstreamBuffer() API through this pointer. */ + PNVENCDESTROYBITSTREAMBUFFER nvEncDestroyBitstreamBuffer; /**< [out]: Client should access ::NvEncDestroyBitstreamBuffer() API through this pointer. */ + PNVENCENCODEPICTURE nvEncEncodePicture; /**< [out]: Client should access ::NvEncEncodePicture() API through this pointer. */ + PNVENCLOCKBITSTREAM nvEncLockBitstream; /**< [out]: Client should access ::NvEncLockBitstream() API through this pointer. */ + PNVENCUNLOCKBITSTREAM nvEncUnlockBitstream; /**< [out]: Client should access ::NvEncUnlockBitstream() API through this pointer. */ + PNVENCLOCKINPUTBUFFER nvEncLockInputBuffer; /**< [out]: Client should access ::NvEncLockInputBuffer() API through this pointer. */ + PNVENCUNLOCKINPUTBUFFER nvEncUnlockInputBuffer; /**< [out]: Client should access ::NvEncUnlockInputBuffer() API through this pointer. */ + PNVENCGETENCODESTATS nvEncGetEncodeStats; /**< [out]: Client should access ::NvEncGetEncodeStats() API through this pointer. */ + PNVENCGETSEQUENCEPARAMS nvEncGetSequenceParams; /**< [out]: Client should access ::NvEncGetSequenceParams() API through this pointer. */ + PNVENCREGISTERASYNCEVENT nvEncRegisterAsyncEvent; /**< [out]: Client should access ::NvEncRegisterAsyncEvent() API through this pointer. */ + PNVENCUNREGISTERASYNCEVENT nvEncUnregisterAsyncEvent; /**< [out]: Client should access ::NvEncUnregisterAsyncEvent() API through this pointer. */ + PNVENCMAPINPUTRESOURCE nvEncMapInputResource; /**< [out]: Client should access ::NvEncMapInputResource() API through this pointer. */ + PNVENCUNMAPINPUTRESOURCE nvEncUnmapInputResource; /**< [out]: Client should access ::NvEncUnmapInputResource() API through this pointer. */ + PNVENCDESTROYENCODER nvEncDestroyEncoder; /**< [out]: Client should access ::NvEncDestroyEncoder() API through this pointer. */ + PNVENCINVALIDATEREFFRAMES nvEncInvalidateRefFrames; /**< [out]: Client should access ::NvEncInvalidateRefFrames() API through this pointer. */ + PNVENCOPENENCODESESSIONEX nvEncOpenEncodeSessionEx; /**< [out]: Client should access ::NvEncOpenEncodeSession() API through this pointer. */ + PNVENCREGISTERRESOURCE nvEncRegisterResource; /**< [out]: Client should access ::NvEncRegisterResource() API through this pointer. */ + PNVENCUNREGISTERRESOURCE nvEncUnregisterResource; /**< [out]: Client should access ::NvEncUnregisterResource() API through this pointer. */ + PNVENCRECONFIGUREENCODER nvEncReconfigureEncoder; /**< [out]: Client should access ::NvEncReconfigureEncoder() API through this pointer. */ + void *reserved1; + PNVENCCREATEMVBUFFER nvEncCreateMVBuffer; /**< [out]: Client should access ::NvEncCreateMVBuffer API through this pointer. */ + PNVENCDESTROYMVBUFFER nvEncDestroyMVBuffer; /**< [out]: Client should access ::NvEncDestroyMVBuffer API through this pointer. */ + PNVENCRUNMOTIONESTIMATIONONLY nvEncRunMotionEstimationOnly; /**< [out]: Client should access ::NvEncRunMotionEstimationOnly API through this pointer. */ + PNVENCGETLASTERROR nvEncGetLastErrorString; /**< [out]: Client should access ::nvEncGetLastErrorString API through this pointer. */ + PNVENCSETIOCUDASTREAMS nvEncSetIOCudaStreams; /**< [out]: Client should access ::nvEncSetIOCudaStreams API through this pointer. */ + PNVENCGETENCODEPRESETCONFIGEX nvEncGetEncodePresetConfigEx; /**< [out]: Client should access ::NvEncGetEncodePresetConfigEx() API through this pointer. */ + PNVENCGETSEQUENCEPARAMEX nvEncGetSequenceParamEx; /**< [out]: Client should access ::NvEncGetSequenceParamEx() API through this pointer. */ + PNVENCRESTOREENCODERSTATE nvEncRestoreEncoderState; /**< [out]: Client should access ::NvEncRestoreEncoderState() API through this pointer. */ + PNVENCLOOKAHEADPICTURE nvEncLookaheadPicture; /**< [out]: Client should access ::NvEncLookaheadPicture() API through this pointer. */ + void *reserved2[275]; /**< [in]: Reserved and must be set to NULL */ +} NV_ENCODE_API_FUNCTION_LIST; + +/** Macro for constructing the version field of ::_NV_ENCODEAPI_FUNCTION_LIST. */ +#define NV_ENCODE_API_FUNCTION_LIST_VER NVENCAPI_STRUCT_VERSION(2) + +// NvEncodeAPICreateInstance +/** + * \ingroup ENCODE_FUNC + * Entry Point to the NvEncodeAPI interface. + * + * Creates an instance of the NvEncodeAPI interface, and populates the + * pFunctionList with function pointers to the API routines implemented by the + * NvEncodeAPI interface. + * + * \param [out] functionList + * + * \return + * ::NV_ENC_SUCCESS + * ::NV_ENC_ERR_INVALID_PTR + */ +NVENCSTATUS NVENCAPI NvEncodeAPICreateInstance(NV_ENCODE_API_FUNCTION_LIST *functionList); + +#ifdef __cplusplus +} +#endif + + +#endif diff --git a/xorgxrdp_helper/encoder_headers/yami_inf.h b/xorgxrdp_helper/encoder_headers/yami_inf.h new file mode 100644 index 0000000000..949bc5f4d2 --- /dev/null +++ b/xorgxrdp_helper/encoder_headers/yami_inf.h @@ -0,0 +1,173 @@ + +#if !defined( __YAMI_INF_H__) +#define __YAMI_INF_H__ 1 + +#if !defined(YI_INT64) +#define YI_INT64 long long +#endif + +#define YI_VERSION_INT(_major, _minor) (((_major) << 16) | (_minor)) + +#define YI_MAJOR 0 +#define YI_MINOR 2 + +#define YI_SUCCESS 0 +#define YI_ERROR_MEMORY 1 +#define YI_ERROR_UNIMP 2 +#define YI_ERROR_TYPE 3 +#define YI_ERROR_OTHER 4 +#define YI_ERROR_VAINITIALIZE 100 +#define YI_ERROR_VACREATECONFIG 101 +#define YI_ERROR_VACREATECONTEXT 102 +#define YI_ERROR_VACREATESURFACES 103 +#define YI_ERROR_VACREATEIMAGE 104 +#define YI_ERROR_VAMAPBUFFER 105 +#define YI_ERROR_VAUNMAPBUFFER 106 +#define YI_ERROR_VAPUTIMAGE 107 +#define YI_ERROR_VASYNCSURFACE 108 +#define YI_ERROR_VAPUTSURFACE 109 +#define YI_ERROR_VACREATEBUFFER 110 +#define YI_ERROR_VABEGINPICTURE 111 +#define YI_ERROR_VARENDERPICTURE 112 +#define YI_ERROR_VAENDPICTURE 113 +#define YI_ERROR_VAEXPORTSURFACE 114 +#define YI_ERROR_CREATEENCODER 200 +#define YI_ERROR_ENCODEGETPARAMETERS 201 +#define YI_ERROR_ENCODESETPARAMETERS 202 +#define YI_ERROR_ENCODESTART 203 +#define YI_ERROR_ENCODEENCODE 204 +#define YI_ERROR_ENCODEGETOUTPUT 205 +#define YI_ERROR_CREATEDECODER 300 +#define YI_ERROR_DECODESTART 301 +#define YI_ERROR_DECODEDECODE 302 +#define YI_ERROR_DECODEGETOUTPUT 303 +#define YI_ERROR_DECODEGETFORMATINFO 304 + +#define YI_TYPE_H264 1 +#define YI_TYPE_MPEG2 2 + +#define YI_H264_ENC_FLAGS_PROFILE_MASK 0x0000000F +#define YI_H264_ENC_FLAGS_PROFILE_MAIN (1 << 0) +#define YI_H264_ENC_FLAGS_PROFILE_HIGH (2 << 0) + +#define YI_TYPE_DRM 1 +#define YI_TYPE_X11 2 + +#define YI_H264_DEC_FLAG_LOWLATENCY 1 + +#define YI_NV12 0x10C +#define YI_YUY2 0x110 + +#ifdef __cplusplus +extern "C" +{ +#endif + +int +yami_get_version(int *version); +int +yami_init(int type, void *display); +int +yami_deinit(void); + +int +yami_encoder_create(void **obj, int width, int height, int type, int flags); +int +yami_encoder_delete(void *obj); +int +yami_encoder_get_width(void *obj, int *width); +int +yami_encoder_get_height(void *obj, int *height); +int +yami_encoder_resize(void *obj, int width, int height); +int +yami_encoder_get_ybuffer(void *obj, void **ydata, int *ydata_stride_bytes); +int +yami_encoder_get_uvbuffer(void *obj, void **uvdata, int *uvdata_stride_bytes); +int +yami_encoder_set_fd_src(void *obj, int fd, int fd_width, int fd_height, + int fd_stride, int fd_size, int fd_bpp); +int +yami_encoder_encode(void *obj, void *cdata, + int *cdata_max_bytes, int force_key_frame); + +int +yami_decoder_create(void **obj, int width, int height, int type, int flags); +int +yami_decoder_delete(void *obj); +int +yami_decoder_decode(void *obj, void *cdata, int cdata_bytes); +int +yami_decoder_decode_time(void *obj, void *cdata, int cdata_bytes, + YI_INT64 time); +int +yami_decoder_get_pixmap(void *obj, void *display, + int width, int height, int *pixmap); +int +yami_decoder_get_fd_dst(void *obj, int *fd, int *fd_width, int *fd_height, + int *fd_stride, int *fd_size, int *fd_bpp, + YI_INT64 *fd_time); + +int +yami_surface_create(void **obj, int width, int height, int type, int flags); +int +yami_surface_delete(void *obj); +int +yami_surface_get_ybuffer(void *obj, void **ydata, int *ydata_stride_bytes); +int +yami_surface_get_uvbuffer(void *obj, void **uvdata, int *uvdata_stride_bytes); +int +yami_surface_get_fd_dst(void *obj, int *fd, int *fd_width, int *fd_height, + int *fd_stride, int *fd_size, int *fd_bpp); + +struct yami_funcs +{ + int (*yami_get_version)(int *version); + int (*yami_init)(int type, void *display); + int (*yami_deinit)(void); + size_t pad0[20 - 3]; + /* encoder */ + int (*yami_encoder_create)(void **obj, int width, int height, int type, int flags); + int (*yami_encoder_delete)(void *obj); + int (*yami_encoder_get_width)(void *obj, int *width); + int (*yami_encoder_get_height)(void *obj, int *height); + int (*yami_encoder_resize)(void *obj, int width, int height); + int (*yami_encoder_get_ybuffer)(void *obj, void **ydata, int *ydata_stride_bytes); + int (*yami_encoder_get_uvbuffer)(void *obj, void **uvdata, int *uvdata_stride_bytes); + int (*yami_encoder_set_fd_src)(void *obj, int fd, int fd_width, int fd_height, + int fd_stride, int fd_size, int fd_bpp); + int (*yami_encoder_encode)(void *obj, void *cdata, + int *cdata_max_bytes, int force_key_frame); + size_t pad1[20 - 9]; + /* decoder */ + int (*yami_decoder_create)(void **obj, int width, int height, int type, int flags); + int (*yami_decoder_delete)(void *obj); + int (*yami_decoder_decode)(void *obj, void *cdata, int cdata_bytes); + int (*yami_decoder_decode_time)(void *obj, void *cdata, int cdata_bytes, + YI_INT64 time); + int (*yami_decoder_get_pixmap)(void *obj, void *display, + int width, int height, int *pixmap); + int (*yami_decoder_get_fd_dst)(void *obj, int *fd, int *fd_width, int *fd_height, + int *fd_stride, int *fd_size, int *fd_bpp, + YI_INT64 *fd_time); + size_t pad2[20 - 6]; + /* surface */ + int (*yami_surface_create)(void **obj, int width, int height, int type, int flags); + int (*yami_surface_delete)(void *obj); + int (*yami_surface_get_ybuffer)(void *obj, void **ydata, int *ydata_stride_bytes); + int (*yami_surface_get_uvbuffer)(void *obj, void **uvdata, int *uvdata_stride_bytes); + int (*yami_surface_get_fd_dst)(void *obj, int *fd, int *fd_width, int *fd_height, + int *fd_stride, int *fd_size, int *fd_bpp); + size_t pad3[20 - 5]; +}; + +typedef int +(*yami_get_funcs_proc)(struct yami_funcs *funcs, int version); +int +yami_get_funcs(struct yami_funcs *funcs, int version); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/xorgxrdp_helper/xorgxrdp_helper.c b/xorgxrdp_helper/xorgxrdp_helper.c new file mode 100644 index 0000000000..f8117c911c --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper.c @@ -0,0 +1,852 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2020-2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include +#include +#include + +#include "arch.h" +#include "parse.h" +#include "trans.h" +#include "os_calls.h" +#include "string_calls.h" +#include "log.h" + +#include "xorgxrdp_helper.h" +#include "xorgxrdp_helper_x11.h" + +#define ARRAYSIZE(x) (sizeof(x)/sizeof(*(x))) + +struct xorgxrdp_info +{ + struct trans *xorg_trans; + struct trans *xrdp_trans; + struct source_info si; + int resizing; +}; + +static int g_shmem_id_mapped = 0; +static int g_shmem_id = 0; +static void *g_shmem_pixels = 0; +static int g_display_num = 0; +int xrdp_invalidate = 0; + +/*****************************************************************************/ +static int +xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct stream *s, int *fd) +{ + int num_drects; + int num_crects; + int flags; + int shmem_bytes; + int shmem_offset; + int frame_id; + int left; + int top; + int width; + int height; + int cdata_bytes; + int index; + int recv_bytes; + enum encoder_result rv; + struct xh_rect *crects; + char *bmpdata; + char msg[4]; + unsigned int num_fds; + void *shmem_ptr = NULL; + + /* dirty pixels */ + in_uint16_le(s, num_drects); + in_uint8s(s, 8 * num_drects); + /* copied pixels */ + in_uint16_le(s, num_crects); + crects = g_new(struct xh_rect, num_crects); + for (index = 0; index < num_crects; index++) + { + in_uint16_le(s, crects[index].x); + in_uint16_le(s, crects[index].y); + in_uint16_le(s, crects[index].w); + in_uint16_le(s, crects[index].h); + } + char *flag_pointer = s->p; + in_uint32_le(s, flags); + in_uint32_le(s, frame_id); + in_uint32_le(s, shmem_bytes); + in_uint32_le(s, shmem_offset); + + in_uint16_le(s, left); + in_uint16_le(s, top); + in_uint16_le(s, width); + in_uint16_le(s, height); + char *final_pointer = s->p; + + if (xi->resizing == 3) + { + if (xrdp_invalidate > 0 && frame_id == 1) + { + // Let it through. We are no longer resizing. + LOG(LOG_LEVEL_DEBUG, "Invalidate received and processing frame ID 1. Unblocking encoder. Invalidate is %d.", xrdp_invalidate); + xi->resizing = 0; + } + else + { + LOG(LOG_LEVEL_DEBUG, "Blocked Incoming Frame ID %d. Invalidate is %d", frame_id, xrdp_invalidate); + return 0; + } + } + + if (xi->resizing > 0) + { + return 0; + } + + *fd = -1; + num_fds = -1; + if (g_tcp_can_recv(trans->sck, 5000) == 0) + { + g_free(crects); + return 1; + } + recv_bytes = g_sck_recv_fd_set(trans->sck, msg, 4, fd, 1, &num_fds); + LOG(LOG_LEVEL_DEBUG, "xorg_process_message_64: " + "g_sck_recv_fd_set rv %d fd %d, num_fds %d", + recv_bytes, *fd, num_fds); + + bmpdata = NULL; + if (recv_bytes == 4) + { + if (num_fds == 1) + { + if (g_file_map(*fd, 1, 1, shmem_bytes, &shmem_ptr) == 0) + { + bmpdata = (char *)shmem_ptr; + bmpdata += shmem_offset; + + if ((bmpdata != NULL) && (flags & 1)) + { + cdata_bytes = 16 * 1024 * 1024; + rv = xorgxrdp_helper_x11_encode_pixmap(left, top, + width, height, 0, + num_crects, crects, + bmpdata + 4, + &cdata_bytes); + if (rv == ENCODER_ERROR) + { + LOG(LOG_LEVEL_ERROR, "error %d", rv); + } + if (rv == KEY_FRAME_ENCODED) + { + s->p = flag_pointer; + out_uint32_le(s, 1 | 2); + s->p = final_pointer; + } + + bmpdata[0] = cdata_bytes; + bmpdata[1] = cdata_bytes >> 8; + bmpdata[2] = cdata_bytes >> 16; + bmpdata[3] = cdata_bytes >> 24; + LOG(LOG_LEVEL_INFO, "cdata_bytes %d", cdata_bytes); + } + } + } + else + { + LOG(LOG_LEVEL_ERROR, + "xorg_process_message_64: num_fds %d", num_fds); + } + } + else + { + LOG(LOG_LEVEL_ERROR, + "xorg_process_message_64: recv_bytes %d", recv_bytes); + } + + if (shmem_ptr != NULL) + { + g_munmap(shmem_ptr, shmem_bytes); + } + g_free(crects); + return 0; +} + +/*****************************************************************************/ +static int +xorg_process_message_61(struct xorgxrdp_info *xi, struct stream *s) +{ + int num_drects; + int num_crects; + int flags; + int shmem_id; + int shmem_offset; + int frame_id; + int width; + int height; + int cdata_bytes; + int index; + enum encoder_result rv; + struct xh_rect *crects; + char *bmpdata; + + /* dirty pixels */ + in_uint16_le(s, num_drects); + in_uint8s(s, 8 * num_drects); + /* copied pixels */ + in_uint16_le(s, num_crects); + crects = g_new(struct xh_rect, num_crects); + for (index = 0; index < num_crects; index++) + { + in_uint16_le(s, crects[index].x); + in_uint16_le(s, crects[index].y); + in_uint16_le(s, crects[index].w); + in_uint16_le(s, crects[index].h); + } + char *flag_pointer = s->p; + in_uint32_le(s, flags); + in_uint32_le(s, frame_id); + in_uint32_le(s, shmem_id); + in_uint32_le(s, shmem_offset); + + in_uint16_le(s, width); + in_uint16_le(s, height); + char *final_pointer = s->p; + + if (xi->resizing == 3) + { + if (xrdp_invalidate > 0 && frame_id == 1) + { + // Let it through. We are no longer resizing. + LOG(LOG_LEVEL_DEBUG, "Invalidate received and processing frame ID 1. Unblocking encoder. Invalidate is %d.", xrdp_invalidate); + xi->resizing = 0; + } + else + { + LOG(LOG_LEVEL_DEBUG, "Blocked Incoming Frame ID %d. Invalidate is %d", frame_id, xrdp_invalidate); + return 0; + } + } + + if (xi->resizing > 0) + { + return 0; + } + + bmpdata = NULL; + if (g_shmem_id_mapped == 0) + { + g_shmem_id = shmem_id; + g_shmem_pixels = (char *) g_shmat(g_shmem_id); + if (g_shmem_pixels == (void *) -1) + { + /* failed */ + g_shmem_id = 0; + g_shmem_pixels = NULL; + g_shmem_id_mapped = 0; + } + else + { + g_shmem_id_mapped = 1; + } + } + else if (g_shmem_id != shmem_id) + { + g_shmem_id = shmem_id; + g_shmdt(g_shmem_pixels); + g_shmem_pixels = (char *) g_shmat(g_shmem_id); + if (g_shmem_pixels == (void *) -1) + { + /* failed */ + g_shmem_id = 0; + g_shmem_pixels = NULL; + g_shmem_id_mapped = 0; + } + } + if (g_shmem_pixels != NULL) + { + bmpdata = (char *)g_shmem_pixels + shmem_offset; + } + if ((bmpdata != NULL) && (flags & 1)) + { + cdata_bytes = 16 * 1024 * 1024; + rv = xorgxrdp_helper_x11_encode_pixmap(0, 0, width, height, 0, + num_crects, crects, + bmpdata + 4, &cdata_bytes); + if (rv == ENCODER_ERROR) + { + LOG(LOG_LEVEL_ERROR, "error %d", rv); + } + if (rv == KEY_FRAME_ENCODED) + { + s->p = flag_pointer; + out_uint32_le(s, 1 | 2); + s->p = final_pointer; + } + + bmpdata[0] = cdata_bytes; + bmpdata[1] = cdata_bytes >> 8; + bmpdata[2] = cdata_bytes >> 16; + bmpdata[3] = cdata_bytes >> 24; + LOG_DEVEL(LOG_LEVEL_INFO, "cdata_bytes %d", cdata_bytes); + } + g_free(crects); + return 0; +} + +/*****************************************************************************/ +/* data going from xorg to xrdp */ +static int +xorg_process_message(struct xorgxrdp_info *xi, struct trans *trans, + struct stream *s) +{ + int type; + int num; + int size; + int index; + char *phold; + int width; + int height; + int magic; + int con_id; + int mon_id; + int fd = -1; + int ret; + + in_uint16_le(s, type); + in_uint16_le(s, num); + in_uint32_le(s, size); + if (type == 3) + { + for (index = 0; index < num; index++) + { + phold = s->p; + in_uint16_le(s, type); + in_uint16_le(s, size); + switch (type) + { + case 61: + xorg_process_message_61(xi, s); + break; + case 64: + if (xorg_process_message_64(xi, trans, s, &fd)) + { + return 1; + } + break; + } + s->p = phold + size; + } + if (xi->resizing > 0) + { + return 0; + } + } + else if (type == 100) + { + for (index = 0; index < num; index++) + { + phold = s->p; + in_uint16_le(s, type); + in_uint16_le(s, size); + LOG(LOG_LEVEL_DEBUG, "100 type %d size %d", type, size); + switch (type) + { + case 1: + LOG(LOG_LEVEL_DEBUG, "calling xorgxrdp_helper_x11_delete_all_pixmaps"); + xorgxrdp_helper_x11_delete_all_pixmaps(); + if (xi->resizing == 1) + { + xi->resizing = 2; + } + break; + case 2: + in_uint16_le(s, width); + in_uint16_le(s, height); + in_uint32_le(s, magic); + in_uint32_le(s, con_id); + in_uint32_le(s, mon_id); + LOG(LOG_LEVEL_DEBUG, "calling xorgxrdp_helper_x11_create_pixmap"); + xorgxrdp_helper_x11_create_pixmap(width, height, magic, + con_id, mon_id); + if (xi->resizing == 2) + { + xi->resizing = 3; + } + break; + } + s->p = phold + size; + } + } + s->p = s->data; + + if (fd == -1) + { + // Using system-v shared memory. + ret = trans_write_copy_s(xi->xrdp_trans, s); + return ret; + } + // Using posix shared memory + ret = trans_force_write_s(xi->xrdp_trans, s); + if (ret) + { + return ret; + } + ret = g_sck_send_fd_set(xi->xrdp_trans->sck, "int", 4, &fd, 1); + if (ret < 0) + { + return 1; + } + return g_file_close(fd); +} + +/*****************************************************************************/ +static int +xorg_data_in(struct trans *trans) +{ + struct stream *s; + int len; + struct xorgxrdp_info *xi; + + xi = (struct xorgxrdp_info *) (trans->callback_data); + s = trans_get_in_s(trans); + switch (trans->extra_flags) + { + case 1: + s->p = s->data; + in_uint8s(s, 4); + in_uint32_le(s, len); + if ((len < 0) || (len > 128 * 1024)) + { + LOG(LOG_LEVEL_ERROR, "bad size %d", len); + return 1; + } + if (len > 0) + { + trans->header_size = len + 8; + trans->extra_flags = 2; + break; + } + /* fall through */ + case 2: + s->p = s->data; + if (xorg_process_message(xi, trans, s) != 0) + { + LOG(LOG_LEVEL_ERROR, "xorg_process_message failed"); + return 1; + } + init_stream(s, 0); + trans->header_size = 8; + trans->extra_flags = 1; + break; + } + return 0; +} + +/*****************************************************************************/ +/* data going from xrdp to xorg */ +static int +xrdp_process_message(struct xorgxrdp_info *xi, struct stream *s) +{ + int len; + int msg_type1; + int msg_type2; + + in_uint32_le(s, len); + in_uint16_le(s, msg_type1); + if (msg_type1 == 103) // client message + { + in_uint32_le(s, msg_type2); + if (msg_type2 == 200) // invalidate + { + LOG(LOG_LEVEL_DEBUG, "Invalidate found (len: %d, msg1: %d, msg2: %d)", len, msg_type1, msg_type2); + /* + 10 is an arbitrary number. Anecdotally, you need the first 10 frames to be key frames to make sure the client + receives at least one of them. + */ + xrdp_invalidate += 10; + } + else if (msg_type2 == 300) // resize + { + LOG(LOG_LEVEL_DEBUG, "Resize found (len: %d, msg1: %d, msg2: %d)", len, msg_type1, msg_type2); + xi->resizing = 1; + } + } + //Reset read pointer + s->p = s->data; + return trans_write_copy_s(xi->xorg_trans, s); +} + +/*****************************************************************************/ +static int +xrdp_data_in(struct trans *trans) +{ + struct stream *s; + int len; + struct xorgxrdp_info *xi; + + xi = (struct xorgxrdp_info *) (trans->callback_data); + s = trans_get_in_s(trans); + switch (trans->extra_flags) + { + case 1: + s->p = s->data; + in_uint32_le(s, len); + if ((len < 0) || (len > 128 * 1024)) + { + LOG(LOG_LEVEL_ERROR, "bad size %d", len); + return 1; + } + if (len > 0) + { + trans->header_size = len; + trans->extra_flags = 2; + break; + } + /* fall through */ + case 2: + s->p = s->data; + if (xrdp_process_message(xi, s) != 0) + { + LOG(LOG_LEVEL_ERROR, "xrdp_process_message failed"); + return 1; + } + init_stream(s, 0); + trans->header_size = 4; + trans->extra_flags = 1; + break; + } + return 0; +} + +/*****************************************************************************/ +static void +sigpipe_func(int sig) +{ + (void) sig; +} + +/*****************************************************************************/ +static int +get_log_path(char *path, int bytes) +{ + char *log_path; + int rv; + + rv = 1; + log_path = g_getenv("XORGXRDP_HELPER_LOG_PATH"); + if (log_path == 0) + { + log_path = g_getenv("XDG_DATA_HOME"); + if (log_path != 0) + { + g_snprintf(path, bytes, "%s%s", log_path, "/xrdp"); + if (g_directory_exist(path) || (g_mkdir(path) == 0)) + { + rv = 0; + } + } + } + else + { + g_snprintf(path, bytes, "%s", log_path); + if (g_directory_exist(path) || (g_mkdir(path) == 0)) + { + rv = 0; + } + } + if (rv != 0) + { + log_path = g_getenv("HOME"); + if (log_path != 0) + { + g_snprintf(path, bytes, "%s%s", log_path, "/.local"); + if (g_directory_exist(path) || (g_mkdir(path) == 0)) + { + g_snprintf(path, bytes, "%s%s", log_path, "/.local/share"); + if (g_directory_exist(path) || (g_mkdir(path) == 0)) + { + g_snprintf(path, bytes, "%s%s", log_path, "/.local/share/xrdp"); + if (g_directory_exist(path) || (g_mkdir(path) == 0)) + { + rv = 0; + } + } + } + } + } + return rv; +} + +/*****************************************************************************/ +static enum logLevels +get_log_level(const char *level_str, enum logLevels default_level) +{ + static const char *levels[] = { + "LOG_LEVEL_ALWAYS", + "LOG_LEVEL_ERROR", + "LOG_LEVEL_WARNING", + "LOG_LEVEL_INFO", + "LOG_LEVEL_DEBUG", + "LOG_LEVEL_TRACE" + }; + unsigned int i; + + if (level_str == NULL || level_str[0] == 0) + { + return default_level; + } + for (i = 0; i < ARRAYSIZE(levels); ++i) + { + if (g_strcasecmp(levels[i], level_str) == 0) + { + return (enum logLevels) i; + } + } + return default_level; +} + +/*****************************************************************************/ +static int +get_display_num_from_display(char *display_text) +{ + int index; + int mode; + int host_index; + int disp_index; + int scre_index; + char host[256]; + char disp[256]; + char scre[256]; + + g_memset(host, 0, 256); + g_memset(disp, 0, 256); + g_memset(scre, 0, 256); + + index = 0; + host_index = 0; + disp_index = 0; + scre_index = 0; + mode = 0; + + while (display_text[index] != 0) + { + if (display_text[index] == ':') + { + mode = 1; + } + else if (display_text[index] == '.') + { + mode = 2; + } + else if (mode == 0) + { + host[host_index] = display_text[index]; + host_index++; + } + else if (mode == 1) + { + disp[disp_index] = display_text[index]; + disp_index++; + } + else if (mode == 2) + { + scre[scre_index] = display_text[index]; + scre_index++; + } + + index++; + } + + host[host_index] = 0; + disp[disp_index] = 0; + scre[scre_index] = 0; + g_display_num = g_atoi(disp); + return 0; +} + +/*****************************************************************************/ +static int +xorgxrdp_helper_setup_log(void) +{ + struct log_config logconfig; + enum logLevels log_level; + char log_path[256]; + char log_file[256]; + char *display_text; + int error; + + if (get_log_path(log_path, 255) != 0) + { + g_writeln("error reading XORGXRDP_HELPER_LOG_PATH and HOME " + "environment variable"); + g_deinit(); + return 1; + } + display_text = g_getenv("DISPLAY"); + if (display_text != NULL) + { + get_display_num_from_display(display_text); + } + g_snprintf(log_file, 255, "%s/xorgxrdp_helper.%d.log", log_path, + g_display_num); + g_writeln("xorgxrdp_helper::xorgxrdp_helper_setup_log: using " + "log file [%s]", log_file); + if (g_file_exist(log_file)) + { + g_file_delete(log_file); + } + log_level = get_log_level(g_getenv("XORGXRDP_HELPER_LOG_LEVEL"), + LOG_LEVEL_INFO); + logconfig.log_file = log_file; + logconfig.fd = -1; + logconfig.log_level = log_level; + logconfig.enable_syslog = 0; + logconfig.syslog_level = LOG_LEVEL_ALWAYS; + logconfig.program_name = "xorgxrdp_helper"; + logconfig.enable_console = 0; + logconfig.enable_pid = 1; +#ifdef LOG_PER_LOGGER_LEVEL + logconfig.per_logger_level = NULL; +#endif + error = log_start_from_param(&logconfig); + + return error; +} + +/*****************************************************************************/ +int +main(int argc, char **argv) +{ + int xorg_fd; + int xrdp_fd; + int error; + intptr_t robjs[16]; + int robj_count; + intptr_t wobjs[16]; + int wobj_count; + int timeout; + struct xorgxrdp_info xi; + + if (argc < 2) + { + g_writeln("need to pass -d"); + return 0; + } + if (strcmp(argv[1], "-d") != 0) + { + g_writeln("need to pass -d"); + return 0; + } + g_init("xorgxrdp_helper"); + + if (xorgxrdp_helper_setup_log() != 0) + { + return 1; + } + LOG(LOG_LEVEL_INFO, "startup"); + g_memset(&xi, 0, sizeof(xi)); + g_signal_pipe(sigpipe_func); + if (xorgxrdp_helper_x11_init() != 0) + { + LOG(LOG_LEVEL_ERROR, "xorgxrdp_helper_x11_init failed"); + return 1; + } + xorg_fd = g_atoi(g_getenv("XORGXRDP_XORG_FD")); + LOG(LOG_LEVEL_INFO, "xorg_fd: %s", g_getenv("XORGXRDP_XORG_FD")); + xrdp_fd = g_atoi(g_getenv("XORGXRDP_XRDP_FD")); + LOG(LOG_LEVEL_INFO, "xorg_fd: %s", g_getenv("XORGXRDP_XRDP_FD")); + + xi.resizing = 0; + + xi.xorg_trans = trans_create(TRANS_MODE_UNIX, 128 * 1024, 128 * 1024); + xi.xorg_trans->sck = xorg_fd; + xi.xorg_trans->status = TRANS_STATUS_UP; + xi.xorg_trans->trans_data_in = xorg_data_in; + xi.xorg_trans->header_size = 8; + xi.xorg_trans->no_stream_init_on_data_in = 1; + xi.xorg_trans->extra_flags = 1; + xi.xorg_trans->callback_data = ξ + xi.xorg_trans->si = &(xi.si); + xi.xorg_trans->my_source = XORGXRDP_SOURCE_XORG; + + xi.xrdp_trans = trans_create(TRANS_MODE_UNIX, 128 * 1024, 128 * 1024); + xi.xrdp_trans->sck = xrdp_fd; + xi.xrdp_trans->status = TRANS_STATUS_UP; + xi.xrdp_trans->trans_data_in = xrdp_data_in; + xi.xrdp_trans->no_stream_init_on_data_in = 1; + xi.xrdp_trans->header_size = 4; + xi.xrdp_trans->extra_flags = 1; + xi.xrdp_trans->callback_data = ξ + xi.xrdp_trans->si = &(xi.si); + xi.xrdp_trans->my_source = XORGXRDP_SOURCE_XRDP; + + for (;;) + { + robj_count = 0; + wobj_count = 0; + timeout = 0; + error = trans_get_wait_objs_rw(xi.xorg_trans, robjs, &robj_count, + wobjs, &wobj_count, &timeout); + if (error != 0) + { + LOG(LOG_LEVEL_INFO, "trans_get_wait_objs_rw failed"); + break; + } + error = trans_get_wait_objs_rw(xi.xrdp_trans, robjs, &robj_count, + wobjs, &wobj_count, &timeout); + if (error != 0) + { + LOG(LOG_LEVEL_INFO, "trans_get_wait_objs_rw failed"); + break; + } + error = xorgxrdp_helper_x11_get_wait_objs(robjs, &robj_count); + if (error != 0) + { + LOG(LOG_LEVEL_INFO, "xorgxrdp_helper_x11_get_wait_objs failed"); + break; + } + error = g_obj_wait(robjs, robj_count, wobjs, wobj_count, timeout); + if (error != 0) + { + LOG(LOG_LEVEL_INFO, "g_obj_wait failed"); + break; + } + error = trans_check_wait_objs(xi.xorg_trans); + if (error != 0) + { + LOG(LOG_LEVEL_INFO, "xorg_trans trans_check_wait_objs failed"); + break; + } + error = trans_check_wait_objs(xi.xrdp_trans); + if (error != 0 && error != 10) + { + LOG(LOG_LEVEL_ERROR, "xrdp_trans trans_check_wait_objs failed"); + break; + } + error = xorgxrdp_helper_x11_check_wait_objs(); + if (error != 0) + { + LOG(LOG_LEVEL_ERROR, "xorgxrdp_helper_x11_check_wait_objs failed"); + break; + } + } + LOG(LOG_LEVEL_INFO, "exit"); + return 0; +} diff --git a/xorgxrdp_helper/xorgxrdp_helper.h b/xorgxrdp_helper/xorgxrdp_helper.h new file mode 100644 index 0000000000..cd9d80e8c7 --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper.h @@ -0,0 +1,45 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2020-2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _XORGXRDP_HELPER_H +#define _XORGXRDP_HELPER_H + +#define XH_YUV420 1 +#define XH_YUV422 2 +#define XH_YUV444 3 + +#define XH_BT601 0 +#define XH_BT709FR 1 +#define XH_BTRFX 2 + +struct xh_rect +{ + short x; + short y; + short w; + short h; +}; + +enum encoder_result +{ + INCREMENTAL_FRAME_ENCODED, + KEY_FRAME_ENCODED, + ENCODER_ERROR +}; + +#endif diff --git a/xorgxrdp_helper/xorgxrdp_helper_egl.c b/xorgxrdp_helper/xorgxrdp_helper_egl.c new file mode 100644 index 0000000000..fb8f1491b3 --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper_egl.c @@ -0,0 +1,159 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include +#include +#include + +#include + +#include +#include + +#include "arch.h" +#include "os_calls.h" +#include "string_calls.h" +#include "xorgxrdp_helper.h" +#include "xorgxrdp_helper_x11.h" +#include "xorgxrdp_helper_egl.h" +#include "log.h" + +EGLDisplay g_egl_display; +EGLContext g_egl_context; +static EGLSurface g_egl_surface; +static EGLConfig g_ecfg; +static EGLint g_num_config; + +/* X11 */ +extern Display *g_display; /* in xorgxrdp_helper_x11.c */ +extern Window g_root_window; /* in xorgxrdp_helper_x11.c */ + +static EGLint g_choose_config_attr[] = +{ + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_NONE +}; + +static EGLint g_create_context_attr[] = +{ + EGL_CONTEXT_MAJOR_VERSION, 3, + EGL_CONTEXT_MINOR_VERSION, 3, + EGL_NONE +}; + +static const EGLint g_create_surface_attr[] = +{ + EGL_TEXTURE_TARGET, EGL_TEXTURE_2D, + EGL_TEXTURE_FORMAT, EGL_TEXTURE_RGBA, + EGL_NONE +}; + +/*****************************************************************************/ +static EGLBoolean +xorgxrdp_helper_check_ext(const char *ext_name) +{ + if (!epoxy_has_egl_extension(g_egl_display, ext_name)) + { + LOG(LOG_LEVEL_INFO, "%s not present", ext_name); + return EGL_FALSE; + } + LOG(LOG_LEVEL_INFO, "%s present", ext_name); + return EGL_TRUE; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_inf_egl_init(void) +{ + int egl_ver; + int ok; + + ok = eglBindAPI(EGL_OPENGL_API); + LOG(LOG_LEVEL_INFO, "eglBindAPI ok %d", ok); + g_egl_display = eglGetDisplay((EGLNativeDisplayType) g_display); + LOG(LOG_LEVEL_INFO, "g_egl_display %p", g_egl_display); + eglInitialize(g_egl_display, NULL, NULL); + egl_ver = epoxy_egl_version(g_egl_display); + LOG(LOG_LEVEL_INFO, "egl_ver %d", egl_ver); + if (egl_ver < 11) /* EGL version 1.1 */ + { + LOG(LOG_LEVEL_INFO, "egl_ver too old %d", egl_ver); + eglTerminate(g_egl_display); + return 1; + } + if ((!xorgxrdp_helper_check_ext("EGL_NOK_texture_from_pixmap")) || + (!xorgxrdp_helper_check_ext("EGL_MESA_image_dma_buf_export")) || + (!xorgxrdp_helper_check_ext("EGL_KHR_image_base"))) + { + LOG(LOG_LEVEL_INFO, "missing ext"); + eglTerminate(g_egl_display); + return 1; + } + eglChooseConfig(g_egl_display, g_choose_config_attr, &g_ecfg, + 1, &g_num_config); + LOG(LOG_LEVEL_INFO, "g_ecfg %p g_num_config %d", g_ecfg, g_num_config); + g_egl_surface = eglCreateWindowSurface(g_egl_display, g_ecfg, + g_root_window, NULL); + LOG(LOG_LEVEL_INFO, "g_egl_surface %p", g_egl_surface); + g_egl_context = eglCreateContext(g_egl_display, g_ecfg, + EGL_NO_CONTEXT, g_create_context_attr); + LOG(LOG_LEVEL_INFO, "g_egl_context %p", g_egl_context); + ok = eglMakeCurrent(g_egl_display, g_egl_surface, g_egl_surface, + g_egl_context); + LOG(LOG_LEVEL_INFO, "eglMakeCurrent ok %d", ok); + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_inf_egl_create_image(Pixmap pixmap, inf_image_t *inf_image) +{ + *inf_image = (inf_image_t)eglCreatePixmapSurface(g_egl_display, + g_ecfg, pixmap, g_create_surface_attr); + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_inf_egl_destroy_image(inf_image_t inf_image) +{ + eglDestroySurface(g_egl_display, (EGLSurface)inf_image); + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_inf_egl_bind_tex_image(inf_image_t inf_image) +{ + eglBindTexImage(g_egl_display, (EGLSurface)inf_image, EGL_BACK_BUFFER); + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_inf_egl_release_tex_image(inf_image_t inf_image) +{ + eglReleaseTexImage(g_egl_display, (EGLSurface)inf_image, EGL_BACK_BUFFER); + return 0; +} diff --git a/xorgxrdp_helper/xorgxrdp_helper_egl.h b/xorgxrdp_helper/xorgxrdp_helper_egl.h new file mode 100644 index 0000000000..11df2cc46a --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper_egl.h @@ -0,0 +1,33 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _XORGXRDP_HELPER_EGL_H +#define _XORGXRDP_HELPER_EGL_H + +int +xorgxrdp_helper_inf_egl_init(void); +int +xorgxrdp_helper_inf_egl_create_image(Pixmap pixmap, inf_image_t *inf_image); +int +xorgxrdp_helper_inf_egl_destroy_image(inf_image_t inf_image); +int +xorgxrdp_helper_inf_egl_bind_tex_image(inf_image_t inf_image); +int +xorgxrdp_helper_inf_egl_release_tex_image(inf_image_t inf_image); + +#endif diff --git a/xorgxrdp_helper/xorgxrdp_helper_glx.c b/xorgxrdp_helper/xorgxrdp_helper_glx.c new file mode 100644 index 0000000000..98dd601c64 --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper_glx.c @@ -0,0 +1,149 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include +#include +#include + +#include + +#include +#include + +#include "arch.h" +#include "os_calls.h" +#include "string_calls.h" +#include "xorgxrdp_helper.h" +#include "xorgxrdp_helper_x11.h" +#include "xorgxrdp_helper_glx.h" +#include "log.h" + +static int g_n_fbconfigs = 0; +static int g_n_pixconfigs = 0; +static GLXFBConfig *g_fbconfigs = NULL; +static GLXFBConfig *g_pixconfigs = NULL; +static GLXContext g_gl_context = 0; + +/* X11 */ +extern Display *g_display; /* in xorgxrdp_helper_x11.c */ +extern Window g_root_window; /* in xorgxrdp_helper_x11.c */ +extern int g_screen_num; /* in xorgxrdp_helper_x11.c */ + +static const int g_fbconfig_attrs[] = +{ + GLX_DRAWABLE_TYPE, GLX_WINDOW_BIT, + GLX_RENDER_TYPE, GLX_RGBA_BIT, + GLX_DOUBLEBUFFER, True, + GLX_RED_SIZE, 8, + GLX_GREEN_SIZE, 8, + GLX_BLUE_SIZE, 8, + None +}; + +static const int g_pixconfig_attrs[] = +{ + GLX_BIND_TO_TEXTURE_RGBA_EXT, True, + GLX_DRAWABLE_TYPE, GLX_PIXMAP_BIT, + GLX_BIND_TO_TEXTURE_TARGETS_EXT, GLX_TEXTURE_2D_BIT_EXT, + GLX_DOUBLEBUFFER, False, + GLX_Y_INVERTED_EXT, True, + None +}; + +static const int g_pixmap_attribs[] = +{ + GLX_TEXTURE_TARGET_EXT, GLX_TEXTURE_2D_EXT, + GLX_TEXTURE_FORMAT_EXT, GLX_TEXTURE_FORMAT_RGBA_EXT, + None +}; + +/*****************************************************************************/ +int +xorgxrdp_helper_inf_glx_init(void) +{ + const char *ext_str; + int glx_ver; + int ok; + + glx_ver = epoxy_glx_version(g_display, g_screen_num); + LOG(LOG_LEVEL_INFO, "glx_ver %d", glx_ver); + if (glx_ver < 11) /* GLX version 1.1 */ + { + LOG(LOG_LEVEL_INFO, "glx_ver too old %d", glx_ver); + return 1; + } + if (!epoxy_has_glx_extension(g_display, g_screen_num, + "GLX_EXT_texture_from_pixmap")) + { + ext_str = glXQueryExtensionsString(g_display, g_screen_num); + LOG(LOG_LEVEL_INFO, "GLX_EXT_texture_from_pixmap not present [%s]", + ext_str); + return 1; + } + LOG(LOG_LEVEL_INFO, "GLX_EXT_texture_from_pixmap present"); + g_fbconfigs = glXChooseFBConfig(g_display, g_screen_num, + g_fbconfig_attrs, &g_n_fbconfigs); + LOG(LOG_LEVEL_INFO, "g_fbconfigs %p", g_fbconfigs); + g_gl_context = glXCreateNewContext(g_display, g_fbconfigs[0], + GLX_RGBA_TYPE, NULL, 1); + LOG(LOG_LEVEL_INFO, "g_gl_context %p", g_gl_context); + ok = glXMakeCurrent(g_display, g_root_window, g_gl_context); + LOG(LOG_LEVEL_INFO, "ok %d", ok); + g_pixconfigs = glXChooseFBConfig(g_display, g_screen_num, + g_pixconfig_attrs, &g_n_pixconfigs); + LOG(LOG_LEVEL_INFO, "g_pixconfigs %p g_n_pixconfigs %d", + g_pixconfigs, g_n_pixconfigs); + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_inf_glx_create_image(Pixmap pixmap, inf_image_t *inf_image) +{ + *inf_image = (inf_image_t)glXCreatePixmap(g_display, g_pixconfigs[0], + pixmap, g_pixmap_attribs); + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_inf_glx_destroy_image(inf_image_t inf_image) +{ + glXDestroyPixmap(g_display, (GLXPixmap)inf_image); + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_inf_glx_bind_tex_image(inf_image_t inf_image) +{ + glXBindTexImageEXT(g_display, (GLXPixmap)inf_image, GLX_FRONT_EXT, NULL); + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_inf_glx_release_tex_image(inf_image_t inf_image) +{ + glXReleaseTexImageEXT(g_display, (GLXPixmap)inf_image, GLX_FRONT_EXT); + return 0; +} diff --git a/xorgxrdp_helper/xorgxrdp_helper_glx.h b/xorgxrdp_helper/xorgxrdp_helper_glx.h new file mode 100644 index 0000000000..117a3a52c8 --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper_glx.h @@ -0,0 +1,33 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _XORGXRDP_HELPER_GLX_H +#define _XORGXRDP_HELPER_GLX_H + +int +xorgxrdp_helper_inf_glx_init(void); +int +xorgxrdp_helper_inf_glx_create_image(Pixmap pixmap, inf_image_t *inf_image); +int +xorgxrdp_helper_inf_glx_destroy_image(inf_image_t inf_image); +int +xorgxrdp_helper_inf_glx_bind_tex_image(inf_image_t inf_image); +int +xorgxrdp_helper_inf_glx_release_tex_image(inf_image_t inf_image); + +#endif diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c new file mode 100644 index 0000000000..f5fdb450e3 --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c @@ -0,0 +1,400 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include +#include +#include +#include "string_calls.h" + +#include + +#include "encoder_headers/nvEncodeAPI_12_1_14.h" + +#include "arch.h" +#include "os_calls.h" +#include "xorgxrdp_helper.h" +#include "xorgxrdp_helper_x11.h" +#include "xorgxrdp_helper_nvenc.h" +#include "log.h" + +#define XH_NVENV_DEFAULT_QP 28 + +typedef NVENCSTATUS +(NVENCAPI *NvEncodeAPICreateInstanceProc) +(NV_ENCODE_API_FUNCTION_LIST *functionList); + +static char g_lib_name[] = "libnvidia-encode.so"; +static char g_lib_name1[] = "libnvidia-encode.so.1"; +static char g_func_name[] = "NvEncodeAPICreateInstance"; + +static NvEncodeAPICreateInstanceProc g_NvEncodeAPICreateInstance = NULL; + +static NV_ENCODE_API_FUNCTION_LIST g_enc_funcs; + +static long g_lib = 0; + +struct enc_info +{ + int width; + int height; + int frameCount; + int pad0; + void *enc; + NV_ENC_OUTPUT_PTR bitstreamBuffer; + NV_ENC_INPUT_PTR mappedResource; + NV_ENC_BUFFER_FORMAT mappedBufferFmt; + NV_ENC_REGISTERED_PTR registeredResource; +}; + +extern int xrdp_invalidate; + +/*****************************************************************************/ +int +xorgxrdp_helper_nvenc_init(void) +{ + NVENCSTATUS nv_error; + + g_lib = g_load_library(g_lib_name); + if (g_lib == 0) + { + g_lib = g_load_library(g_lib_name1); + if (g_lib == 0) + { + LOG(LOG_LEVEL_ERROR, "load library for %s/%s failed", g_lib_name, g_lib_name1); + return 1; + } + } + g_NvEncodeAPICreateInstance = g_get_proc_address(g_lib, g_func_name); + if (g_NvEncodeAPICreateInstance == NULL) + { + LOG(LOG_LEVEL_ERROR, "get proc address for %s failed", g_func_name); + return 1; + } + g_memset(&g_enc_funcs, 0, sizeof(g_enc_funcs)); + g_enc_funcs.version = NV_ENCODE_API_FUNCTION_LIST_VER; + nv_error = g_NvEncodeAPICreateInstance(&g_enc_funcs); + LOG(LOG_LEVEL_INFO, "NvEncodeAPICreateInstance rv %d", nv_error); + if (nv_error != NV_ENC_SUCCESS) + { + return 1; + } + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, + int tex_format, struct enc_info **ei) +{ + NV_ENC_CREATE_BITSTREAM_BUFFER bitstreamParams; + NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS params; + NV_ENC_INITIALIZE_PARAMS createEncodeParams; + NV_ENC_MAP_INPUT_RESOURCE mapInputResource; + NV_ENC_INPUT_RESOURCE_OPENGL_TEX res; + NV_ENC_REGISTER_RESOURCE reg_res; + NV_ENC_CONFIG encCfg; + NV_ENC_PRESET_CONFIG preset_config; + NVENCSTATUS nv_error; + struct enc_info *lei; + char *rateControlMode_str; + char *averageBitRate_str; + char *qp_str; + int qp_int; + int averageBitRate_int; + int rc_set; + + lei = g_new0(struct enc_info, 1); + if (lei == NULL) + { + return 1; + } + + g_memset(¶ms, 0, sizeof(params)); + params.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER; + params.deviceType = NV_ENC_DEVICE_TYPE_OPENGL; + params.apiVersion = NVENCAPI_VERSION; + nv_error = g_enc_funcs.nvEncOpenEncodeSessionEx(¶ms, &(lei->enc)); + LOG(LOG_LEVEL_INFO, "nvEncOpenEncodeSessionEx rv %d enc %p", nv_error, lei->enc); + if (nv_error != NV_ENC_SUCCESS) + { + return 1; + } + + g_memset(&createEncodeParams, 0, sizeof(createEncodeParams)); + createEncodeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; + createEncodeParams.encodeGUID = NV_ENC_CODEC_H264_GUID; + createEncodeParams.presetGUID = NV_ENC_PRESET_P4_GUID; + createEncodeParams.tuningInfo = NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY; + createEncodeParams.encodeWidth = width; + createEncodeParams.encodeHeight = height; + createEncodeParams.darWidth = width; + createEncodeParams.darHeight = height; + createEncodeParams.enablePTD = 1; + + g_memset(&preset_config, 0, sizeof(preset_config)); + preset_config.version = NV_ENC_PRESET_CONFIG_VER; + preset_config.presetCfg.version = NV_ENC_CONFIG_VER; + + nv_error = g_enc_funcs.nvEncGetEncodePresetConfigEx( + lei->enc, createEncodeParams.encodeGUID, + createEncodeParams.presetGUID, + createEncodeParams.tuningInfo, + &preset_config); + + LOG(LOG_LEVEL_INFO, "nvEncGetEncodePresetConfig rv %d", nv_error); + if (nv_error != NV_ENC_SUCCESS) + { + return 1; + } + + g_memset(&encCfg, 0, sizeof(encCfg)); + encCfg.version = NV_ENC_CONFIG_VER; + encCfg.profileGUID = NV_ENC_H264_PROFILE_MAIN_GUID; + encCfg.gopLength = NVENC_INFINITE_GOPLENGTH; + encCfg.frameIntervalP = 1; /* 1 + B_Frame_Count */ + encCfg.frameFieldMode = NV_ENC_PARAMS_FRAME_FIELD_MODE_FRAME; + encCfg.mvPrecision = NV_ENC_MV_PRECISION_QUARTER_PEL; + + /* these env vars can be added / changed in sesman.ini SessionVariables + example + XRDP_NVENC_RATE_CONTROL_MODE=NV_ENC_PARAMS_RC_CONSTQP + XRDP_NVENC_QP=30 + or + XRDP_NVENC_RATE_CONTROL_MODE=NV_ENC_PARAMS_RC_VBR + XRDP_NVENC_AVERAGE_BITRATE=2000000 */ + rateControlMode_str = g_getenv("XRDP_NVENC_RATE_CONTROL_MODE"); + averageBitRate_str = g_getenv("XRDP_NVENC_AVERAGE_BITRATE"); + qp_str = g_getenv("XRDP_NVENC_QP"); + rc_set = 0; + if (rateControlMode_str != NULL) + { + if (g_strcmp(rateControlMode_str, "NV_ENC_PARAMS_RC_CONSTQP") == 0) + { + if (qp_str != NULL) + { + qp_int = g_atoi(qp_str); + if ((qp_int >= 0) && (qp_int <= 51)) + { + LOG(LOG_LEVEL_INFO, + "using NV_ENC_PARAMS_RC_CONSTQP qp %d", + qp_int); + encCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CONSTQP; + encCfg.rcParams.constQP.qpInterP = qp_int; + encCfg.rcParams.constQP.qpInterB = qp_int; + encCfg.rcParams.constQP.qpIntra = qp_int; + rc_set = 1; + } + } + } + else if (g_strcmp(rateControlMode_str, "NV_ENC_PARAMS_RC_VBR") == 0) + { + if (averageBitRate_str != NULL) + { + averageBitRate_int = g_atoi(averageBitRate_str); + if ((averageBitRate_int >= 5000) && + (averageBitRate_int <= 1000000000)) + { + LOG(LOG_LEVEL_INFO, + "using NV_ENC_PARAMS_RC_VBR averageBitRate %d", + averageBitRate_int); + encCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_VBR; + encCfg.rcParams.averageBitRate = averageBitRate_int; + rc_set = 1; + } + } + } + } + if (!rc_set) + { + LOG(LOG_LEVEL_INFO, + "using default NV_ENC_PARAMS_RC_CONSTQP qp %d", + XH_NVENV_DEFAULT_QP); + encCfg.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CONSTQP; + encCfg.rcParams.constQP.qpInterP = XH_NVENV_DEFAULT_QP; + encCfg.rcParams.constQP.qpInterB = XH_NVENV_DEFAULT_QP; + encCfg.rcParams.constQP.qpIntra = XH_NVENV_DEFAULT_QP; + rc_set = 1; + } + encCfg.rcParams.cbQPIndexOffset = -1; + encCfg.encodeCodecConfig.h264Config.chromaFormatIDC = 1; + encCfg.encodeCodecConfig.h264Config.idrPeriod = NVENC_INFINITE_GOPLENGTH; + encCfg.encodeCodecConfig.h264Config.repeatSPSPPS = 1; + encCfg.encodeCodecConfig.h264Config.disableSPSPPS = 0; + encCfg.encodeCodecConfig.h264Config.outputAUD = 1; + + createEncodeParams.encodeConfig = &encCfg; + nv_error = g_enc_funcs.nvEncInitializeEncoder(lei->enc, + &createEncodeParams); + LOG(LOG_LEVEL_INFO, "nvEncInitializeEncoder rv %d", nv_error); + if (nv_error != NV_ENC_SUCCESS) + { + return 1; + } + + g_memset(&res, 0, sizeof(res)); + res.texture = tex; + res.target = GL_TEXTURE_2D; + + g_memset(®_res, 0, sizeof(reg_res)); + reg_res.version = NV_ENC_REGISTER_RESOURCE_VER; + reg_res.resourceType = NV_ENC_INPUT_RESOURCE_TYPE_OPENGL_TEX; + reg_res.width = width; + reg_res.height = height; + if (tex_format == XH_YUV420) + { + reg_res.pitch = width; + reg_res.bufferFormat = NV_ENC_BUFFER_FORMAT_NV12; + } + else + { + reg_res.pitch = width * 4; + reg_res.bufferFormat = NV_ENC_BUFFER_FORMAT_AYUV; + } + reg_res.resourceToRegister = &res; + reg_res.bufferUsage = NV_ENC_INPUT_IMAGE; + nv_error = g_enc_funcs.nvEncRegisterResource(lei->enc, ®_res); + LOG(LOG_LEVEL_INFO, "nvEncRegisterResource rv %d", nv_error); + if (nv_error != NV_ENC_SUCCESS) + { + return 1; + } + + g_memset(&mapInputResource, 0, sizeof(mapInputResource)); + mapInputResource.version = NV_ENC_LOCK_INPUT_BUFFER_VER; + mapInputResource.registeredResource = reg_res.registeredResource; + nv_error = g_enc_funcs.nvEncMapInputResource(lei->enc, &mapInputResource); + LOG(LOG_LEVEL_INFO, "nvEncMapInputResource rv %d", nv_error); + if (nv_error != NV_ENC_SUCCESS) + { + return 1; + } + + g_memset(&bitstreamParams, 0, sizeof(bitstreamParams)); + bitstreamParams.version = NV_ENC_CREATE_BITSTREAM_BUFFER_VER; + nv_error = g_enc_funcs.nvEncCreateBitstreamBuffer(lei->enc, + &bitstreamParams); + LOG(LOG_LEVEL_INFO, "nvEncCreateBitstreamBuffer rv %d", nv_error); + if (nv_error != NV_ENC_SUCCESS) + { + return 1; + } + + lei->bitstreamBuffer = bitstreamParams.bitstreamBuffer; + lei->mappedResource = mapInputResource.mappedResource; + lei->mappedBufferFmt = mapInputResource.mappedBufferFmt; + lei->registeredResource = reg_res.registeredResource; + lei->width = width; + lei->height = height; + + *ei = lei; + + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_nvenc_delete_encoder(struct enc_info *ei) +{ + g_enc_funcs.nvEncUnmapInputResource(ei->enc, ei->mappedResource); + g_enc_funcs.nvEncUnregisterResource(ei->enc, ei->registeredResource); + g_enc_funcs.nvEncDestroyBitstreamBuffer(ei->enc, ei->bitstreamBuffer); + g_enc_funcs.nvEncDestroyEncoder(ei->enc); + g_free(ei); + return 0; +} + +/*****************************************************************************/ +enum encoder_result +xorgxrdp_helper_nvenc_encode(struct enc_info *ei, int tex, + void *cdata, int *cdata_bytes) +{ + NV_ENC_PIC_PARAMS picParams; + NV_ENC_LOCK_BITSTREAM lockBitstream; + NVENCSTATUS nv_error; + enum encoder_result rv; + + g_memset(&picParams, 0, sizeof(picParams)); + picParams.version = NV_ENC_PIC_PARAMS_VER; + picParams.inputBuffer = ei->mappedResource; + picParams.bufferFmt = ei->mappedBufferFmt; + picParams.inputWidth = ei->width; + picParams.inputHeight = ei->height; + picParams.outputBitstream = ei->bitstreamBuffer; + picParams.inputTimeStamp = ei->frameCount; + picParams.pictureStruct = NV_ENC_PIC_STRUCT_FRAME; + if (xrdp_invalidate > 0 || ei->frameCount == 0) + { + picParams.encodePicFlags |= NV_ENC_PIC_FLAG_FORCEIDR; + picParams.encodePicFlags |= NV_ENC_PIC_FLAG_OUTPUT_SPSPPS; + picParams.encodePicFlags |= NV_ENC_PIC_FLAG_FORCEINTRA; + LOG(LOG_LEVEL_INFO, "Forcing NVENC H264 IDR SPSPPS for frame id: %d," + "invalidate is: %d", ei->frameCount, xrdp_invalidate); + xrdp_invalidate = MAX(0, xrdp_invalidate - 1); + } + nv_error = g_enc_funcs.nvEncEncodePicture(ei->enc, &picParams); + rv = ENCODER_ERROR; + if (nv_error != NV_ENC_SUCCESS) + { + LOG(LOG_LEVEL_ERROR, "error nvEncEncodePicture %d", nv_error); + return rv; + } + g_memset(&lockBitstream, 0, sizeof(lockBitstream)); + lockBitstream.version = NV_ENC_LOCK_BITSTREAM_VER; + lockBitstream.outputBitstream = ei->bitstreamBuffer; + lockBitstream.doNotWait = 0; + nv_error = g_enc_funcs.nvEncLockBitstream(ei->enc, &lockBitstream); + if (nv_error == NV_ENC_SUCCESS) + { + if (*cdata_bytes >= ((int) (lockBitstream.bitstreamSizeInBytes))) + { + g_memcpy(cdata, lockBitstream.bitstreamBufferPtr, + lockBitstream.bitstreamSizeInBytes); + *cdata_bytes = lockBitstream.bitstreamSizeInBytes; + rv = INCREMENTAL_FRAME_ENCODED; + } + else + { + LOG(LOG_LEVEL_ERROR, "error not enough room %d %d", + *cdata_bytes, + (int) (lockBitstream.bitstreamSizeInBytes)); + } + nv_error = g_enc_funcs.nvEncUnlockBitstream(ei->enc, + lockBitstream.outputBitstream); + if (nv_error != NV_ENC_SUCCESS) + { + LOG(LOG_LEVEL_ERROR, "error nvEncUnlockBitstream %d", + nv_error); + } + } + else + { + LOG(LOG_LEVEL_ERROR, "error nvEncLockBitstream %d", + nv_error); + } + ++ei->frameCount; + if (rv == INCREMENTAL_FRAME_ENCODED + && (picParams.encodePicFlags & NV_ENC_PIC_FLAG_FORCEIDR)) + { + return KEY_FRAME_ENCODED; + } + return rv; +} diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.h b/xorgxrdp_helper/xorgxrdp_helper_nvenc.h new file mode 100644 index 0000000000..152b69df37 --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.h @@ -0,0 +1,33 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _XORGXRDP_HELPER_NVENC_H +#define _XORGXRDP_HELPER_NVENC_H + +int +xorgxrdp_helper_nvenc_init(void); +int +xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, + int tex_format, struct enc_info **ei); +int +xorgxrdp_helper_nvenc_delete_encoder(struct enc_info *ei); +enum encoder_result +xorgxrdp_helper_nvenc_encode(struct enc_info *ei, int tex, + void *cdata, int *cdata_bytes); + +#endif diff --git a/xorgxrdp_helper/xorgxrdp_helper_shaders.c b/xorgxrdp_helper/xorgxrdp_helper_shaders.c new file mode 100644 index 0000000000..0c60adeef5 --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper_shaders.c @@ -0,0 +1,410 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* GLSL shaders + * this file is not compiled directly, it is included in + * xorgxrdp_helper_x11.c */ + +static const GLchar g_vs[] = "\ +#version 460 core\n\ +layout(location = 0) in vec4 position;\n\ +out vec4 gl_Position;\n\ +void main(void)\n\ +{\n\ + gl_Position = vec4(position.xy, 0.0, 1.0);\n\ +}\n"; + +static const GLchar g_fs_copy[] = "\ +#version 460 core\n\ +uniform sampler2D tex;\n\ +uniform vec2 tex_size;\n\ +out texture gl_FragColor;\n\ +void main(void)\n\ +{\n\ + gl_FragColor = texture(tex, gl_FragCoord.xy / tex_size);\n\ +}\n"; + +/* +Original version. +static const GLchar g_fs_rgb_to_yuv420[] = "\ +uniform sampler2D tex;\n\ +uniform vec2 tex_size;\n\ +uniform vec4 ymath;\n\ +uniform vec4 umath;\n\ +uniform vec4 vmath;\n\ +void main(void)\n\ +{\n\ + vec4 pix;\n\ + float x;\n\ + float y;\n\ + x = gl_FragCoord.x;\n\ + y = gl_FragCoord.y;\n\ + if (y < tex_size.y)\n\ + {\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(ymath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + y = floor(y - tex_size.y) * 2.0 + 0.5;\n\ + if (mod(x, 2.0) < 1.0)\n\ + {\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix += texture2D(tex, vec2(x + 1.0, y) / tex_size);\n\ + pix += texture2D(tex, vec2(x, y + 1.0) / tex_size);\n\ + pix += texture2D(tex, vec2(x + 1.0, y + 1.0) / tex_size);\n\ + pix /= 4.0;\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix += texture2D(tex, vec2(x - 1.0, y) / tex_size);\n\ + pix += texture2D(tex, vec2(x, y + 1.0) / tex_size);\n\ + pix += texture2D(tex, vec2(x - 1.0, y + 1.0) / tex_size);\n\ + pix /= 4.0;\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + }\n\ +}\n"; +*/ + +// Optimized version. +static const GLchar g_fs_rgb_to_yuv420[] = "\ +#version 460 core\n\ +uniform sampler2D tex;\n\ +uniform vec2 tex_size;\n\ +uniform vec4 ymath;\n\ +uniform vec4 umath;\n\ +uniform vec4 vmath;\n\ +out vec4 FragColor;\n\ +void main()\n\ +{\n\ + vec4 pix;\n\ + float x = gl_FragCoord.x;\n\ + float y = gl_FragCoord.y;\n\ + if (y < tex_size.y)\n\ + {\n\ + pix = texture(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix.r = clamp(dot(ymath, pix), 0.0, 1.0);\n\ + }\n\ + else\n\ + {\n\ + y = floor(y - tex_size.y) * 2.0 + 0.5;\n\ + float xOffset = mod(x, 2.0);\n\ + float antiAliasMod = xOffset == 0 ? 1.0 : -1.0;\n\ + pix = texture(tex, vec2(x - xOffset, y) / tex_size);\n\ + pix += texture(tex, vec2(x - xOffset + antiAliasMod, y) / tex_size);\n\ + pix += texture(tex, vec2(x - xOffset, y + 1.0) / tex_size);\n\ + pix += texture(tex, vec2(x - xOffset + antiAliasMod, y + 1.0) / tex_size);\n\ + pix /= 4.0;\n\ + pix.a = 1.0;\n\ + if (xOffset < 1.0)\n\ + {\n\ + pix.r = clamp(dot(umath, pix), 0.0, 1.0);\n\ + }\n\ + else\n\ + {\n\ + pix.r = clamp(dot(vmath, pix), 0.0, 1.0);\n\ + }\n\ + }\n\ + FragColor = vec4(pix.r, 0.0, 0.0, 1.0);\n\ +}\n"; + +static const GLchar g_fs_rgb_to_yuv422[] = "\ +uniform sampler2D tex;\n\ +uniform vec2 tex_size;\n\ +uniform vec4 ymath;\n\ +uniform vec4 umath;\n\ +uniform vec4 vmath;\n\ +void main(void)\n\ +{\n\ + vec4 pix;\n\ + vec4 pix1;\n\ + vec4 pixs;\n\ + float x;\n\ + float y;\n\ + x = gl_FragCoord.x;\n\ + x = floor(x) * 2.0 + 0.5;\n\ + y = gl_FragCoord.y;\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix1 = texture2D(tex, vec2(x + 1.0, y) / tex_size);\n\ + pixs = (pix + pix1) / 2.0;\n\ + pix.a = 1.0;\n\ + pix1.a = 1.0;\n\ + pixs.a = 1.0;\n\ + pix.r = dot(ymath, pix);\n\ + pix.g = dot(umath, pixs);\n\ + pix.b = dot(ymath, pix1);\n\ + pix.a = dot(vmath, pixs);\n\ + gl_FragColor = clamp(pix, 0.0, 1.0);\n\ +}\n"; + +static const GLchar g_fs_rgb_to_yuv444[] = "\ +uniform sampler2D tex;\n\ +uniform vec2 tex_size;\n\ +uniform vec4 ymath;\n\ +uniform vec4 umath;\n\ +uniform vec4 vmath;\n\ +void main(void)\n\ +{\n\ + vec4 pix;\n\ + pix = texture2D(tex, gl_FragCoord.xy / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(dot(vmath, pix), dot(umath, pix), dot(ymath, pix), 1.0);\n\ + gl_FragColor = clamp(pix, 0.0, 1.0);\n\ +}\n"; + +/* +RGB + 00 10 20 30 40 50 60 70 80 90 A0 B0 C0 D0 E0 F0 + 01 11 21 31 41 51 61 71 81 91 A1 B1 C1 D1 E1 F1 + 02 12 22 32 42 52 62 72 82 92 A2 B2 C2 D2 E2 F2 + 03 13 23 33 43 53 63 73 83 93 A3 B3 C3 D3 E3 F3 + 04 14 24 34 44 54 64 74 84 94 A4 B4 C4 D4 E4 F4 + 05 15 25 35 45 55 65 75 85 95 A5 B5 C5 D5 E5 F5 + 06 16 26 36 46 56 66 76 86 96 A6 B6 C6 D6 E6 F6 + 07 17 27 37 47 57 67 77 87 97 A7 B7 C7 D7 E7 F7 + 08 18 28 38 48 58 68 78 88 98 A8 B8 C8 D8 E8 F8 + 09 19 29 39 49 59 69 79 89 99 A9 B9 C9 D9 E9 F9 + 0A 1A 2A 3A 4A 5A 6A 7A 8A 9A AA BA CA DA EA FA + 0B 1B 2B 3B 4B 5B 6B 7B 8B 9B AB BB CB DB EB FB + 0C 1C 2C 3C 4C 5C 6C 7C 8C 9C AC BC CC DC EC FC + 0D 1D 2D 3D 4D 5D 6D 7D 8D 9D AD BD CD DD ED FD + 0E 1E 2E 3E 4E 5E 6E 7E 8E 9E AE BE CE DE EE FE + 0F 1F 2F 3F 4F 5F 6F 7F 8F 9F AF BF CF DF EF FF + +MAIN VIEW - NV12 + + /---------------------Y-----------------------\ + 00 10 20 30 40 50 60 70 80 90 A0 B0 C0 D0 E0 F0 + 01 11 21 31 41 51 61 71 81 91 A1 B1 C1 D1 E1 F1 + ... + 0F 1F 2F 3F 4F 5F 6F 7F 8F 9F AF BF CF DF EF FF + + /U /V /U /V /U /V /U /V /U /V /U /V /U /V /U /V + 00 00 20 20 40 40 60 60 80 80 A0 A0 C0 C0 E0 E0 + 02 02 22 22 42 42 62 62 82 82 A2 A2 C2 C2 E2 E2 + ... + 0E 0E 2E 2E 4E 4E 6E 6E 8E 8E AE AE CE CE EE EE +*/ +/* + TODO: This appears to be identical to g_fs_rgb_to_yuv420. + Keeping this now because this is a dev prototype, but we might + want to remove this in the future. +*/ +static const GLchar g_fs_rgb_to_yuv420_mv[] = "\ +uniform sampler2D tex;\n\ +uniform vec2 tex_size;\n\ +uniform vec4 ymath;\n\ +uniform vec4 umath;\n\ +uniform vec4 vmath;\n\ +void main(void)\n\ +{\n\ + vec4 pix;\n\ + float x;\n\ + float y;\n\ + x = gl_FragCoord.x;\n\ + y = gl_FragCoord.y;\n\ + if (y < tex_size.y)\n\ + {\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(ymath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + y = floor(y - tex_size.y) * 2.0 + 0.5;\n\ + if (mod(x, 2.0) < 1.0)\n\ + {\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix += texture2D(tex, vec2(x + 1.0, y) / tex_size);\n\ + pix += texture2D(tex, vec2(x, y + 1.0) / tex_size);\n\ + pix += texture2D(tex, vec2(x + 1.0, y + 1.0) / tex_size);\n\ + pix /= 4.0;\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix += texture2D(tex, vec2(x - 1.0, y) / tex_size);\n\ + pix += texture2D(tex, vec2(x, y + 1.0) / tex_size);\n\ + pix += texture2D(tex, vec2(x - 1.0, y + 1.0) / tex_size);\n\ + pix /= 4.0;\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + }\n\ +}\n"; + +/* +AUXILIARY VIEW - NV12 + + /---------------------U-----------------------\ + 01 11 21 31 41 51 61 71 81 91 A1 B1 C1 D1 E1 F1 + 03 13 23 33 43 53 63 73 83 93 A3 B3 C3 D3 E3 F4 + ... + 0F 1F 2F 3F 4F 5F 6F 7F 8F 9F AF BF CF DF EF FF + /---------------------V-----------------------\ + 01 11 21 31 41 51 61 71 81 91 A1 B1 C1 D1 E1 F1 + 03 13 23 33 43 53 63 73 83 93 A3 B3 C3 D3 E3 F4 + ... + 0F 1F 2F 3F 4F 5F 6F 7F 8F 9F AF BF CF DF EF FF + ... (8 LINES U, 8 LINES V, REPEAT) + + /U /V /U /V /U /V /U /V /U /V /U /V /U /V /U /V + 10 10 30 30 50 50 70 70 90 90 B0 B0 D0 D0 F0 F0 + 12 12 32 32 52 52 72 72 92 92 B2 B2 D2 D2 F2 F2 + ... + 1E 1E 3E 3E 5E 5E 7E 7E 9E 9E BE BE DE DE FE FE +*/ +static const GLchar g_fs_rgb_to_yuv420_av[] = "\ +uniform sampler2D tex;\n\ +uniform vec2 tex_size;\n\ +uniform vec4 umath;\n\ +uniform vec4 vmath;\n\ +void main(void)\n\ +{\n\ + vec4 pix;\n\ + float x;\n\ + float y;\n\ + float y1;\n\ + x = gl_FragCoord.x;\n\ + y = gl_FragCoord.y;\n\ + if (y < tex_size.y)\n\ + {\n\ + y1 = mod(y, 16.0);\n\ + if (y1 < 8.0)\n\ + {\n\ + y = floor(y / 16.0) * 8.0 + y1;\n\ + y = floor(y) * 2.0 + 1.5;\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + y = floor(y / 16.0) * 8.0 + (y1 - 8.0);\n\ + y = floor(y) * 2.0 + 1.5;\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + }\n\ + else\n\ + {\n\ + y = floor(y - tex_size.y) * 2.0 + 0.5;\n\ + if (mod(x, 2.0) < 1.0)\n\ + {\n\ + pix = texture2D(tex, vec2(x + 1.0, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + }\n\ +}\n"; + +/* +AUXILIARY VIEW V2 - NV12 + + /----------U----------\ /----------V----------\ + 10 30 50 70 90 B0 D0 F0 10 30 50 70 90 B0 D0 F0 + 11 31 51 71 91 B1 D1 F1 11 31 51 71 91 B1 D1 F1 + ... + 1F 3F 5F 7F 9F BF DF FF 1F 3F 5F 7F 9F BF DF FF + + /----------U----------\ /----------V----------\ + 01 21 41 61 81 A1 C1 E1 01 21 41 61 81 A1 C1 E1 + 03 23 43 63 83 A3 C3 E3 03 23 43 63 83 A3 C3 E3 + ... + 0F 2F 4F 6F 8F AF CF EF 0F 2F 4F 6F 8F AF CF EF +*/ +static const GLchar g_fs_rgb_to_yuv420_av_v2[] = "\ +uniform sampler2D tex;\n\ +uniform vec2 tex_size;\n\ +uniform vec4 umath;\n\ +uniform vec4 vmath;\n\ +void main(void)\n\ +{\n\ + vec4 pix;\n\ + float x;\n\ + float y;\n\ + float x1;\n\ + x = gl_FragCoord.x;\n\ + y = gl_FragCoord.y;\n\ + x1 = tex_size.x / 2.0;\n\ + if (y < tex_size.y)\n\ + {\n\ + if (x < x1)\n\ + {\n\ + x = floor(x) * 2.0 + 1.5;\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + x = floor(x - x1) * 2.0 + 1.5;\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + }\n\ + else\n\ + {\n\ + y = floor(y - tex_size.y) * 2.0 + 1.5;\n\ + if (x < x1)\n\ + {\n\ + x = floor(x) * 2.0 + 0.5;\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + x = floor(x - x1) * 2.0 + 0.5;\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + }\n\ +}\n"; diff --git a/xorgxrdp_helper/xorgxrdp_helper_x11.c b/xorgxrdp_helper/xorgxrdp_helper_x11.c new file mode 100644 index 0000000000..8bee522ce4 --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper_x11.c @@ -0,0 +1,788 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2020-2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Currently nvenc requires GLX because NVidia's EGL does not have + * EGL_NOK_texture_from_pixmap extension but NVidia's GLX does have + * GLX_EXT_texture_from_pixmap. We require one if those, + * also, yami required EGL because it used dma bufs. + * I do not think any vendor's GLX support dma bufs */ +/* Things like render on one GPU and encode with another is possible + * but not supported now. */ +/* One suggestion about dma bufs and GLX, one can use the DRI3 + * extension to get dma buffs for pixmaps */ + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include +#include +#include + +#include + +#include + +#include "arch.h" +#include "os_calls.h" +#include "string_calls.h" +#include "xorgxrdp_helper.h" +#include "xorgxrdp_helper_x11.h" +#include "xorgxrdp_helper_glx.h" +#include "xorgxrdp_helper_egl.h" +#include "log.h" + +#if defined(XRDP_NVENC) +#include "xorgxrdp_helper_nvenc.h" +#endif + +#if defined(XRDP_YAMI) +#include "xorgxrdp_helper_yami.h" +#endif + +/* X11 */ +Display *g_display = NULL; +static int g_x_socket = 0; +int g_screen_num = 0; +static Screen *g_screen = NULL; +Window g_root_window = None; +static Visual *g_vis = NULL; +static GC g_gc; + +/* encoders: nvenc or yami */ +struct enc_funcs +{ + int (*init)(void); + int (*create_enc)(int width, int height, int tex, int tex_format, + struct enc_info **ei); + int (*destroy_enc)(struct enc_info *ei); + enum encoder_result (*encode)(struct enc_info *ei, int tex, + void *cdata, int *cdata_bytes); +}; + +static struct enc_funcs g_enc_funcs[] = +{ + { +#if defined(XRDP_YAMI) + xorgxrdp_helper_yami_init, + xorgxrdp_helper_yami_create_encoder, + xorgxrdp_helper_yami_delete_encoder, + xorgxrdp_helper_yami_encode +#else + NULL, NULL, NULL, NULL +#endif + }, + { +#if defined(XRDP_NVENC) + xorgxrdp_helper_nvenc_init, + xorgxrdp_helper_nvenc_create_encoder, + xorgxrdp_helper_nvenc_delete_encoder, + xorgxrdp_helper_nvenc_encode +#else + NULL, NULL, NULL, NULL +#endif + } +}; + +/* GL interface: EGL or GLX */ +struct inf_funcs +{ + int (*init)(void); + int (*create_image)(Pixmap pixmap, inf_image_t *inf_image); + int (*destroy_image)(inf_image_t inf_image); + int (*bind_tex_image)(inf_image_t inf_image); + int (*release_tex_image)(inf_image_t inf_image); +}; + +static struct inf_funcs g_inf_funcs[] = +{ + { + xorgxrdp_helper_inf_egl_init, + xorgxrdp_helper_inf_egl_create_image, + xorgxrdp_helper_inf_egl_destroy_image, + xorgxrdp_helper_inf_egl_bind_tex_image, + xorgxrdp_helper_inf_egl_release_tex_image + }, + { + xorgxrdp_helper_inf_glx_init, + xorgxrdp_helper_inf_glx_create_image, + xorgxrdp_helper_inf_glx_destroy_image, + xorgxrdp_helper_inf_glx_bind_tex_image, + xorgxrdp_helper_inf_glx_release_tex_image + } +}; + +/* 0 = EGL, 1 = GLX */ +/* 0 = yami, 1 = nvenc */ +#define INF_EGL 0 +#define INF_GLX 1 +#define ENC_YAMI 0 +#define ENC_NVENC 1 +static int g_inf = INF_EGL; +static int g_enc = ENC_YAMI; + +struct mon_info +{ + int width; + int height; + Pixmap pixmap; + inf_image_t inf_image; + GLuint bmp_texture; + GLuint enc_texture; + int tex_format; + GLfloat *(*get_vertices)(GLuint *vertices_bytes, + GLuint *vertices_pointes, + int num_crects, struct xh_rect *crects, + int width, int height); + struct xh_rect viewport; + struct enc_info *ei; +}; + +#define MAX_MON 16 +static struct mon_info g_mons[MAX_MON]; + +static GLuint g_quad_vao = 0; +static GLuint g_fb = 0; + +#define XH_SHADERCOPY 0 +#define XH_SHADERRGB2YUV420 1 +#define XH_SHADERRGB2YUV422 2 +#define XH_SHADERRGB2YUV444 3 +#define XH_SHADERRGB2YUV420MV 4 +#define XH_SHADERRGB2YUV420AV 5 +#define XH_SHADERRGB2YUV420AVV2 6 + +#define XH_NUM_SHADERS 7 + +struct shader_info +{ + GLuint vertex_shader; + GLuint fragment_shader; + GLuint program; + GLint tex_loc; + GLint tex_size_loc; + GLint ymath_loc; + GLint umath_loc; + GLint vmath_loc; + int current_matrix; +}; +static struct shader_info g_si[XH_NUM_SHADERS]; + +/* *INDENT-OFF* */ +static const GLfloat g_vertices[] = +{ + -1.0f, 1.0f, + -1.0f, -1.0f, + 1.0f, 1.0f, + 1.0f, -1.0f +}; +/* *INDENT-ON* */ + +struct rgb2yuv_matrix +{ + GLfloat ymath[4]; + GLfloat umath[4]; + GLfloat vmath[4]; +}; + +static struct rgb2yuv_matrix g_rgb2yux_matrix[3] = +{ + { + /* yuv bt601 lagecy */ + { 66.0 / 256.0, 129.0 / 256.0, 25.0 / 256.0, 16.0 / 256.0 }, + { -38.0 / 256.0, -74.0 / 256.0, 112.0 / 256.0, 128.0 / 256.0 }, + { 112.0 / 256.0, -94.0 / 256.0, -18.0 / 256.0, 128.0 / 256.0 } + }, + { + /* yuv bt709 full range, used in gfx h264 */ + { 54.0 / 256.0, 183.0 / 256.0, 18.0 / 256.0, 0.0 / 256.0 }, + { -29.0 / 256.0, -99.0 / 256.0, 128.0 / 256.0, 128.0 / 256.0 }, + { 128.0 / 256.0, -116.0 / 256.0, -12.0 / 256.0, 128.0 / 256.0 } + }, + { + /* yuv remotefx and gfx progressive remotefx */ + { 0.299000, 0.587000, 0.114000, 0.0 }, + { -0.168935, -0.331665, 0.500590, 0.5 }, + { 0.499830, -0.418531, -0.081282, 0.5 } + } +}; + +#include "xorgxrdp_helper_shaders.c" + +/*****************************************************************************/ +int +xorgxrdp_helper_x11_init(void) +{ + const GLchar *vsource[XH_NUM_SHADERS]; + const GLchar *fsource[XH_NUM_SHADERS]; + GLint linked; + GLint compiled; + GLint vlength; + GLint flength; + GLuint quad_vbo; + int index; + int gl_ver; + int major_opcode, first_event, first_error; + + /* x11 */ + g_display = XOpenDisplay(0); + if (g_display == NULL) + { + return 1; + } + g_x_socket = XConnectionNumber(g_display); + g_screen_num = DefaultScreen(g_display); + g_screen = ScreenOfDisplay(g_display, g_screen_num); + g_root_window = RootWindowOfScreen(g_screen); + g_vis = XDefaultVisual(g_display, g_screen_num); + g_gc = DefaultGC(g_display, 0); + if (XQueryExtension(g_display, "NV-CONTROL", &major_opcode, &first_event, + &first_error)) + { + LOG(LOG_LEVEL_INFO, "detected NVIDIA XServer"); + g_inf = INF_GLX; + g_enc = ENC_NVENC; + if (g_inf_funcs[g_inf].init() != 0) + { + LOG(LOG_LEVEL_ERROR, "GLX init failed"); + return 1; + } + LOG(LOG_LEVEL_INFO, "using GLX"); + } + else + { + g_inf = INF_EGL; + g_enc = ENC_YAMI; + if (g_inf_funcs[g_inf].init() != 0) + { + LOG(LOG_LEVEL_ERROR, "EGL init failed"); + return 1; + } + LOG(LOG_LEVEL_INFO, "using EGL"); + } + gl_ver = epoxy_gl_version(); + LOG(LOG_LEVEL_INFO, "gl_ver %d", gl_ver); + if (gl_ver < 30) + { + LOG(LOG_LEVEL_ERROR, "gl_ver too old %d", gl_ver); + return 1; + } + LOG(LOG_LEVEL_INFO, "vendor: %s", + (const char *) glGetString(GL_VENDOR)); + LOG(LOG_LEVEL_INFO, "version: %s", + (const char *) glGetString(GL_VERSION)); + /* create vertex array */ + glGenVertexArrays(1, &g_quad_vao); + glBindVertexArray(g_quad_vao); + glGenBuffers(1, &quad_vbo); + glBindBuffer(GL_ARRAY_BUFFER, quad_vbo); + glBufferData(GL_ARRAY_BUFFER, sizeof(g_vertices), g_vertices, + GL_STATIC_DRAW); + glEnableVertexAttribArray(0); + glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, sizeof(float) * 2, NULL); + glGenFramebuffers(1, &g_fb); + glBindBuffer(GL_ARRAY_BUFFER, 0); + glBindVertexArray(0); + glDeleteBuffers(1, &quad_vbo); + + /* create copy shader */ + vsource[XH_SHADERCOPY] = g_vs; + fsource[XH_SHADERCOPY] = g_fs_copy; + /* create rgb2yuv shader */ + vsource[XH_SHADERRGB2YUV420] = g_vs; + fsource[XH_SHADERRGB2YUV420] = g_fs_rgb_to_yuv420; + /* create rgb2yuv shader */ + vsource[XH_SHADERRGB2YUV422] = g_vs; + fsource[XH_SHADERRGB2YUV422] = g_fs_rgb_to_yuv422; + /* create rgb2yuv shader */ + vsource[XH_SHADERRGB2YUV444] = g_vs; + fsource[XH_SHADERRGB2YUV444] = g_fs_rgb_to_yuv444; + + vsource[XH_SHADERRGB2YUV420MV] = g_vs; + fsource[XH_SHADERRGB2YUV420MV] = g_fs_rgb_to_yuv420_mv; + + vsource[XH_SHADERRGB2YUV420AV] = g_vs; + fsource[XH_SHADERRGB2YUV420AV] = g_fs_rgb_to_yuv420_av; + + vsource[XH_SHADERRGB2YUV420AVV2] = g_vs; + fsource[XH_SHADERRGB2YUV420AVV2] = g_fs_rgb_to_yuv420_av_v2; + + for (index = 0; index < XH_NUM_SHADERS; index++) + { + g_si[index].vertex_shader = glCreateShader(GL_VERTEX_SHADER); + g_si[index].fragment_shader = glCreateShader(GL_FRAGMENT_SHADER); + vlength = g_strlen(vsource[index]); + flength = g_strlen(fsource[index]); + glShaderSource(g_si[index].vertex_shader, 1, + &(vsource[index]), &vlength); + glShaderSource(g_si[index].fragment_shader, 1, + &(fsource[index]), &flength); + glCompileShader(g_si[index].vertex_shader); + glGetShaderiv(g_si[index].vertex_shader, GL_COMPILE_STATUS, + &compiled); + LOG(LOG_LEVEL_INFO, "vertex_shader compiled %d", compiled); + glCompileShader(g_si[index].fragment_shader); + glGetShaderiv(g_si[index].fragment_shader, GL_COMPILE_STATUS, + &compiled); + LOG(LOG_LEVEL_INFO, "fragment_shader compiled %d", compiled); + g_si[index].program = glCreateProgram(); + glAttachShader(g_si[index].program, g_si[index].vertex_shader); + glAttachShader(g_si[index].program, g_si[index].fragment_shader); + glLinkProgram(g_si[index].program); + glGetProgramiv(g_si[index].program, GL_LINK_STATUS, &linked); + LOG(LOG_LEVEL_INFO, "linked %d", linked); + g_si[index].tex_loc = + glGetUniformLocation(g_si[index].program, "tex"); + g_si[index].tex_size_loc = + glGetUniformLocation(g_si[index].program, "tex_size"); + g_si[index].ymath_loc = + glGetUniformLocation(g_si[index].program, "ymath"); + g_si[index].umath_loc = + glGetUniformLocation(g_si[index].program, "umath"); + g_si[index].vmath_loc = + glGetUniformLocation(g_si[index].program, "vmath"); + LOG(LOG_LEVEL_INFO, "tex_loc %d " + "tex_size_loc %d ymath_loc %d umath_loc %d vmath_loc %d", + g_si[index].tex_loc, g_si[index].tex_size_loc, + g_si[index].ymath_loc, g_si[index].umath_loc, + g_si[index].vmath_loc); + /* set default matrix */ + glUseProgram(g_si[index].program); + if (g_si[index].ymath_loc >= 0) + { + glUniform4fv(g_si[index].ymath_loc, 1, g_rgb2yux_matrix[1].ymath); + } + if (g_si[index].umath_loc >= 0) + { + glUniform4fv(g_si[index].umath_loc, 1, g_rgb2yux_matrix[1].umath); + } + if (g_si[index].vmath_loc >= 0) + { + glUniform4fv(g_si[index].vmath_loc, 1, g_rgb2yux_matrix[1].vmath); + } + glUseProgram(0); + } + g_memset(g_mons, 0, sizeof(g_mons)); + if (g_enc_funcs[g_enc].init() != 0) + { + LOG(LOG_LEVEL_ERROR, "encoder init failed"); + return 1; + } + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_x11_get_wait_objs(intptr_t *objs, int *obj_count) +{ + objs[*obj_count] = g_x_socket; + (*obj_count)++; + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_x11_check_wait_objs(void) +{ + XEvent xevent; + + while (XPending(g_display) > 0) + { + LOG_DEVEL(LOG_LEVEL_INFO, "loop"); + XNextEvent(g_display, &xevent); + } + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_x11_delete_all_pixmaps(void) +{ + int index; + struct mon_info *mi; + + for (index = 0; index < MAX_MON; index++) + { + mi = g_mons + index; + if (mi->pixmap != 0) + { + g_enc_funcs[g_enc].destroy_enc(mi->ei); + glDeleteTextures(1, &(mi->bmp_texture)); + glDeleteTextures(1, &(mi->enc_texture)); + g_inf_funcs[g_inf].destroy_image(mi->inf_image); + XFreePixmap(g_display, mi->pixmap); + mi->pixmap = 0; + } + } + return 0; +} + +/*****************************************************************************/ +static GLfloat * +get_vertices_all(GLuint *vertices_bytes, GLuint *vertices_pointes, + int num_crects, struct xh_rect *crects, + int width, int height) +{ + GLfloat *vertices; + + (void)num_crects; + (void)crects; + (void)width; + (void)height; + + vertices = g_new(GLfloat, 12); + if (vertices == NULL) + { + return NULL; + } + vertices[0] = -1; + vertices[1] = 1; + vertices[2] = -1; + vertices[3] = -1; + vertices[4] = 1; + vertices[5] = 1; + vertices[6] = -1; + vertices[7] = -1; + vertices[8] = 1; + vertices[9] = 1; + vertices[10] = 1; + vertices[11] = -1; + *vertices_bytes = sizeof(GLfloat) * 12; + *vertices_pointes = 6; + return vertices; +} + +/*****************************************************************************/ +static GLfloat * +get_vertices420(GLuint *vertices_bytes, GLuint *vertices_pointes, + int num_crects, struct xh_rect *crects, + int width, int height) +{ + GLfloat *vertices; + GLfloat *vert; + GLfloat x1; + GLfloat x2; + GLfloat y1; + GLfloat y2; + int index; + GLfloat fwidth; + GLfloat fheight; + const GLfloat fac13 = 1.0 / 3.0; + const GLfloat fac23 = 2.0 / 3.0; + const GLfloat fac43 = 4.0 / 3.0; + struct xh_rect *crect; + + if (num_crects < 1) + { + return get_vertices_all(vertices_bytes, vertices_pointes, + num_crects, crects, width, height); + } + vertices = g_new(GLfloat, num_crects * 24); + if (vertices == NULL) + { + return NULL; + } + fwidth = width / 2.0; + fheight = height / 2.0; + for (index = 0; index < num_crects; index++) + { + crect = crects + index; + LOG_DEVEL(LOG_LEVEL_INFO, "rect index %d x %d y %d w %d h %d", + index, crect->x, crect->y, crect->w, crect->h); + x1 = crect->x / fwidth; + y1 = crect->y / fheight; + x2 = (crect->x + crect->w) / fwidth; + y2 = (crect->y + crect->h) / fheight; + vert = vertices + index * 24; + /* y box */ + vert[0] = x1 - 1.0; + vert[1] = y1 * fac23 - 1.0; + vert[2] = x1 - 1.0; + vert[3] = y2 * fac23 - 1.0; + vert[4] = x2 - 1.0; + vert[5] = y1 * fac23 - 1.0; + vert[6] = x1 - 1.0; + vert[7] = y2 * fac23 - 1.0; + vert[8] = x2 - 1.0; + vert[9] = y1 * fac23 - 1.0; + vert[10] = x2 - 1.0; + vert[11] = y2 * fac23 - 1.0; + /* uv box */ + vert[12] = x1 - 1.0; + vert[13] = (y1 * fac13 + fac43) - 1.0; + vert[14] = x1 - 1.0; + vert[15] = (y2 * fac13 + fac43) - 1.0; + vert[16] = x2 - 1.0; + vert[17] = (y1 * fac13 + fac43) - 1.0; + vert[18] = x1 - 1.0; + vert[19] = (y2 * fac13 + fac43) - 1.0; + vert[20] = x2 - 1.0; + vert[21] = (y1 * fac13 + fac43) - 1.0; + vert[22] = x2 - 1.0; + vert[23] = (y2 * fac13 + fac43) - 1.0; + } + *vertices_bytes = sizeof(GLfloat) * num_crects * 24; + *vertices_pointes = num_crects * 12; + return vertices; +} + +/*****************************************************************************/ +static GLfloat * +get_vertices444(GLuint *vertices_bytes, GLuint *vertices_pointes, + int num_crects, struct xh_rect *crects, + int width, int height) +{ + GLfloat *vertices; + GLfloat *vert; + GLfloat x1; + GLfloat x2; + GLfloat y1; + GLfloat y2; + int index; + GLfloat fwidth; + GLfloat fheight; + struct xh_rect *crect; + + if (num_crects < 1) + { + return get_vertices_all(vertices_bytes, vertices_pointes, + num_crects, crects, width, height); + } + vertices = g_new(GLfloat, num_crects * 12); + if (vertices == NULL) + { + return NULL; + } + fwidth = width / 2.0; + fheight = height / 2.0; + for (index = 0; index < num_crects; index++) + { + crect = crects + index; + x1 = crect->x / fwidth; + y1 = crect->y / fheight; + x2 = (crect->x + crect->w) / fwidth; + y2 = (crect->y + crect->h) / fheight; + vert = vertices + index * 12; + vert[0] = x1 - 1.0; + vert[1] = y1 - 1.0; + vert[2] = x1 - 1.0; + vert[3] = y2 - 1.0; + vert[4] = x2 - 1.0; + vert[5] = y1 - 1.0; + vert[6] = x1 - 1.0; + vert[7] = y2 - 1.0; + vert[8] = x2 - 1.0; + vert[9] = y1 - 1.0; + vert[10] = x2 - 1.0; + vert[11] = y2 - 1.0; + } + *vertices_bytes = sizeof(GLfloat) * num_crects * 12; + *vertices_pointes = num_crects * 6; + return vertices; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_x11_create_pixmap(int width, int height, int magic, + int con_id, int mon_id) +{ + struct mon_info *mi; + Pixmap pixmap; + XImage *ximage; + int img[64]; + inf_image_t inf_image; + GLuint bmp_texture; + GLuint enc_texture; + + mi = g_mons + mon_id % MAX_MON; + if (mi->pixmap != 0) + { + LOG(LOG_LEVEL_ERROR, "error already setup"); + return 1; + } + LOG(LOG_LEVEL_INFO, "width %d height %d, " + "magic 0x%8.8x, con_id %d mod_id %d", width, height, + magic, con_id, mon_id); + pixmap = XCreatePixmap(g_display, g_root_window, width, height, 24); + LOG(LOG_LEVEL_INFO, "pixmap %d", (int) pixmap); + + if (g_inf_funcs[g_inf].create_image(pixmap, &inf_image) != 0) + { + return 1; + } + LOG(LOG_LEVEL_INFO, "inf_image %p", (void *) inf_image); + + g_memset(img, 0, sizeof(img)); + img[0] = magic; + img[1] = con_id; + img[2] = mon_id; + ximage = XCreateImage(g_display, g_vis, 24, ZPixmap, 0, (char *) img, + 4, 4, 32, 0); + XPutImage(g_display, pixmap, g_gc, ximage, 0, 0, 0, 0, 4, 4); + XFree(ximage); + + glEnable(GL_TEXTURE_2D); + /* texture that gets encoded */ + glGenTextures(1, &enc_texture); + glBindTexture(GL_TEXTURE_2D, enc_texture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + if (g_enc == ENC_NVENC) + { + LOG(LOG_LEVEL_INFO, "using XH_YUV420"); + mi->tex_format = XH_YUV420; + glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height * 3 / 2, 0, + GL_RED, GL_UNSIGNED_BYTE, NULL); + mi->get_vertices = get_vertices420; + mi->viewport.x = 0; + mi->viewport.y = 0; + mi->viewport.w = width; + mi->viewport.h = height * 3 / 2; + } + else if (g_enc == ENC_YAMI) + { + LOG(LOG_LEVEL_INFO, "using XH_YUV422"); + mi->tex_format = XH_YUV422; + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width / 2, height, 0, + GL_RGBA, GL_UNSIGNED_INT_8_8_8_8, NULL); + mi->get_vertices = get_vertices444; /* same as 444 */ + mi->viewport.x = 0; + mi->viewport.y = 0; + mi->viewport.w = width / 2; + mi->viewport.h = height; + } + else + { + LOG(LOG_LEVEL_INFO, "using XH_YUV444"); + mi->tex_format = XH_YUV444; + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, + GL_RGBA, GL_UNSIGNED_INT_8_8_8_8, NULL); + mi->get_vertices = get_vertices444; + mi->viewport.x = 0; + mi->viewport.y = 0; + mi->viewport.w = width; + mi->viewport.h = height; + } + /* texture that binds with pixmap */ + glGenTextures(1, &bmp_texture); + glBindTexture(GL_TEXTURE_2D, bmp_texture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glBindTexture(GL_TEXTURE_2D, 0); + + if (g_enc_funcs[g_enc].create_enc(width, height, + enc_texture, mi->tex_format, + &(mi->ei)) != 0) + { + return 1; + } + + mi->pixmap = pixmap; + mi->inf_image = inf_image; + mi->enc_texture = enc_texture; + mi->width = width; + mi->height = height; + mi->bmp_texture = bmp_texture; + + return 0; +} + +/*****************************************************************************/ +static void +xorgxrdp_helper_x11_run_shader(int width, int height, + struct mon_info *mi, + struct shader_info *si, + int num_crects, struct xh_rect *crects) +{ + GLuint vao; + GLuint vbo; + GLfloat *vertices; + GLuint vertices_bytes; + GLuint vertices_pointes; + + /* rgb to yuv */ + glEnable(GL_TEXTURE_2D); + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, mi->bmp_texture); + g_inf_funcs[g_inf].bind_tex_image(mi->inf_image); + glBindFramebuffer(GL_FRAMEBUFFER, g_fb); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, mi->enc_texture, 0); + glUseProgram(si->program); + /* setup vertices from crects */ + vertices = mi->get_vertices(&vertices_bytes, &vertices_pointes, + num_crects, crects, width, height); + if (vertices == NULL) + { + LOG(LOG_LEVEL_ERROR, "error get_vertices failed num_crects %d", + num_crects); + return; + } + glGenVertexArrays(1, &vao); + glGenBuffers(1, &vbo); + glBindVertexArray(vao); + glBindBuffer(GL_ARRAY_BUFFER, vbo); + glBufferData(GL_ARRAY_BUFFER, vertices_bytes, vertices, GL_STATIC_DRAW); + glEnableVertexAttribArray(0); + glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, sizeof(float) * 2, NULL); + /* uniforms */ + glUniform2f(si->tex_size_loc, mi->width, mi->height); + /* viewport and draw */ + glViewport(mi->viewport.x, mi->viewport.y, mi->viewport.w, mi->viewport.h); + glDrawArrays(GL_TRIANGLES, 0, vertices_pointes); + /* cleanup */ + glBindBuffer(GL_ARRAY_BUFFER, 0); + glBindVertexArray(0); + glDeleteBuffers(1, &vbo); + glDeleteVertexArrays(1, &vao); + g_free(vertices); + glBindFramebuffer(GL_FRAMEBUFFER, 0); + g_inf_funcs[g_inf].release_tex_image(mi->inf_image); + glBindTexture(GL_TEXTURE_2D, 0); + glUseProgram(0); +} + +/*****************************************************************************/ +enum encoder_result +xorgxrdp_helper_x11_encode_pixmap(int left, int top, int width, int height, + int mon_id, int num_crects, + struct xh_rect *crects, + void *cdata, int *cdata_bytes) +{ + struct mon_info *mi; + struct shader_info *si; + enum encoder_result rv; + + mi = g_mons + mon_id % MAX_MON; + if ((width != mi->width) || (height != mi->height)) + { + LOG(LOG_LEVEL_ERROR, "error width %d should be %d " + "height %d should be %d", + width, mi->width, height, mi->height); + return ENCODER_ERROR; + } + si = g_si + mi->tex_format % XH_NUM_SHADERS; + xorgxrdp_helper_x11_run_shader(width, height, mi, si, num_crects, crects); + /* sync before encoding */ + XFlush(g_display); + glFinish(); + /* encode */ + rv = g_enc_funcs[g_enc].encode(mi->ei, mi->enc_texture, + cdata, cdata_bytes); + return rv; +} diff --git a/xorgxrdp_helper/xorgxrdp_helper_x11.h b/xorgxrdp_helper/xorgxrdp_helper_x11.h new file mode 100644 index 0000000000..b6ff1d973b --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper_x11.h @@ -0,0 +1,46 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2020-2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _XORGXRDP_HELPER_X11_H +#define _XORGXRDP_HELPER_X11_H + +/* generic type that can hold either a GLXPixmap(XID, unsigned int or long) + * or EGLSurface(void*) */ +typedef intptr_t inf_image_t; + +/* forward declaration used in xorgxrdp_helper_nvenc and xorgxrdp_helper_yami */ +struct enc_info; + +int +xorgxrdp_helper_x11_init(void); +int +xorgxrdp_helper_x11_get_wait_objs(intptr_t *objs, int *obj_count); +int +xorgxrdp_helper_x11_check_wait_objs(void); +int +xorgxrdp_helper_x11_delete_all_pixmaps(void); +int +xorgxrdp_helper_x11_create_pixmap(int width, int height, int magic, + int con_id, int mon_id); +enum encoder_result +xorgxrdp_helper_x11_encode_pixmap(int left, int top, int width, int height, + int mon_id, int num_crects, + struct xh_rect *crects, + void *cdata, int *cdata_bytes); + +#endif diff --git a/xorgxrdp_helper/xorgxrdp_helper_yami.c b/xorgxrdp_helper/xorgxrdp_helper_yami.c new file mode 100644 index 0000000000..5da3205ff8 --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper_yami.c @@ -0,0 +1,274 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include +#include +#include + +#include +#include + +#include "encoder_headers/yami_inf.h" + +#include "arch.h" +#include "os_calls.h" +#include "xorgxrdp_helper.h" +#include "xorgxrdp_helper_x11.h" +#include "xorgxrdp_helper_yami.h" +#include "log.h" + +extern EGLDisplay g_egl_display; /* in xorgxrdp_helper_egl.c */ +extern EGLContext g_egl_context; /* in xorgxrdp_helper_egl.c */ + +static char g_lib_name[] = "/opt/yami/lib/libyami_inf.so"; +static char g_lib_name1[] = "libyami_inf.so"; +static char g_func_name[] = "yami_get_funcs"; +/* if VA_DRM_DEVICE is not defined */ +static char g_drm_name[] = "/dev/dri/renderD128"; + +static yami_get_funcs_proc g_yami_get_funcs = NULL; + +static struct yami_funcs g_enc_funcs; + +static long g_lib = 0; + +static int g_fd = -1; + +struct enc_info +{ + int width; + int height; + int frameCount; + int pad0; + void *enc; +}; + +extern int xrdp_invalidate; + +/*****************************************************************************/ +int +xorgxrdp_helper_yami_init(void) +{ + int error; + int version; + char *drm_dev; + + LOG(LOG_LEVEL_INFO, "xorgxrdp_helper_yami_init:"); + g_lib = g_load_library(g_lib_name); + if (g_lib == 0) + { + g_lib = g_load_library(g_lib_name1); + if (g_lib == 0) + { + LOG(LOG_LEVEL_ERROR, "load library for %s/%s failed", + g_lib_name, g_lib_name1); + return 1; + } + } + else + { + LOG(LOG_LEVEL_INFO, "loaded library %s", g_lib_name); + } + g_yami_get_funcs = g_get_proc_address(g_lib, g_func_name); + if (g_yami_get_funcs == NULL) + { + LOG(LOG_LEVEL_ERROR, "get proc address for %s failed", g_func_name); + return 1; + } + g_memset(&g_enc_funcs, 0, sizeof(g_enc_funcs)); + error = g_yami_get_funcs(&g_enc_funcs, YI_VERSION_INT(YI_MAJOR, YI_MINOR)); + LOG(LOG_LEVEL_INFO, "yami_get_funcs rv %d", error); + if (error != YI_SUCCESS) + { + LOG(LOG_LEVEL_ERROR, "g_yami_get_funcs failed"); + return 1; + } + error = g_enc_funcs.yami_get_version(&version); + if (error != YI_SUCCESS) + { + LOG(LOG_LEVEL_ERROR, "yami_get_version failed"); + return 1; + } + if (version < YI_VERSION_INT(YI_MAJOR, YI_MINOR)) + { + LOG(LOG_LEVEL_ERROR, "yami version too old 0x%8.8x", + version); + return 1; + } + LOG(LOG_LEVEL_INFO, "yami version 0x%8.8x ok", version); + drm_dev = getenv("VA_DRM_DEVICE"); + if (drm_dev == NULL) + { + drm_dev = g_drm_name; + } + g_fd = g_file_open_ex(drm_dev, 1, 1, 0, 0); + if (g_fd == -1) + { + LOG(LOG_LEVEL_ERROR, "open %s failed", g_drm_name); + return 1; + } + LOG(LOG_LEVEL_INFO, "open %s ok, fd %d", g_drm_name, g_fd); + error = g_enc_funcs.yami_init(YI_TYPE_DRM, (void *) (size_t) g_fd); + if (error != 0) + { + LOG(LOG_LEVEL_ERROR, "yami_init failed"); + return 1; + } + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_yami_create_encoder(int width, int height, int tex, + int tex_format, struct enc_info **ei) +{ + int error; + struct enc_info *lei; + + LOG(LOG_LEVEL_INFO, "xorgxrdp_helper_yami_create_encoder:"); + lei = g_new0(struct enc_info, 1); + if (lei == NULL) + { + return 1; + } + error = g_enc_funcs.yami_encoder_create(&(lei->enc), width, height, + YI_TYPE_H264, + YI_H264_ENC_FLAGS_PROFILE_MAIN); + if (error != YI_SUCCESS) + { + LOG(LOG_LEVEL_ERROR, "yami_encoder_create failed %d", error); + g_free(lei); + return 1; + } + LOG(LOG_LEVEL_INFO, "yami_encoder_create ok"); + lei->width = width; + lei->height = height; + *ei = lei; + return 0; +} + +/*****************************************************************************/ +int +xorgxrdp_helper_yami_delete_encoder(struct enc_info *ei) +{ + LOG(LOG_LEVEL_INFO, "xorgxrdp_helper_yami_delete_encoder:"); + g_enc_funcs.yami_encoder_delete(ei->enc); + g_free(ei); + return 0; +} + +static const EGLint g_create_image_attr[] = +{ + EGL_NONE +}; + +/*****************************************************************************/ +enum encoder_result +xorgxrdp_helper_yami_encode(struct enc_info *ei, int tex, + void *cdata, int *cdata_bytes) +{ + EGLClientBuffer cb; + EGLImageKHR image; + EGLint stride; + EGLint offset; + int fd; + int error; + int fourcc; + int num_planes; + int force_key_frame = 0; + EGLuint64KHR modifiers; + + LOG_DEVEL(LOG_LEVEL_INFO, "tex %d", tex); + LOG_DEVEL(LOG_LEVEL_INFO, "g_egl_display %p", g_egl_display); + LOG_DEVEL(LOG_LEVEL_INFO, "g_egl_context %p", g_egl_context); + cb = (EGLClientBuffer) (size_t) tex; + image = eglCreateImageKHR(g_egl_display, g_egl_context, + EGL_GL_TEXTURE_2D_KHR, + cb, g_create_image_attr); + LOG_DEVEL(LOG_LEVEL_INFO, "image %p", image); + if (image == EGL_NO_IMAGE_KHR) + { + LOG(LOG_LEVEL_ERROR, "eglCreateImageKHR failed"); + return 1; + } + if (!eglExportDMABUFImageQueryMESA(g_egl_display, image, + &fourcc, &num_planes, + &modifiers)) + { + LOG(LOG_LEVEL_ERROR, "eglExportDMABUFImageQueryMESA failed"); + eglDestroyImageKHR(g_egl_display, image); + return 1; + } + LOG_DEVEL(LOG_LEVEL_INFO, "fourcc 0x%8.8X num_planes %d modifiers %d", + fourcc, num_planes, (int) modifiers); + if (num_planes != 1) + { + LOG(LOG_LEVEL_ERROR, "eglExportDMABUFImageQueryMESA return " + "bad num_planes %d", num_planes); + eglDestroyImageKHR(g_egl_display, image); + return 1; + } + if (!eglExportDMABUFImageMESA(g_egl_display, image, &fd, + &stride, &offset)) + { + LOG(LOG_LEVEL_ERROR, "eglExportDMABUFImageMESA failed"); + eglDestroyImageKHR(g_egl_display, image); + return 1; + } + LOG_DEVEL(LOG_LEVEL_INFO, "fd %d stride %d offset %d", fd, + stride, offset); + LOG_DEVEL(LOG_LEVEL_INFO, "width %d height %d", ei->width, ei->height); + error = g_enc_funcs.yami_encoder_set_fd_src(ei->enc, fd, + ei->width, ei->height, + stride, + stride * ei->height, + YI_YUY2); + LOG_DEVEL(LOG_LEVEL_INFO, "yami_encoder_set_fd_src rv %d", error); + if (error != YI_SUCCESS) + { + LOG(LOG_LEVEL_ERROR, "yami_encoder_set_fd_src failed"); + g_file_close(fd); + eglDestroyImageKHR(g_egl_display, image); + return 1; + } + if (xrdp_invalidate > 0 || ei->frameCount == 0) + { + LOG(LOG_LEVEL_INFO, "Forcing YAMI H264 Keyframe for frame id: %d," + "invalidate is: %d", ei->frameCount, xrdp_invalidate); + xrdp_invalidate = MAX(0, xrdp_invalidate - 1); + force_key_frame = 1; + } + error = g_enc_funcs.yami_encoder_encode(ei->enc, cdata, cdata_bytes, + force_key_frame); + LOG_DEVEL(LOG_LEVEL_INFO, "encoder_encode rv %d cdata_bytes %d", + error, *cdata_bytes); + if (error != YI_SUCCESS) + { + LOG(LOG_LEVEL_ERROR, "yami_encoder_encode failed"); + g_file_close(fd); + eglDestroyImageKHR(g_egl_display, image); + return 1; + } + g_file_close(fd); + eglDestroyImageKHR(g_egl_display, image); + return force_key_frame ? KEY_FRAME_ENCODED : INCREMENTAL_FRAME_ENCODED; +} diff --git a/xorgxrdp_helper/xorgxrdp_helper_yami.h b/xorgxrdp_helper/xorgxrdp_helper_yami.h new file mode 100644 index 0000000000..26989c7124 --- /dev/null +++ b/xorgxrdp_helper/xorgxrdp_helper_yami.h @@ -0,0 +1,33 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2022 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _XORGXRDP_HELPER_YAMI_H +#define _XORGXRDP_HELPER_YAMI_H + +int +xorgxrdp_helper_yami_init(void); +int +xorgxrdp_helper_yami_create_encoder(int width, int height, int tex, + int tex_format, struct enc_info **ei); +int +xorgxrdp_helper_yami_delete_encoder(struct enc_info *ei); +enum encoder_result +xorgxrdp_helper_yami_encode(struct enc_info *ei, int tex, + void *cdata, int *cdata_bytes); + +#endif diff --git a/xrdp/Makefile.am b/xrdp/Makefile.am index 3e11f4eebc..d121ca8b6d 100644 --- a/xrdp/Makefile.am +++ b/xrdp/Makefile.am @@ -19,6 +19,7 @@ AM_CPPFLAGS = \ $(IMLIB2_CFLAGS) XRDP_EXTRA_LIBS = +XRDP_EXTRA_SOURCES = if XRDP_RFXCODEC AM_CPPFLAGS += -DXRDP_RFXCODEC @@ -26,6 +27,13 @@ AM_CPPFLAGS += -I$(top_srcdir)/librfxcodec/include XRDP_EXTRA_LIBS += $(top_builddir)/librfxcodec/src/.libs/librfxencode.a endif +if XRDP_X264 +AM_CPPFLAGS += -DXRDP_X264 +AM_CPPFLAGS += $(XRDP_X264_CFLAGS) +XRDP_EXTRA_LIBS += $(XRDP_X264_LIBS) +XRDP_EXTRA_SOURCES += xrdp_encoder_x264.c xrdp_encoder_x264.h +endif + if XRDP_PIXMAN AM_CPPFLAGS += -DXRDP_PIXMAN AM_CPPFLAGS += $(PIXMAN_CFLAGS) @@ -60,10 +68,11 @@ xrdp_SOURCES = \ xrdp_process.c \ xrdp_region.c \ xrdp_types.h \ + xrdp_wm.c \ xrdp_egfx.c \ xrdp_egfx.h \ - xrdp_wm.c \ - xrdp_main_utils.c + xrdp_main_utils.c \ + $(XRDP_EXTRA_SOURCES) xrdp_LDADD = \ $(top_builddir)/common/libcommon.la \ diff --git a/xrdp/xrdp.h b/xrdp/xrdp.h index 0db9f35bf9..67c44b2e6d 100644 --- a/xrdp/xrdp.h +++ b/xrdp/xrdp.h @@ -502,6 +502,8 @@ xrdp_mm_egfx_send_planar_bitmap(struct xrdp_mm *self, struct xrdp_bitmap *bitmap, struct xrdp_rect *rect); int +xrdp_mm_can_resize(struct xrdp_mm *self); +int server_begin_update(struct xrdp_mod *mod); int server_end_update(struct xrdp_mod *mod); @@ -543,7 +545,8 @@ server_set_pointer_large(struct xrdp_mod *mod, int x, int y, char *data, char *mask, int bpp, int width, int height); int -server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects, +server_paint_rects_ex(struct xrdp_mod *mod, + int num_drects, short *drects, int num_crects, short *crects, char *data, int left, int top, int width, int height, diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 8c3d9d0b17..c8262a456d 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -32,7 +32,12 @@ #include "rfxcodec_encode.h" #endif +#ifdef XRDP_X264 +#include "xrdp_encoder_x264.h" +#endif + #define XRDP_SURCMD_PREFIX_BYTES 256 +#define OUT_DATA_BYTES_DEFAULT_SIZE (16 * 1024 * 1024) #ifdef XRDP_RFXCODEC /* LH3 LL3, HH3 HL3, HL2 LH2, LH1 HH2, HH1 HL1 todo check this */ @@ -43,6 +48,14 @@ static const unsigned char g_rfx_quantization_values[] = }; #endif +struct enc_rect +{ + short x1; + short y1; + short x2; + short y2; +}; + /*****************************************************************************/ static int process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc); @@ -77,6 +90,8 @@ xrdp_enc_data_done_destructor(void *item, void *closure) struct xrdp_encoder * xrdp_encoder_create(struct xrdp_mm *mm) { + LOG(LOG_LEVEL_INFO, "xrdp_encoder_create:"); + struct xrdp_encoder *self; struct xrdp_client_info *client_info; char buf[1024]; @@ -104,15 +119,23 @@ xrdp_encoder_create(struct xrdp_mm *mm) } self->mm = mm; - if (client_info->jpeg_codec_id != 0) + if (mm->egfx_flags & XRDP_EGFX_H264) { - LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_create: starting jpeg codec session"); - self->codec_id = client_info->jpeg_codec_id; + LOG(LOG_LEVEL_INFO, + "xrdp_encoder_create: starting h264 codec session gfx"); self->in_codec_mode = 1; - self->codec_quality = client_info->jpeg_prop[0]; - client_info->capture_code = 0; - client_info->capture_format = XRDP_a8b8g8r8; - self->process_enc = process_enc_jpg; + client_info->capture_code = 3; + client_info->capture_format = +#if XRDP_AVC444 + XRDP_yuv444_v2_stream_709fr; +#else + XRDP_nv12_709fr; +#endif + self->process_enc = process_enc_h264; + self->gfx = 1; +#if defined(XRDP_X264) + self->codec_handle = xrdp_encoder_x264_create(); +#endif } #ifdef XRDP_RFXCODEC else if (mm->egfx_flags & XRDP_EGFX_RFX_PRO) @@ -136,7 +159,8 @@ xrdp_encoder_create(struct xrdp_mm *mm) } else if (client_info->rfx_codec_id != 0) { - LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_create: starting rfx codec session"); + LOG_DEVEL(LOG_LEVEL_INFO, + "xrdp_encoder_create: starting rfx codec session"); self->codec_id = client_info->rfx_codec_id; self->in_codec_mode = 1; client_info->capture_code = 2; @@ -146,14 +170,29 @@ xrdp_encoder_create(struct xrdp_mm *mm) RFX_FORMAT_YUV, 0); } #endif + else if (client_info->jpeg_codec_id != 0) + { + LOG_DEVEL(LOG_LEVEL_INFO, + "xrdp_encoder_create: starting jpeg codec session"); + self->codec_id = client_info->jpeg_codec_id; + self->in_codec_mode = 1; + self->codec_quality = client_info->jpeg_prop[0]; + client_info->capture_code = 0; + client_info->capture_format = XRDP_a8b8g8r8; + self->process_enc = process_enc_jpg; + } else if (client_info->h264_codec_id != 0) { - LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_create: starting h264 codec session"); + LOG_DEVEL(LOG_LEVEL_INFO, + "xrdp_encoder_create: starting h264 codec session"); self->codec_id = client_info->h264_codec_id; self->in_codec_mode = 1; client_info->capture_code = 3; client_info->capture_format = XRDP_nv12; self->process_enc = process_enc_h264; +#if defined(XRDP_X264) + self->codec_handle = xrdp_encoder_x264_create(); +#endif } else { @@ -226,7 +265,12 @@ xrdp_encoder_delete(struct xrdp_encoder *self) rfxcodec_encode_destroy(self->codec_handle); } #endif - +#if defined(XRDP_X264) + else if (self->process_enc == process_enc_h264) + { + xrdp_encoder_x264_delete(self->codec_handle); + } +#endif /* destroy wait objects used for signalling */ g_delete_wait_obj(self->xrdp_encoder_event_to_proc); g_delete_wait_obj(self->xrdp_encoder_event_processed); @@ -280,13 +324,14 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) LOG_DEVEL(LOG_LEVEL_DEBUG, "process_enc_jpg: x %d y %d cx %d cy %d", x, y, cx, cy); out_data_bytes = MAX((cx + 4) * cy * 4, 8192); - if ((out_data_bytes < 1) || (out_data_bytes > 16 * 1024 * 1024)) + if ((out_data_bytes < 1) + || (out_data_bytes > OUT_DATA_BYTES_DEFAULT_SIZE)) { LOG_DEVEL(LOG_LEVEL_ERROR, "process_enc_jpg: error 2"); return 1; } - out_data = (char *) g_malloc(out_data_bytes + 256 + 2, 0); - if (out_data == 0) + out_data = g_new(char, out_data_bytes + XRDP_SURCMD_PREFIX_BYTES + 2); + if (out_data == NULL) { LOG_DEVEL(LOG_LEVEL_ERROR, "process_enc_jpg: error 3"); return 1; @@ -298,17 +343,24 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) enc->width, enc->height, enc->width * 4, x, y, cx, cy, quality, - out_data + 256 + 2, &out_data_bytes); + out_data + 256 + 2, + &out_data_bytes); if (error < 0) { - LOG_DEVEL(LOG_LEVEL_ERROR, "process_enc_jpg: jpeg error %d bytes %d", + LOG_DEVEL(LOG_LEVEL_ERROR, + "process_enc_jpg: jpeg error %d bytes %d", error, out_data_bytes); g_free(out_data); return 1; } - LOG_DEVEL(LOG_LEVEL_WARNING, "jpeg error %d bytes %d", error, out_data_bytes); - enc_done = (XRDP_ENC_DATA_DONE *) - g_malloc(sizeof(XRDP_ENC_DATA_DONE), 1); + LOG_DEVEL(LOG_LEVEL_WARNING, + "jpeg error %d bytes %d", error, out_data_bytes); + enc_done = g_new0(XRDP_ENC_DATA_DONE, 1); + if (enc_done == NULL) + { + LOG(LOG_LEVEL_INFO, "process_enc_jpg: error 3"); + return 1; + } enc_done->comp_bytes = out_data_bytes + 2; enc_done->pad_bytes = 256; enc_done->comp_pad_data = out_data; @@ -471,6 +523,330 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) } #endif +#define SAVE_VIDEO 0 + +#if SAVE_VIDEO +#include +#include +#include +#include + +static int n_save_data(const char *data, int data_size, int width, int height) +{ + int fd; + struct _header + { + char tag[4]; + int width; + int height; + int bytes_follow; + } header; + + fd = open("video.bin", O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + lseek(fd, 0, SEEK_END); + header.tag[0] = 'B'; + header.tag[1] = 'E'; + header.tag[2] = 'E'; + header.tag[3] = 'F'; + header.width = width; + header.height = height; + header.bytes_follow = data_size; + if (write(fd, &header, 16) != 16) + { + g_printf("save_data: write failed\n"); + } + + if (write(fd, data, data_size) != data_size) + { + g_printf("save_data: write failed\n"); + } + close(fd); + return 0; +} +#endif + +#if defined(XRDP_X264) + +/*****************************************************************************/ +static int +build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, + int width, int height) +{ + int index; + int x, y, cx, cy; + int location; + struct enc_rect rect; + const uint8_t qp = 22; // Default set by Microsoft. + const uint8_t r = 0; // Required to be 0. + const uint8_t p = 0; // Progressively encoded flag. + int qpVal = 0; + qpVal |= qp & 0x3F; + qpVal |= (r & 1) << 6; + qpVal |= (p & 1) << 7; + + out_uint32_le(s, rcount); /* numRegionRects */ + for (index = 0; index < rcount; index++) + { + location = index * 4; + x = rrects[location + 0]; + y = rrects[location + 1]; + cx = rrects[location + 2]; + cy = rrects[location + 3]; + /* RDPGFX_RECT16 */ + rect.x1 = MAX(0, x - 1); + rect.y1 = MAX(0, y - 1); + rect.x2 = MIN(x + cx + 1, width); + rect.y2 = MIN(y + cy + 1, height); + out_uint16_le(s, rect.x1); + out_uint16_le(s, rect.y1); + out_uint16_le(s, rect.x2); + out_uint16_le(s, rect.y2); + } + for (index = 0; index < rcount; index++) + { + // 2.2.4.4.2 RDPGFX_AVC420_QUANT_QUALITY + out_uint8(s, qpVal); /* qp */ + out_uint8(s, 100); /* quality level 0..100 (Microsoft uses 100) */ + } + int comp_bytes_pre = 4 + rcount * 8 + rcount * 2; + return comp_bytes_pre; +} + +static int +build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) +{ + int index; + int x; + int y; + int cx; + int cy; + int out_data_bytes; +#if XRDP_AVC444 + int out_data_bytes1; +#endif + int rcount; + short *rrects; + int error; + char *out_data; + int out_data_alloc_size; + XRDP_ENC_DATA_DONE *enc_done; + struct fifo *fifo_processed; + tbus mutex; + tbus event_processed; + struct stream ls; + struct stream *s; + int comp_bytes_pre; +#if XRDP_AVC444 + int comp_bytes_pre1; +#endif + int enc_done_flags; + int scr_width; + int scr_height; + + LOG(LOG_LEVEL_DEBUG, "process_enc_x264:"); + LOG(LOG_LEVEL_DEBUG, "process_enc_x264: num_crects %d num_drects %d", + enc->num_crects, enc->num_drects); + + scr_width = self->mm->wm->screen->width; + scr_height = self->mm->wm->screen->height; + + fifo_processed = self->fifo_processed; + mutex = self->mutex; + event_processed = self->xrdp_encoder_event_processed; + + rcount = enc->num_drects; + rrects = enc->drects; + if (rcount > 15) + { + rcount = enc->num_crects; + rrects = enc->crects; + } + + out_data_bytes = OUT_DATA_BYTES_DEFAULT_SIZE; + index = XRDP_SURCMD_PREFIX_BYTES + 16 + 2 + enc->num_drects * 8; + out_data_alloc_size = out_data_bytes + index; + out_data = g_new(char, out_data_alloc_size); + if (out_data == NULL) + { + return 0; + } + + s = &ls; + g_memset(s, 0, sizeof(struct stream)); + ls.data = out_data + XRDP_SURCMD_PREFIX_BYTES; + ls.p = ls.data; + ls.size = out_data_alloc_size; + +#if XRDP_AVC444 + out_data_bytes1 = 0; + comp_bytes_pre1 = 0; +#endif + + if (self->gfx) + { +#if XRDP_AVC444 + /* size of avc420EncodedBitmapstream1 */ + s_push_layer(s, mcs_hdr, 4); +#endif + /* RFX_AVC420_METABLOCK */ + comp_bytes_pre = build_rfx_avc420_metablock(s, rrects, rcount, + scr_width, scr_height); + enc_done_flags = 1; + } + else + { + out_uint32_le(s, 0); /* flags */ + out_uint32_le(s, 0); /* session id */ + out_uint16_le(s, enc->width); /* src_width */ + out_uint16_le(s, enc->height); /* src_height */ + out_uint16_le(s, enc->width); /* dst_width */ + out_uint16_le(s, enc->height); /* dst_height */ + out_uint16_le(s, rcount); + for (index = 0; index < rcount; index++) + { + x = rrects[index * 4 + 0]; + y = rrects[index * 4 + 1]; + cx = rrects[index * 4 + 2]; + cy = rrects[index * 4 + 3]; + out_uint16_le(s, x); + out_uint16_le(s, y); + out_uint16_le(s, cx); + out_uint16_le(s, cy); + } + s_push_layer(s, iso_hdr, 4); + comp_bytes_pre = 4 + 4 + 2 + 2 + 2 + 2 + 2 + rcount * 8 + 4; + enc_done_flags = 0; + } + error = 0; + if (enc->flags & 1) + { + /* already compressed */ + uint8_t *ud = (uint8_t *) (enc->data); + int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24); + if ((cbytes < 1) || (cbytes > out_data_bytes)) + { + LOG(LOG_LEVEL_INFO, "process_enc_h264: bad h264 bytes %d", cbytes); + g_free(out_data); + return 0; + } + LOG(LOG_LEVEL_DEBUG, + "process_enc_h264: already compressed and size is %d", cbytes); + out_data_bytes = cbytes; + g_memcpy(s->p, enc->data + 4, out_data_bytes); + } + else + { +#if defined(XRDP_X264) + error = xrdp_encoder_x264_encode(self->codec_handle, 0, + enc->width, enc->height, 0, + enc->data, + s->p, &out_data_bytes); +#endif + } + LOG_DEVEL(LOG_LEVEL_TRACE, + "process_enc_h264: xrdp_encoder_x264_encode rv %d " + "out_data_bytes %d width %d height %d", + error, out_data_bytes, enc->width, enc->height); + if (error != 0) + { + LOG_DEVEL(LOG_LEVEL_TRACE, + "process_enc_h264: xrdp_encoder_x264_encode failed rv %d", + error); + g_free(out_data); + return 0; + } + +#if XRDP_AVC444 + + s->p += out_data_bytes; + + /* chroma 444 */ + /* RFX_AVC420_METABLOCK */ + comp_bytes_pre1 = build_rfx_avc420_metablock(s, rrects, rcount, + scr_width, scr_height); + out_data_bytes1 = OUT_DATA_BYTES_DEFAULT_SIZE; + error = xrdp_encoder_x264_encode(self->codec_handle, 0, + enc->width, enc->height, 0, + enc->data, + s->p, &out_data_bytes1); + if (error != 0) + { + LOG_DEVEL(LOG_LEVEL_TRACE, + "process_enc_h264: xrdp_encoder_x264_encode failed rv %d", + error); + g_free(out_data); + return 0; + } + s->p += out_data_bytes1; + s_push_layer(s, sec_hdr, 0); + s_pop_layer(s, mcs_hdr); + out_uint32_le(s, comp_bytes_pre + out_data_bytes); + s_pop_layer(s, sec_hdr); + + s->end = s->p; +#else + s->end = s->p + out_data_bytes; +#endif + + if (s->iso_hdr != NULL) + { + /* not used in gfx */ + s_pop_layer(s, iso_hdr); + out_uint32_le(s, out_data_bytes); + } + +#if SAVE_VIDEO + n_save_data(s->p, out_data_bytes, enc->width, enc->height); +#endif + + enc_done = g_new0(XRDP_ENC_DATA_DONE, 1); + if (enc_done == NULL) + { + return 0; + } +#if XRDP_AVC444 + enc_done->comp_bytes = 4 + comp_bytes_pre + + out_data_bytes + + comp_bytes_pre1 + + out_data_bytes1; +#else + enc_done->comp_bytes = comp_bytes_pre + out_data_bytes; +#endif + enc_done->pad_bytes = XRDP_SURCMD_PREFIX_BYTES; + enc_done->comp_pad_data = out_data; + enc_done->enc = enc; + enc_done->last = 1; + enc_done->cx = scr_width; + enc_done->cy = scr_height; + enc_done->flags = enc_done_flags; + +#if 0 + g_writeln("comp_bytes_pre %d out_data_bytes %d comp_bytes_pre1 %d out_data_bytes1 %d", + comp_bytes_pre, out_data_bytes, comp_bytes_pre1, out_data_bytes1); + g_hexdump(enc_done->comp_pad_data + enc_done->pad_bytes, enc_done->comp_bytes); +#endif + + /* done with msg */ + /* inform main thread done */ + tc_mutex_lock(mutex); + fifo_add_item(fifo_processed, enc_done); + tc_mutex_unlock(mutex); + /* signal completion for main thread */ + g_set_wait_obj(event_processed); + + return 0; +} + +/*****************************************************************************/ +/* called from encoder thread */ +static int +process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) +{ + return build_enc_h264(self, enc); +} + +#else + /*****************************************************************************/ /* called from encoder thread */ static int @@ -480,6 +856,8 @@ process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) return 0; } +#endif + /** * Encoder thread main loop *****************************************************************************/ @@ -503,7 +881,7 @@ proc_enc_msg(void *arg) LOG_DEVEL(LOG_LEVEL_INFO, "proc_enc_msg: thread is running"); self = (struct xrdp_encoder *) arg; - if (self == 0) + if (self == NULL) { LOG_DEVEL(LOG_LEVEL_DEBUG, "proc_enc_msg: self nil"); return 0; @@ -541,7 +919,7 @@ proc_enc_msg(void *arg) if (g_is_wait_obj_set(lterm_obj)) /* xrdp_mm term */ { - LOG_DEVEL(LOG_LEVEL_DEBUG, "proc_enc_msg: xrdp_mm term"); + LOG(LOG_LEVEL_INFO, "proc_enc_msg: xrdp_mm term"); break; } diff --git a/xrdp/xrdp_encoder.h b/xrdp/xrdp_encoder.h index a5c32a58a2..d2447560ae 100644 --- a/xrdp/xrdp_encoder.h +++ b/xrdp/xrdp_encoder.h @@ -42,8 +42,12 @@ struct xrdp_enc_data { struct xrdp_mod *mod; int num_drects; + int pad0; + // Damage Rects (Damage as reported by X/drivers) short *drects; /* 4 * num_drects */ int num_crects; + int pad1; + // Change Rects (Actual change being communicated) short *crects; /* 4 * num_crects */ char *data; int left; diff --git a/xrdp/xrdp_encoder_x264.c b/xrdp/xrdp_encoder_x264.c new file mode 100644 index 0000000000..495d012753 --- /dev/null +++ b/xrdp/xrdp_encoder_x264.c @@ -0,0 +1,202 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2016 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * x264 Encoder + */ + +#if defined(HAVE_CONFIG_H) +#include +#endif + +#include +#include +#include +#include +#include + +#include "xrdp.h" +#include "arch.h" +#include "os_calls.h" +#include "xrdp_encoder_x264.h" + +struct x264_encoder +{ + x264_t *x264_enc_han; + char *yuvdata; + x264_param_t x264_params; + int width; + int height; +}; + +struct x264_global +{ + struct x264_encoder encoders[16]; +}; + +/*****************************************************************************/ +void * +xrdp_encoder_x264_create(void) +{ + struct x264_global *xg; + + LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_encoder_x264_create:"); + xg = (struct x264_global *) g_malloc(sizeof(struct x264_global), 1); + if (xg == 0) + { + return 0; + } + return xg; +} + +/*****************************************************************************/ +int +xrdp_encoder_x264_delete(void *handle) +{ + struct x264_global *xg; + struct x264_encoder *xe; + int index; + + if (handle == 0) + { + return 0; + } + xg = (struct x264_global *) handle; + for (index = 0; index < 16; index++) + { + xe = &(xg->encoders[index]); + if (xe->x264_enc_han != 0) + { + x264_encoder_close(xe->x264_enc_han); + } + g_free(xe->yuvdata); + } + g_free(xg); + return 0; +} + +/*****************************************************************************/ +int +xrdp_encoder_x264_encode(void *handle, int session, + int width, int height, int format, const char *data, + char *cdata, int *cdata_bytes) +{ + struct x264_global *xg; + struct x264_encoder *xe; + const char *src8; + char *dst8; + int index; + x264_nal_t *nals; + int num_nals; + int frame_size; + int frame_area; + + x264_picture_t pic_in; + x264_picture_t pic_out; + + LOG(LOG_LEVEL_TRACE, "xrdp_encoder_x264_encode:"); + xg = (struct x264_global *) handle; + xe = &(xg->encoders[session]); + if ((xe->x264_enc_han == 0) || (xe->width != width) || (xe->height != height)) + { + if (xe->x264_enc_han != 0) + { + x264_encoder_close(xe->x264_enc_han); + xe->x264_enc_han = 0; + g_free(xe->yuvdata); + xe->yuvdata = 0; + } + if ((width > 0) && (height > 0)) + { + //x264_param_default_preset(&(xe->x264_params), "superfast", "zerolatency"); + x264_param_default_preset(&(xe->x264_params), "ultrafast", "zerolatency"); + xe->x264_params.i_threads = 1; + xe->x264_params.i_width = width; + xe->x264_params.i_height = height; + xe->x264_params.i_fps_num = 24; + xe->x264_params.i_fps_den = 1; + //xe->x264_params.b_cabac = 1; + //xe->x264_params.i_bframe = 0; + //xe->x264_params.rc.i_rc_method = X264_RC_CQP; + //xe->x264_params.rc.i_qp_constant = 23; + //x264_param_apply_profile(&(xe->x264_params), "high"); + x264_param_apply_profile(&(xe->x264_params), "main"); + //x264_param_apply_profile(&(xe->x264_params), "baseline"); + xe->x264_enc_han = x264_encoder_open(&(xe->x264_params)); + if (xe->x264_enc_han == 0) + { + return 1; + } + xe->yuvdata = (char *) g_malloc(width * height * 2, 0); + if (xe->yuvdata == 0) + { + x264_encoder_close(xe->x264_enc_han); + xe->x264_enc_han = 0; + return 2; + } + } + xe->width = width; + xe->height = height; + } + + if ((data != 0) && (xe->x264_enc_han != 0)) + { + src8 = data; + dst8 = xe->yuvdata; + for (index = 0; index < height; index++) + { + g_memcpy(dst8, src8, width); + src8 += width; + dst8 += xe->x264_params.i_width; + } + + src8 = data; + src8 += width * height; + dst8 = xe->yuvdata; + + frame_area = xe->x264_params.i_width * xe->x264_params.i_height - 1; + dst8 += frame_area; + for (index = 0; index < height; index += 2) + { + g_memcpy(dst8, src8, width); + src8 += width; + dst8 += xe->x264_params.i_width; + } + + g_memset(&pic_in, 0, sizeof(pic_in)); + pic_in.img.i_csp = X264_CSP_NV12; + pic_in.img.i_plane = 2; + pic_in.img.plane[0] = (unsigned char *) (xe->yuvdata); + pic_in.img.plane[1] = (unsigned char *) (xe->yuvdata + frame_area); + pic_in.img.i_stride[0] = width; + pic_in.img.i_stride[1] = xe->x264_params.i_width; + num_nals = 0; + frame_size = x264_encoder_encode(xe->x264_enc_han, &nals, &num_nals, + &pic_in, &pic_out); + LOG(LOG_LEVEL_TRACE, "i_type %d", pic_out.i_type); + if (frame_size < 1) + { + return 3; + } + if (*cdata_bytes < frame_size) + { + return 4; + } + g_memcpy(cdata, nals[0].p_payload, frame_size); + *cdata_bytes = frame_size; + } + return 0; +} diff --git a/xrdp/xrdp_encoder_x264.h b/xrdp/xrdp_encoder_x264.h new file mode 100644 index 0000000000..0ccc15fbe5 --- /dev/null +++ b/xrdp/xrdp_encoder_x264.h @@ -0,0 +1,36 @@ +/** + * xrdp: A Remote Desktop Protocol server. + * + * Copyright (C) Jay Sorg 2016 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Yami Encoder + */ + +#ifndef _XRDP_ENCODER_X264_H +#define _XRDP_ENCODER_X264_H + +#include "arch.h" + +void * +xrdp_encoder_x264_create(void); +int +xrdp_encoder_x264_delete(void *handle); +int +xrdp_encoder_x264_encode(void *handle, int session, + int width, int height, int format, const char *data, + char *cdata, int *cdata_bytes); + +#endif + diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index 8769a61d97..33b875a831 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -27,17 +27,15 @@ #include "guid.h" #include "ms-rdpedisp.h" #include "ms-rdpbcgr.h" - #include "scp.h" - #include - #include "xrdp_encoder.h" #include "xrdp_sockets.h" #include "xrdp_egfx.h" +#include "libxrdp.h" +#include "xrdp_channel.h" #include - /* Forward declarations */ static int xrdp_mm_chansrv_connect(struct xrdp_mm *self, const char *port); @@ -49,8 +47,6 @@ struct xrdp_mm * xrdp_mm_create(struct xrdp_wm *owner) { struct xrdp_mm *self; - char buf[1024]; - int pid; self = (struct xrdp_mm *)g_malloc(sizeof(struct xrdp_mm), 1); self->wm = owner; @@ -58,16 +54,6 @@ xrdp_mm_create(struct xrdp_wm *owner) self->login_names->auto_free = 1; self->login_values = list_create(); self->login_values->auto_free = 1; - self->resize_queue = list_create(); - self->resize_queue->auto_free = 1; - - self->uid = -1; /* Never good to default UIDs to 0 */ - - pid = g_getpid(); - /* setup wait objects for signalling */ - g_snprintf(buf, sizeof(buf), "xrdp_%8.8x_resize_ready", pid); - self->resize_ready = g_create_wait_obj(buf); - self->resize_data = NULL; LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_mm_create: bpp %d mcs_connection_type %d " "jpeg_codec_id %d v3_codec_id %d rfx_codec_id %d " @@ -170,6 +156,7 @@ xrdp_mm_delete(struct xrdp_mm *self) list_delete(self->resize_queue); g_free(self->resize_data); g_delete_wait_obj(self->resize_ready); + xrdp_egfx_shutdown_full(self->egfx); g_free(self); } @@ -1092,50 +1079,280 @@ xrdp_mm_egfx_send_planar_bitmap(struct xrdp_mm *self, /******************************************************************************/ static int -dynamic_monitor_open_response(intptr_t id, int chan_id, int creation_status) +xrdp_mm_egfx_invalidate_all(struct xrdp_mm *self) { - struct xrdp_process *pro; - struct xrdp_wm *wm; - struct stream *s; - int bytes; + struct xrdp_rect xr_rect; + struct xrdp_bitmap *screen; + int error; - LOG_DEVEL(LOG_LEVEL_TRACE, "dynamic_monitor_open_response: chan_id %d " - "creation_status 0x%8.8x", chan_id, creation_status); - if (creation_status != 0) + LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_invalidate_all:"); + + screen = self->wm->screen; + + xr_rect.left = 0; + xr_rect.top = 0; + xr_rect.right = screen->width; + xr_rect.bottom = screen->height; + if (self->wm->screen_dirty_region == NULL) { - LOG(LOG_LEVEL_ERROR, "dynamic_monitor_open_response: error"); - return 1; + self->wm->screen_dirty_region = xrdp_region_create(self->wm); } - pro = (struct xrdp_process *) id; - wm = pro->wm; - make_stream(s); - init_stream(s, 1024); - out_uint32_le(s, 5); /* DISPLAYCONTROL_PDU_TYPE_CAPS */ - out_uint32_le(s, 8 + 12); - out_uint32_le(s, CLIENT_MONITOR_DATA_MAXIMUM_MONITORS); /* MaxNumMonitors */ - out_uint32_le(s, 4096); /* MaxMonitorAreaFactorA */ - out_uint32_le(s, 2048); /* MaxMonitorAreaFactorB */ - s_mark_end(s); - bytes = (int) (s->end - s->data); - libxrdp_drdynvc_data(wm->session, chan_id, s->data, bytes); - free_stream(s); + error = xrdp_region_add_rect(self->wm->screen_dirty_region, &xr_rect); + return error; +} + +/******************************************************************************/ +int +advance_resize_state_machine(struct xrdp_mm *mm, + enum display_resize_state new_state) +{ + struct display_control_monitor_layout_data *description = mm->resize_data; + LOG_DEVEL(LOG_LEVEL_INFO, + "advance_resize_state_machine:" + " Processing resize to: %d x %d." + " Advancing state from %s to %s." + " Previous state took %d MS.", + description->description.session_width, + description->description.session_height, + XRDP_DISPLAY_RESIZE_STATE_TO_STR(description->state), + XRDP_DISPLAY_RESIZE_STATE_TO_STR(new_state), + g_time3() - description->last_state_update_timestamp); + description->state = new_state; + description->last_state_update_timestamp = g_time3(); + g_set_wait_obj(mm->resize_ready); return 0; } /******************************************************************************/ static int -dynamic_monitor_close_response(intptr_t id, int chan_id) +xrdp_mm_egfx_caps_advertise(void *user, int caps_count, + int *versions, int *flagss) { - LOG_DEVEL(LOG_LEVEL_TRACE, "dynamic_monitor_close_response:"); + struct xrdp_mm *self; + struct xrdp_bitmap *screen; + int index; + int best_index; + int best_h264_index; + int best_pro_index; + int error; + int version; + int flags; + + LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_caps_advertise:"); + self = (struct xrdp_mm *) user; + screen = self->wm->screen; + if (screen->data == NULL) + { + LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_caps_advertise: can not do gfx"); + } + best_index = -1; + best_h264_index = -1; + best_pro_index = -1; + for (index = 0; index < caps_count; index++) + { + version = versions[index]; + flags = flagss[index]; + LOG(LOG_LEVEL_INFO, " version 0x%8.8x flags 0x%8.8x (index: %d)", version, flags, index); + switch (version) + { + case XR_RDPGFX_CAPVERSION_8: + best_pro_index = index; + break; + case XR_RDPGFX_CAPVERSION_81: + if (flags & XR_RDPGFX_CAPS_FLAG_AVC420_ENABLED) + { + best_h264_index = index; + } + best_pro_index = index; + break; + case XR_RDPGFX_CAPVERSION_10: + best_pro_index = index; + break; + case XR_RDPGFX_CAPVERSION_101: + best_pro_index = index; + break; + case XR_RDPGFX_CAPVERSION_102: + best_pro_index = index; + break; + case XR_RDPGFX_CAPVERSION_103: + best_pro_index = index; + break; + case XR_RDPGFX_CAPVERSION_104: + if (!(flags & XR_RDPGFX_CAPS_FLAG_AVC_DISABLED)) + { + best_h264_index = index; + } + best_pro_index = index; + break; + case XR_RDPGFX_CAPVERSION_105: + best_pro_index = index; + break; + case XR_RDPGFX_CAPVERSION_106: + best_pro_index = index; + break; + } + } + if (best_pro_index >= 0) + { + best_index = best_pro_index; + self->egfx_flags = 2; + } + if (best_h264_index >= 0) /* prefer h264, todo use setting in xrdp.ini for this */ + { +#if defined(XRDP_X264) || defined(XRDP_NVENC) + best_index = best_h264_index; + self->egfx_flags = 1; +#endif + } + if (best_index >= 0) + { + LOG(LOG_LEVEL_INFO, " replying version 0x%8.8x flags 0x%8.8x", + versions[best_index], flagss[best_index]); + error = xrdp_egfx_send_capsconfirm(self->egfx, + versions[best_index], + flagss[best_index]); + LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_caps_advertise: xrdp_egfx_send_capsconfirm " + "error %d best_index %d", error, best_index); + error = xrdp_egfx_send_reset_graphics(self->egfx, + screen->width, screen->height, + self->wm->client_info->display_sizes.monitorCount, + self->wm->client_info->display_sizes.minfo_wm); + LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_caps_advertise: xrdp_egfx_send_reset_graphics " + "error %d monitorCount %d", + error, self->wm->client_info->display_sizes.monitorCount); + self->egfx_up = 1; + xrdp_egfx_send_create_surface(self->egfx, self->egfx->surface_id, + screen->width, screen->height, + XR_PIXEL_FORMAT_XRGB_8888); + xrdp_egfx_send_map_surface(self->egfx, self->egfx->surface_id, 0, 0); + self->encoder = xrdp_encoder_create(self); + xrdp_mm_egfx_invalidate_all(self); + + if (self->resize_data != NULL + && self->resize_data->state == WMRZ_EGFX_INITALIZING) + { + advance_resize_state_machine(self, WMRZ_EGFX_INITIALIZED); + } + LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_caps_advertise: egfx created."); + if (self->gfx_delay_autologin) + { + self->gfx_delay_autologin = 0; + xrdp_wm_set_login_state(self->wm, WMLS_START_CONNECT); + } + } + else + { + struct xrdp_rect lrect; + + LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_caps_advertise: no good gfx, canceling"); + lrect.left = 0; + lrect.top = 0; + lrect.right = screen->width; + lrect.bottom = screen->height; + self->wm->client_info->gfx = 0; + xrdp_encoder_delete(self->encoder); + self->encoder = xrdp_encoder_create(self); + xrdp_bitmap_invalidate(screen, &lrect); + } return 0; } -/******************************************************************************/ +/*****************************************************************************/ static int -dynamic_monitor_data_first(intptr_t id, int chan_id, char *data, int bytes, - int total_bytes) +xrdp_mm_update_module_frame_ack(struct xrdp_mm *self) { - LOG_DEVEL(LOG_LEVEL_TRACE, "dynamic_monitor_data_first:"); + int fif; + struct xrdp_encoder *encoder; + + encoder = self->encoder; + fif = encoder->frames_in_flight; + if (encoder->frame_id_client + fif > encoder->frame_id_server) + { + if (encoder->frame_id_server > encoder->frame_id_server_sent) + { + LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_update_module_ack: frame_id_server %d", + encoder->frame_id_server); + encoder->frame_id_server_sent = encoder->frame_id_server; + self->mod->mod_frame_ack(self->mod, 0, encoder->frame_id_server); + } + } + return 0; +} + +static int +xrdp_mm_egfx_frame_ack(void *user, uint32_t queue_depth, int frame_id, + int frames_decoded) +{ + struct xrdp_mm *self; + struct xrdp_encoder *encoder; + + LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_egfx_frame_ack:"); + self = (struct xrdp_mm *) user; + encoder = self->encoder; + if (encoder == NULL) + { + LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_frame_ack: encoder is nil"); + return 0; + } + if (queue_depth == XR_SUSPEND_FRAME_ACKNOWLEDGEMENT) + { + LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_frame_ack: queue_depth %d frame_id %d " + "frames_decoded %d", queue_depth, frame_id, frames_decoded); + if (encoder->gfx_ack_off == 0) + { + LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_frame_ack: client request turn off frame acks."); + encoder->gfx_ack_off = 1; + frame_id = -1; + } + } + else + { + if (encoder->gfx_ack_off) + { + LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_egfx_frame_ack: client request turn on " + "frame acks"); + encoder->gfx_ack_off = 0; + } + } + LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_egfx_frame_ack: incoming %d, client %d, server %d", + frame_id, encoder->frame_id_client, encoder->frame_id_server); + if (frame_id < 0 || frame_id > encoder->frame_id_server) + { + /* if frame_id is negative or bigger then what server last sent + just ack all sent frames */ + /* some clients can send big number just to clear all + pending frames */ + encoder->frame_id_client = encoder->frame_id_server; + } + else + { + /* frame acks can come out of order so ignore older one */ + encoder->frame_id_client = MAX(frame_id, encoder->frame_id_client); + } + xrdp_mm_update_module_frame_ack(self); + return 0; +} + +/******************************************************************************/ +int +egfx_initialize(struct xrdp_mm *self) +{ + /* 0x100 RNS_UD_CS_SUPPORT_DYNVC_GFX_PROTOCOL */ + if (self->wm->client_info->mcs_early_capability_flags & 0x100) + { + LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_mm_drdynvc_up: gfx capable client"); + if (xrdp_egfx_create(self, &(self->egfx)) == 0) + { + self->egfx->user = self; + self->egfx->caps_advertise = xrdp_mm_egfx_caps_advertise; + self->egfx->frame_ack = xrdp_mm_egfx_frame_ack; + } + else + { + LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_mm_drdynvc_up: xrdp_egfx_create failed"); + return 1; + } + } return 0; } @@ -1162,27 +1379,6 @@ sync_dynamic_monitor_data(struct xrdp_wm *wm, MAXIMUM_MONITOR_SIZE); } -int -advance_resize_state_machine(struct xrdp_mm *mm, - enum display_resize_state new_state) -{ - struct display_control_monitor_layout_data *description = mm->resize_data; - LOG_DEVEL(LOG_LEVEL_INFO, - "advance_resize_state_machine:" - " Processing resize to: %d x %d." - " Advancing state from %s to %s." - " Previous state took %d MS.", - description->description.session_width, - description->description.session_height, - XRDP_DISPLAY_RESIZE_STATE_TO_STR(description->state), - XRDP_DISPLAY_RESIZE_STATE_TO_STR(new_state), - g_time3() - description->last_state_update_timestamp); - description->state = new_state; - description->last_state_update_timestamp = g_time3(); - g_set_wait_obj(mm->resize_ready); - return 0; -} - /******************************************************************************/ static int dynamic_monitor_data(intptr_t id, int chan_id, char *data, int bytes) @@ -1216,6 +1412,7 @@ dynamic_monitor_data(intptr_t id, int chan_id, char *data, int bytes) s = &ls; in_uint32_le(s, msg_type); in_uint32_le(s, msg_length); + LOG_DEVEL(LOG_LEVEL_DEBUG, "dynamic_monitor_data: msg_type %d msg_length %d", msg_type, msg_length); @@ -1240,6 +1437,7 @@ dynamic_monitor_data(intptr_t id, int chan_id, char *data, int bytes) } return 0; } + in_uint32_le(s, monitor_layout_size); if (monitor_layout_size != 40) { @@ -1269,7 +1467,8 @@ dynamic_monitor_data(intptr_t id, int chan_id, char *data, int bytes) list_add_item(wm->mm->resize_queue, (tintptr)display_size_data); g_set_wait_obj(wm->mm->resize_ready); LOG(LOG_LEVEL_DEBUG, "dynamic_monitor_data:" - " received width %d, received height %d.", display_size_data->session_width, display_size_data->session_height); + " received width %d, received height %d.", + display_size_data->session_width, display_size_data->session_height); return 0; } @@ -1282,6 +1481,55 @@ advance_error(int error, return error; } +/******************************************************************************/ +static int +dynamic_monitor_open_response(intptr_t id, int chan_id, int creation_status) +{ + struct xrdp_process *pro; + struct xrdp_wm *wm; + struct stream *s; + int bytes; + + LOG_DEVEL(LOG_LEVEL_TRACE, "dynamic_monitor_open_response: chan_id %d " + "creation_status 0x%8.8x", chan_id, creation_status); + if (creation_status != 0) + { + LOG(LOG_LEVEL_ERROR, "dynamic_monitor_open_response: error"); + return 1; + } + pro = (struct xrdp_process *) id; + wm = pro->wm; + make_stream(s); + init_stream(s, 1024); + out_uint32_le(s, 5); /* DISPLAYCONTROL_PDU_TYPE_CAPS */ + out_uint32_le(s, 8 + 12); + out_uint32_le(s, CLIENT_MONITOR_DATA_MAXIMUM_MONITORS); /* MaxNumMonitors */ + out_uint32_le(s, 4096); /* MaxMonitorAreaFactorA */ + out_uint32_le(s, 2048); /* MaxMonitorAreaFactorB */ + s_mark_end(s); + bytes = (int) (s->end - s->data); + libxrdp_drdynvc_data(wm->session, chan_id, s->data, bytes); + free_stream(s); + return 0; +} + +/******************************************************************************/ +static int +dynamic_monitor_close_response(intptr_t id, int chan_id) +{ + LOG_DEVEL(LOG_LEVEL_TRACE, "dynamic_monitor_close_response:"); + return 0; +} + +/******************************************************************************/ +static int +dynamic_monitor_data_first(intptr_t id, int chan_id, char *data, int bytes, + int total_bytes) +{ + LOG_DEVEL(LOG_LEVEL_TRACE, "dynamic_monitor_data_first:"); + return 0; +} + /******************************************************************************/ static int process_display_control_monitor_layout_data(struct xrdp_wm *wm) @@ -1289,6 +1537,9 @@ process_display_control_monitor_layout_data(struct xrdp_wm *wm) int error = 0; struct xrdp_mm *mm; struct xrdp_mod *module; + struct xrdp_rdp *rdp; + struct xrdp_sec *sec; + struct xrdp_channel *chan; LOG_DEVEL(LOG_LEVEL_TRACE, "process_display_control_monitor_layout_data:"); @@ -1326,6 +1577,56 @@ process_display_control_monitor_layout_data(struct xrdp_wm *wm) xrdp_encoder_delete(mm->encoder); mm->encoder = NULL; } + if (mm->egfx == 0) + { + advance_resize_state_machine(mm, WMRZ_SERVER_MONITOR_RESIZE); + } + else + { + advance_resize_state_machine(mm, WMRZ_EGFX_DELETE_SURFACE); + } + break; + case WMRZ_EGFX_DELETE_SURFACE: + if (error == 0 && module != 0) + { + xrdp_egfx_shutdown_delete_surface(mm->egfx); + } + advance_resize_state_machine(mm, WMRZ_EGFX_CONN_CLOSE); + break; + case WMRZ_EGFX_CONN_CLOSE: + if (error == 0 && module != 0) + { + xrdp_egfx_shutdown_close_connection(wm->mm->egfx); + } + advance_resize_state_machine(mm, WMRZ_EGFX_CONN_CLOSING); + break; + // Also processed in xrdp_egfx_close_response + case WMRZ_EGFX_CONN_CLOSING: + rdp = (struct xrdp_rdp *) (mm->wm->session->rdp); + sec = rdp->sec_layer; + chan = sec->chan_layer; + + // Continue to check to see if the connection is closed. If it + // ever is, advance the state machine! + if (chan->drdynvcs[mm->egfx->channel_id].status + == XRDP_DRDYNVC_STATUS_CLOSED + || (g_time3() - description->last_state_update_timestamp) > 100) + { + advance_resize_state_machine(mm, WMRZ_EGFX_CONN_CLOSED); + break; + } + g_set_wait_obj(mm->resize_ready); + break; + case WMRZ_EGFX_CONN_CLOSED: + advance_resize_state_machine(mm, WRMZ_EGFX_DELETE); + break; + case WRMZ_EGFX_DELETE: + if (error == 0 && module != 0) + { + xrdp_egfx_shutdown_delete(wm->mm->egfx); + mm->egfx = NULL; + mm->egfx_up = 0; + } advance_resize_state_machine(mm, WMRZ_SERVER_MONITOR_RESIZE); break; case WMRZ_SERVER_MONITOR_RESIZE: @@ -1338,9 +1639,10 @@ process_display_control_monitor_layout_data(struct xrdp_wm *wm) " mod_server_monitor_resize failed %d", error); return advance_error(error, mm); } - advance_resize_state_machine(mm, WMRZ_SERVER_VERSION_MESSAGE); + advance_resize_state_machine( + mm, WMRZ_SERVER_VERSION_MESSAGE_START); break; - case WMRZ_SERVER_VERSION_MESSAGE: + case WMRZ_SERVER_VERSION_MESSAGE_START: error = module->mod_server_version_message(module); if (error != 0) { @@ -1349,6 +1651,12 @@ process_display_control_monitor_layout_data(struct xrdp_wm *wm) " mod_server_version_message failed %d", error); return advance_error(error, mm); } + advance_resize_state_machine( + mm, WMRZ_SERVER_MONITOR_MESSAGE_PROCESSING); + break; + // Not processed here. Processed in server_reset + // case WMRZ_SERVER_MONITOR_MESSAGE_PROCESSING: + case WMRZ_SERVER_MONITOR_MESSAGE_PROCESSED: advance_resize_state_machine(mm, WMRZ_XRDP_CORE_RESIZE); break; case WMRZ_XRDP_CORE_RESIZE: @@ -1400,6 +1708,22 @@ process_display_control_monitor_layout_data(struct xrdp_wm *wm) return advance_error(error, mm); } sync_dynamic_monitor_data(wm, &(description->description)); + advance_resize_state_machine(mm, WMRZ_EGFX_INITIALIZE); + break; + case WMRZ_EGFX_INITIALIZE: + if (error == 0 && mm->egfx == NULL && mm->egfx_up == 0) + { + egfx_initialize(mm); + advance_resize_state_machine(mm, WMRZ_EGFX_INITALIZING); + } + else + { + advance_resize_state_machine(mm, WMRZ_EGFX_INITIALIZED); + } + break; + // Not processed here. Processed in xrdp_mm_egfx_caps_advertise + // case WMRZ_EGFX_INITALIZING: + case WMRZ_EGFX_INITIALIZED: advance_resize_state_machine(mm, WMRZ_ENCODER_CREATE); break; case WMRZ_ENCODER_CREATE: @@ -1450,7 +1774,7 @@ dynamic_monitor_process_queue(struct xrdp_mm *self) return 0; } - if (self->resize_data == NULL) + if (self->resize_data == NULL && self->resize_queue != NULL) { if (self->resize_queue->count <= 0) { @@ -1518,6 +1842,11 @@ dynamic_monitor_process_queue(struct xrdp_mm *self) " Resize data is not null."); } + if (self->resize_data == NULL) + { + return 0; + } + if (self->resize_data->state == WMRZ_COMPLETE) { LOG(LOG_LEVEL_INFO, "dynamic_monitor_process_queue: Clearing" @@ -1552,6 +1881,8 @@ dynamic_monitor_initialize(struct xrdp_mm *self) struct xrdp_drdynvc_procs d_procs; int flags; int error; + char buf[1024]; + int pid; LOG_DEVEL(LOG_LEVEL_TRACE, "dynamic_monitor_initialize:"); @@ -1567,9 +1898,19 @@ dynamic_monitor_initialize(struct xrdp_mm *self) &(self->dynamic_monitor_chanid)); if (error != 0) { - LOG(LOG_LEVEL_ERROR, "xrdp_mm_drdynvc_up: " - "libxrdp_drdynvc_open failed %d", error); + LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_mm_drdynvc_up: libxrdp_drdynvc_open failed %d", error); + return error; } + + // Initialize xrdp_mm specific variables. + self->resize_queue = list_create(); + self->resize_queue->auto_free = 1; + pid = g_getpid(); + /* setup wait objects for signaling */ + g_snprintf(buf, sizeof(buf), "xrdp_%8.8x_resize_ready", pid); + self->resize_ready = g_create_wait_obj(buf); + self->resize_data = NULL; + return error; } @@ -1583,6 +1924,12 @@ xrdp_mm_drdynvc_up(struct xrdp_mm *self) LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_drdynvc_up:"); + error = egfx_initialize(self); + if (error != 0) + { + return error; + } + enable_dynamic_resize = xrdp_mm_get_value(self, "enable_dynamic_resizing"); /* * User can disable dynamic resizing if necessary @@ -1593,10 +1940,16 @@ xrdp_mm_drdynvc_up(struct xrdp_mm *self) LOG(LOG_LEVEL_INFO, "User has disabled dynamic resizing."); return error; } + error = dynamic_monitor_initialize(self); + if (error != 0) + { + LOG(LOG_LEVEL_INFO, "Dynamic monitor initialize failed." + " Client likely does not support it."); + return error; + } ignore_marker = (struct display_control_monitor_layout_data *) g_malloc(sizeof(struct display_control_monitor_layout_data), 1); list_add_item(self->resize_queue, (tintptr)ignore_marker); - error = dynamic_monitor_initialize(self); return error; } @@ -1897,7 +2250,7 @@ xrdp_mm_trans_process_drdynvc_data(struct xrdp_mm *self, } in_uint32_le(s, chansrv_chan_id); in_uint32_le(s, data_bytes); - if ((!s_check_rem(s, data_bytes))) + if (!s_check_rem(s, data_bytes)) { return 1; } @@ -2795,7 +3148,10 @@ xrdp_mm_connect_sm(struct xrdp_mm *self) } } - +#define MIN_MS_BETWEEN_FRAMES 40 +/* can not change this to zero yet, g_obj_wait in os_calls.c treats + everything less then 1 to mean wait forever */ +#define MIN_MS_TO_WAIT_FOR_MORE_UPDATES 1 /*****************************************************************************/ int xrdp_mm_get_wait_objs(struct xrdp_mm *self, @@ -2842,6 +3198,24 @@ xrdp_mm_get_wait_objs(struct xrdp_mm *self, read_objs[(*rcount)++] = self->resize_ready; } + if (self->wm->screen_dirty_region != NULL) + { + if (xrdp_region_not_empty(self->wm->screen_dirty_region)) + { + int now = g_time3(); + int next_screen_draw_time = self->wm->last_screen_draw_time + + MIN_MS_BETWEEN_FRAMES; + int diff = next_screen_draw_time - now; + int ltimeout = *timeout; + diff = MAX(diff, MIN_MS_TO_WAIT_FOR_MORE_UPDATES); + diff = MIN(diff, MIN_MS_BETWEEN_FRAMES); + LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_get_wait_objs: not empty diff %d", diff); + if ((ltimeout < 0) || (ltimeout > diff)) + { + *timeout = diff; + } + } + } return rv; } @@ -2919,28 +3293,6 @@ xrdp_mm_check_chan(struct xrdp_mm *self) return 0; } -/*****************************************************************************/ -static int -xrdp_mm_update_module_frame_ack(struct xrdp_mm *self) -{ - int fif; - struct xrdp_encoder *encoder; - - encoder = self->encoder; - fif = encoder->frames_in_flight; - if (encoder->frame_id_client + fif > encoder->frame_id_server) - { - if (encoder->frame_id_server > encoder->frame_id_server_sent) - { - LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_update_module_ack: frame_id_server %d", - encoder->frame_id_server); - encoder->frame_id_server_sent = encoder->frame_id_server; - self->mod->mod_frame_ack(self->mod, 0, encoder->frame_id_server); - } - } - return 0; -} - /*****************************************************************************/ static int xrdp_mm_process_enc_done(struct xrdp_mm *self) @@ -2951,6 +3303,9 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) int y; int cx; int cy; + struct xrdp_egfx_rect rect; + + LOG(LOG_LEVEL_TRACE, "xrdp_mm_process_enc_done:"); LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_process_enc_done:"); @@ -2973,20 +3328,51 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) cy = enc_done->cy; if (enc_done->comp_bytes > 0) { - if (!enc_done->continuation) + LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_process_enc_done: x %d y %d cx %d cy %d " + "frame_id %d use_frame_acks %d", x, y, cx, cy, + enc_done->enc->frame_id, + self->wm->client_info->use_frame_acks); + if (enc_done->flags & 1) /* gfx h264 */ { - libxrdp_fastpath_send_frame_marker(self->wm->session, 0, - enc_done->enc->frame_id); + xrdp_egfx_send_frame_start(self->egfx, + enc_done->enc->frame_id, 0); + rect.x1 = x; + rect.y1 = y; + rect.x2 = x + cx; + rect.y2 = y + cy; + xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, +#if XRDP_AVC444 + XR_RDPGFX_CODECID_AVC444V2, +#else + XR_RDPGFX_CODECID_AVC420, +#endif + XR_PIXEL_FORMAT_XRGB_8888, + &rect, + enc_done->comp_pad_data + enc_done->pad_bytes, + enc_done->comp_bytes); + xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); } - libxrdp_fastpath_send_surface(self->wm->session, - enc_done->comp_pad_data, - enc_done->pad_bytes, - enc_done->comp_bytes, - x, y, x + cx, y + cy, - 32, self->encoder->codec_id, - cx, cy); - if (enc_done->last) + else if (enc_done->flags & 2) /* gfx progressive rfx */ { + xrdp_egfx_send_frame_start(self->egfx, + enc_done->enc->frame_id, 0); + xrdp_egfx_send_wire_to_surface2(self->egfx, self->egfx->surface_id, 9, 1, + XR_PIXEL_FORMAT_XRGB_8888, + enc_done->comp_pad_data + enc_done->pad_bytes, + enc_done->comp_bytes); + xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); + } + else + { + libxrdp_fastpath_send_frame_marker(self->wm->session, 0, + enc_done->enc->frame_id); + libxrdp_fastpath_send_surface(self->wm->session, + enc_done->comp_pad_data, + enc_done->pad_bytes, + enc_done->comp_bytes, + x, y, x + cx, y + cy, + 32, self->encoder->codec_id, + cx, cy); libxrdp_fastpath_send_frame_marker(self->wm->session, 1, enc_done->enc->frame_id); } @@ -2996,16 +3382,35 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) { enc = enc_done->enc; LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: last set"); - if (self->wm->client_info->use_frame_acks == 0) + if (enc_done->flags & 3) /* gfx */ { - self->mod->mod_frame_ack(self->mod, - enc->flags, - enc->frame_id); + if (self->encoder->gfx_ack_off) + { + /* gfx and client turned off client frame acks */ + self->mod->mod_frame_ack(self->mod, + enc->flags, + enc->frame_id); + } + else + { + self->encoder->frame_id_server = enc->frame_id; + xrdp_mm_update_module_frame_ack(self); + } } else { - self->encoder->frame_id_server = enc->frame_id; - xrdp_mm_update_module_frame_ack(self); + if (self->wm->client_info->use_frame_acks == 0) + { + /* surface commmand and client does not do frame acks */ + self->mod->mod_frame_ack(self->mod, + enc->flags, + enc->frame_id); + } + else + { + self->encoder->frame_id_server = enc_done->enc->frame_id; + xrdp_mm_update_module_frame_ack(self); + } } g_free(enc->drects); g_free(enc->crects); @@ -3126,7 +3531,6 @@ xrdp_mm_check_wait_objs(struct xrdp_mm *self) } } } - return rv; } @@ -3162,6 +3566,20 @@ xrdp_mm_frame_ack(struct xrdp_mm *self, int frame_id) return 0; } +int +xrdp_mm_can_resize(struct xrdp_mm *self) +{ + if (self == 0) + { + return 0; + } + if (self->gfx_delay_autologin == 1) + { + return 0; + } + return xrdp_wm_can_resize(self->wm); +} + #if 0 /*****************************************************************************/ struct xrdp_painter * @@ -3406,7 +3824,7 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects, wm = (struct xrdp_wm *)(mod->wm); mm = wm->mm; - LOG_DEVEL(LOG_LEVEL_DEBUG, "server_paint_rects: %p", mm->encoder); + LOG(LOG_LEVEL_TRACE, "server_paint_rects: %p", mm->encoder); if (mm->encoder != 0) { @@ -3477,7 +3895,12 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects, return 0; } - LOG(LOG_LEVEL_TRACE, "server_paint_rects:"); + if (wm->client_info->gfx) + { + LOG(LOG_LEVEL_DEBUG, "server_paint_rects: gfx session and no encoder"); + mm->mod->mod_frame_ack(mm->mod, flags, frame_id); + return 0; + } p = (struct xrdp_painter *)(mod->painter); if (p == 0) @@ -3789,15 +4212,22 @@ server_draw_text(struct xrdp_mod *mod, int font, } /*****************************************************************************/ - /* Note : if this is called on a multimon setup, the client is resized * to a single monitor */ int server_reset(struct xrdp_mod *mod, int width, int height, int bpp) { struct xrdp_wm *wm; + struct xrdp_mm *mm; + + LOG(LOG_LEVEL_INFO, "server_reset:"); wm = (struct xrdp_wm *)(mod->wm); + if (wm == 0) + { + return 1; + } + mm = wm->mm; if (wm->client_info == 0) { @@ -3810,6 +4240,42 @@ server_reset(struct xrdp_mod *mod, int width, int height, int bpp) return 0; } + // bpp of zero is impossible. + // This is a signal from xup that + // It is finished resizing. + if (bpp == 0) + { + if (mm == 0) + { + return 1; + } + if (!xrdp_wm_can_resize(wm)) + { + return 1; + } + if (mm->resize_data == NULL) + { + mm->mod->mod_server_monitor_full_invalidate(mm->mod, width, height); + return 0; + } + if (mm->resize_data != NULL + && mm->resize_data->state + == WMRZ_SERVER_MONITOR_MESSAGE_PROCESSING) + { + LOG(LOG_LEVEL_INFO, + "server_reset: Advancing server monitor resized."); + advance_resize_state_machine( + mm, WMRZ_SERVER_MONITOR_MESSAGE_PROCESSED); + } + else if (mm->resize_data != NULL + && mm->resize_data->description.session_height == 0 + && mm->resize_data->description.session_width == 0) + { + mm->mod->mod_server_monitor_full_invalidate(mm->mod, width, height); + } + return 0; + } + /* if same (and only one monitor on client) don't need to do anything */ if (wm->client_info->display_sizes.session_width == (uint32_t)width && wm->client_info->display_sizes.session_height == (uint32_t)height && @@ -3820,6 +4286,8 @@ server_reset(struct xrdp_mod *mod, int width, int height, int bpp) return 0; } + LOG(LOG_LEVEL_INFO, "server_reset: Actually resetting the server."); + /* reset lib, client_info gets updated in libxrdp_reset */ if (libxrdp_reset(wm->session, width, height, bpp) != 0) { diff --git a/xrdp/xrdp_painter.c b/xrdp/xrdp_painter.c index fce0d3780d..91db83416b 100644 --- a/xrdp/xrdp_painter.c +++ b/xrdp/xrdp_painter.c @@ -133,7 +133,6 @@ xrdp_painter_send_dirty(struct xrdp_painter *self) } LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_painter_send_dirty:" " x %d y %d cx %d cy %d", - rect.left, rect.top, cx, cy); libxrdp_send_bitmap(self->session, cx, cy, bpp, ldata, rect.left, rect.top, cx, cy); g_free(ldata); @@ -163,7 +162,6 @@ xrdp_painter_create(struct xrdp_wm *wm, struct xrdp_session *session) self->session = session; self->rop = 0xcc; /* copy will use 0xcc */ self->clip_children = 1; - if (self->session->client_info->no_orders_supported || self->session->client_info->gfx) { diff --git a/xrdp/xrdp_types.h b/xrdp/xrdp_types.h index d0bce05706..0c7c535d70 100644 --- a/xrdp/xrdp_types.h +++ b/xrdp/xrdp_types.h @@ -28,6 +28,7 @@ #include "xrdp_constants.h" #include "fifo.h" #include "guid.h" +#include "xrdp_client_info.h" #define MAX_NR_CHANNELS 16 #define MAX_CHANNEL_NAME 16 @@ -343,7 +344,9 @@ enum display_resize_state WMRZ_EGFX_CONN_CLOSED, WRMZ_EGFX_DELETE, WMRZ_SERVER_MONITOR_RESIZE, - WMRZ_SERVER_VERSION_MESSAGE, + WMRZ_SERVER_VERSION_MESSAGE_START, + WMRZ_SERVER_MONITOR_MESSAGE_PROCESSING, + WMRZ_SERVER_MONITOR_MESSAGE_PROCESSED, WMRZ_XRDP_CORE_RESIZE, WMRZ_EGFX_INITIALIZE, WMRZ_EGFX_INITALIZING, @@ -362,7 +365,12 @@ enum display_resize_state (status) == WMRZ_EGFX_CONN_CLOSED ? "WMRZ_EGFX_CONN_CLOSED" : \ (status) == WRMZ_EGFX_DELETE ? "WMRZ_EGFX_DELETE" : \ (status) == WMRZ_SERVER_MONITOR_RESIZE ? "WMRZ_SERVER_MONITOR_RESIZE" : \ - (status) == WMRZ_SERVER_VERSION_MESSAGE ? "WMRZ_SERVER_VERSION_MESSAGE" : \ + (status) == WMRZ_SERVER_VERSION_MESSAGE_START ? \ + "WMRZ_SERVER_VERSION_MESSAGE_START" : \ + (status) == WMRZ_SERVER_MONITOR_MESSAGE_PROCESSING ? \ + "WMRZ_SERVER_MONITOR_MESSAGE_PROCESSING" : \ + (status) == WMRZ_SERVER_MONITOR_MESSAGE_PROCESSED ? \ + "WMRZ_SERVER_MONITOR_MESSAGE_PROCESSED" : \ (status) == WMRZ_XRDP_CORE_RESIZE ? "WMRZ_XRDP_CORE_RESIZE" : \ (status) == WMRZ_EGFX_INITIALIZE ? "WMRZ_EGFX_INITIALIZE" : \ (status) == WMRZ_EGFX_INITALIZING ? "WMRZ_EGFX_INITALIZING" : \ diff --git a/xrdp/xrdp_wm.c b/xrdp/xrdp_wm.c index 8d0a48c317..4e59fd1b82 100644 --- a/xrdp/xrdp_wm.c +++ b/xrdp/xrdp_wm.c @@ -68,7 +68,6 @@ xrdp_wm_create(struct xrdp_process *owner, /* to store configuration from xrdp.ini */ self->xrdp_config = g_new0(struct xrdp_config, 1); - return self; } @@ -80,7 +79,6 @@ xrdp_wm_delete(struct xrdp_wm *self) { return; } - xrdp_region_delete(self->screen_dirty_region); xrdp_mm_delete(self->mm); xrdp_cache_delete(self->cache); diff --git a/xup/xup.c b/xup/xup.c index 4215d0bcf7..987c50e121 100644 --- a/xup/xup.c +++ b/xup/xup.c @@ -1174,47 +1174,33 @@ process_server_paint_rect_shmem_ex(struct mod *amod, struct stream *s) in_uint16_le(s, height); bmpdata = 0; - if (flags == 0) /* screen */ + if (amod->screen_shmem_id_mapped == 0) { - /* Do we need to map (or remap) the memory - * area shared with the X server ? */ - if (amod->screen_shmem_id_mapped == 0 || - amod->screen_shmem_id != shmem_id) + amod->screen_shmem_id = shmem_id; + amod->screen_shmem_pixels = (char *) g_shmat(amod->screen_shmem_id); + if (amod->screen_shmem_pixels == (void *) -1) { - if (amod->screen_shmem_id_mapped != 0) - { - g_shmdt(amod->screen_shmem_pixels); - } - amod->screen_shmem_pixels = (char *) g_shmat(shmem_id); - if (amod->screen_shmem_pixels == (void *) -1) - { - /* failed */ - if (amod->screen_shmem_id_mapped == 0) - { - LOG(LOG_LEVEL_ERROR, - "Can't attach to shared memory id %d [%s]", - shmem_id, g_get_strerror()); - } - else - { - LOG(LOG_LEVEL_ERROR, - "Can't attach to shared memory id %d from id %d [%s]", - shmem_id, amod->screen_shmem_id, g_get_strerror()); - } - amod->screen_shmem_id = 0; - amod->screen_shmem_pixels = 0; - amod->screen_shmem_id_mapped = 0; - } - else - { - amod->screen_shmem_id = shmem_id; - amod->screen_shmem_id_mapped = 1; - } + /* failed */ + amod->screen_shmem_id = 0; + amod->screen_shmem_pixels = 0; + amod->screen_shmem_id_mapped = 0; } - - if (amod->screen_shmem_pixels != 0) + else { - bmpdata = amod->screen_shmem_pixels + shmem_offset; + amod->screen_shmem_id_mapped = 1; + } + } + else if (amod->screen_shmem_id != shmem_id) + { + amod->screen_shmem_id = shmem_id; + g_shmdt(amod->screen_shmem_pixels); + amod->screen_shmem_pixels = (char *) g_shmat(amod->screen_shmem_id); + if (amod->screen_shmem_pixels == (void *) -1) + { + /* failed */ + amod->screen_shmem_id = 0; + amod->screen_shmem_pixels = 0; + amod->screen_shmem_id_mapped = 0; } } else @@ -1225,7 +1211,10 @@ process_server_paint_rect_shmem_ex(struct mod *amod, struct stream *s) flags, frame_id, shmem_id, shmem_offset, width, height); } - + if (amod->screen_shmem_pixels != 0) + { + bmpdata = amod->screen_shmem_pixels + shmem_offset; + } if (bmpdata != 0) { rv = amod->server_paint_rects(amod, num_drects, ldrects, @@ -1238,7 +1227,7 @@ process_server_paint_rect_shmem_ex(struct mod *amod, struct stream *s) rv = 1; } - //LOG_DEVEL(LOG_LEVEL_TRACE, "frame_id %d", frame_id); + //g_writeln("frame_id %d", frame_id); //send_paint_rect_ex_ack(amod, flags, frame_id); g_free(lcrects); @@ -1534,7 +1523,7 @@ lib_mod_process_orders(struct mod *mod, int type, struct stream *s) { int rv; - LOG_DEVEL(LOG_LEVEL_DEBUG, "lib_mod_process_orders: type %d", type); + LOG_DEVEL(LOG_LEVEL_INFO, "lib_mod_process_orders: type %d", type); rv = 0; switch (type) { @@ -1679,74 +1668,116 @@ lib_mod_process_message(struct mod *mod, struct stream *s) int type; char *phold; - LOG_DEVEL(LOG_LEVEL_TRACE, "lib_mod_process_message:"); + int width; + int height; + int magic; + int con_id; + int mon_id; + + LOG_DEVEL(LOG_LEVEL_INFO, "lib_mod_process_message:"); + in_uint16_le(s, type); + in_uint16_le(s, num_orders); + in_uint32_le(s, len); + LOG_DEVEL(LOG_LEVEL_INFO, "lib_mod_process_message: type %d", type); + rv = 0; - if (rv == 0) + if (type == 1) /* original order list */ { - in_uint16_le(s, type); - in_uint16_le(s, num_orders); - in_uint32_le(s, len); - LOG_DEVEL(LOG_LEVEL_TRACE, "lib_mod_process_message: type %d", type); - - if (type == 1) /* original order list */ + for (index = 0; index < num_orders; index++) { - for (index = 0; index < num_orders; index++) + in_uint16_le(s, type); + rv = lib_mod_process_orders(mod, type, s); + + if (rv != 0) { - in_uint16_le(s, type); - rv = lib_mod_process_orders(mod, type, s); + break; + } + } + } + else if (type == 2) /* caps */ + { + LOG_DEVEL(LOG_LEVEL_TRACE, + "lib_mod_process_message: type 2 len %d", len); + for (index = 0; index < num_orders; index++) + { + phold = s->p; + in_uint16_le(s, type); + in_uint16_le(s, len); - if (rv != 0) - { + switch (type) + { + default: + LOG_DEVEL(LOG_LEVEL_TRACE, + "lib_mod_process_message: unknown" + " cap type %d len %d", + type, len); break; - } } + s->p = phold + len; } - else if (type == 2) /* caps */ + lib_send_client_info(mod); + } + else if (type == 3) /* order list with len after type */ + { + LOG_DEVEL(LOG_LEVEL_INFO, "lib_mod_process_message: type 3 len %d", len); + for (index = 0; index < num_orders; index++) { - LOG_DEVEL(LOG_LEVEL_TRACE, - "lib_mod_process_message: type 2 len %d", len); - for (index = 0; index < num_orders; index++) - { - phold = s->p; - in_uint16_le(s, type); - in_uint16_le(s, len); + phold = s->p; + in_uint16_le(s, type); + in_uint16_le(s, len); + rv = lib_mod_process_orders(mod, type, s); - switch (type) - { - default: - LOG_DEVEL(LOG_LEVEL_TRACE, - "lib_mod_process_message: unknown" - " cap type %d len %d", - type, len); - break; - } - - s->p = phold + len; + if (rv != 0) + { + break; } - lib_send_client_info(mod); + s->p = phold + len; } - else if (type == 3) /* order list with len after type */ + } + else if (type == 100) // xorgxrdp_helper commands. + { + LOG_DEVEL(LOG_LEVEL_INFO, + "lib_mod_process_message: type 100 len %d", len); + for (index = 0; index < num_orders; index++) { - for (index = 0; index < num_orders; index++) + phold = s->p; + in_uint16_le(s, type); + in_uint16_le(s, len); + switch (type) { - phold = s->p; - in_uint16_le(s, type); - in_uint16_le(s, len); - rv = lib_mod_process_orders(mod, type, s); - - if (rv != 0) - { + case 1: // xorgxrdp_helper_x11_delete_all_pixmaps + // No-op for now. + break; + case 2: // xorgxrdp_helper_x11_create_pixmap + in_uint16_le(s, width); + in_uint16_le(s, height); + in_uint32_le(s, magic); + in_uint32_le(s, con_id); + in_uint32_le(s, mon_id); + + LOG(LOG_LEVEL_INFO, "Received" + " xorgxrdp_helper_x11_create_pixmap command." + " width: %d, height: %d, magic: %d, con_id %d," + " mon_id %d", + width, height, magic, con_id, mon_id); + rv = mod->server_reset(mod, width, height, 0); + break; + case 3: + in_uint16_le(s, width); + in_uint16_le(s, height); + LOG(LOG_LEVEL_INFO, "Received memory_allocation_complete" + " command. width: %d, height: %d", + width, height); + rv = mod->server_reset(mod, width, height, 0); break; - } - - s->p = phold + len; } + s->p = phold + len; } - else - { - LOG_DEVEL(LOG_LEVEL_TRACE, "unknown type %d", type); - } + } + else + { + LOG_DEVEL(LOG_LEVEL_TRACE, "unknown type %d", type); } return rv; From 76eca91d9d0752e3a8a68d6e61d6c4617f16cd10 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 21 Oct 2023 01:33:06 -0400 Subject: [PATCH 02/39] Updating shaders --- xorgxrdp_helper/xorgxrdp_helper_shaders.c | 188 ++++++++++------------ 1 file changed, 84 insertions(+), 104 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper_shaders.c b/xorgxrdp_helper/xorgxrdp_helper_shaders.c index 0c60adeef5..167e0ed958 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_shaders.c +++ b/xorgxrdp_helper/xorgxrdp_helper_shaders.c @@ -39,8 +39,6 @@ void main(void)\n\ gl_FragColor = texture(tex, gl_FragCoord.xy / tex_size);\n\ }\n"; -/* -Original version. static const GLchar g_fs_rgb_to_yuv420[] = "\ uniform sampler2D tex;\n\ uniform vec2 tex_size;\n\ @@ -88,50 +86,6 @@ void main(void)\n\ }\n\ }\n\ }\n"; -*/ - -// Optimized version. -static const GLchar g_fs_rgb_to_yuv420[] = "\ -#version 460 core\n\ -uniform sampler2D tex;\n\ -uniform vec2 tex_size;\n\ -uniform vec4 ymath;\n\ -uniform vec4 umath;\n\ -uniform vec4 vmath;\n\ -out vec4 FragColor;\n\ -void main()\n\ -{\n\ - vec4 pix;\n\ - float x = gl_FragCoord.x;\n\ - float y = gl_FragCoord.y;\n\ - if (y < tex_size.y)\n\ - {\n\ - pix = texture(tex, vec2(x, y) / tex_size);\n\ - pix.a = 1.0;\n\ - pix.r = clamp(dot(ymath, pix), 0.0, 1.0);\n\ - }\n\ - else\n\ - {\n\ - y = floor(y - tex_size.y) * 2.0 + 0.5;\n\ - float xOffset = mod(x, 2.0);\n\ - float antiAliasMod = xOffset == 0 ? 1.0 : -1.0;\n\ - pix = texture(tex, vec2(x - xOffset, y) / tex_size);\n\ - pix += texture(tex, vec2(x - xOffset + antiAliasMod, y) / tex_size);\n\ - pix += texture(tex, vec2(x - xOffset, y + 1.0) / tex_size);\n\ - pix += texture(tex, vec2(x - xOffset + antiAliasMod, y + 1.0) / tex_size);\n\ - pix /= 4.0;\n\ - pix.a = 1.0;\n\ - if (xOffset < 1.0)\n\ - {\n\ - pix.r = clamp(dot(umath, pix), 0.0, 1.0);\n\ - }\n\ - else\n\ - {\n\ - pix.r = clamp(dot(vmath, pix), 0.0, 1.0);\n\ - }\n\ - }\n\ - FragColor = vec4(pix.r, 0.0, 0.0, 1.0);\n\ -}\n"; static const GLchar g_fs_rgb_to_yuv422[] = "\ uniform sampler2D tex;\n\ @@ -215,12 +169,16 @@ MAIN VIEW - NV12 Keeping this now because this is a dev prototype, but we might want to remove this in the future. */ + +/* static const GLchar g_fs_rgb_to_yuv420_mv[] = "\ +#version 460 core\n\ uniform sampler2D tex;\n\ uniform vec2 tex_size;\n\ uniform vec4 ymath;\n\ uniform vec4 umath;\n\ uniform vec4 vmath;\n\ +out vec4 FragColor;\n\ void main(void)\n\ {\n\ vec4 pix;\n\ @@ -230,38 +188,81 @@ void main(void)\n\ y = gl_FragCoord.y;\n\ if (y < tex_size.y)\n\ {\n\ - pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix = texture(tex, vec2(x, y) / tex_size);\n\ pix.a = 1.0;\n\ pix = vec4(clamp(dot(ymath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - gl_FragColor = pix;\n\ + FragColor = pix;\n\ }\n\ else\n\ {\n\ y = floor(y - tex_size.y) * 2.0 + 0.5;\n\ if (mod(x, 2.0) < 1.0)\n\ {\n\ - pix = texture2D(tex, vec2(x, y) / tex_size);\n\ - pix += texture2D(tex, vec2(x + 1.0, y) / tex_size);\n\ - pix += texture2D(tex, vec2(x, y + 1.0) / tex_size);\n\ - pix += texture2D(tex, vec2(x + 1.0, y + 1.0) / tex_size);\n\ - pix /= 4.0;\n\ + pix = texture(tex, vec2(x, y) / tex_size);\n\ + pix += texture(tex, vec2(x + 1.0, y) / tex_size);\n\ + pix += texture(tex, vec2(x, y + 1.0) / tex_size);\n\ + pix += texture(tex, vec2(x + 1.0, y + 1.0) / tex_size);\n\ + pix *= 0.25;\n\ pix.a = 1.0;\n\ pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - gl_FragColor = pix;\n\ + FragColor = pix;\n\ }\n\ else\n\ {\n\ - pix = texture2D(tex, vec2(x, y) / tex_size);\n\ - pix += texture2D(tex, vec2(x - 1.0, y) / tex_size);\n\ - pix += texture2D(tex, vec2(x, y + 1.0) / tex_size);\n\ - pix += texture2D(tex, vec2(x - 1.0, y + 1.0) / tex_size);\n\ - pix /= 4.0;\n\ + pix = texture(tex, vec2(x, y) / tex_size);\n\ + pix += texture(tex, vec2(x + 1.0, y) / tex_size);\n\ + pix += texture(tex, vec2(x, y + 1.0) / tex_size);\n\ + pix += texture(tex, vec2(x + 1.0, y + 1.0) / tex_size);\n\ + pix *= 0.25;\n\ pix.a = 1.0;\n\ pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - gl_FragColor = pix;\n\ + FragColor = pix;\n\ }\n\ }\n\ }\n"; +*/ + +static const GLchar g_fs_rgb_to_yuv420_mv[] = "\ +#version 460 core\n\ +uniform sampler2D tex;\n\ +uniform vec2 tex_size;\n\ +uniform vec4 ymath;\n\ +uniform vec4 umath;\n\ +uniform vec4 vmath;\n\ +out vec4 FragColor;\n\ +void main()\n\ +{\n\ + vec4 pix;\n\ + float x = gl_FragCoord.x;\n\ + float y = gl_FragCoord.y;\n\ + if (y < tex_size.y)\n\ + {\n\ + pix = texture(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix.r = clamp(dot(ymath, pix), 0.0, 1.0);\n\ + }\n\ + else\n\ + {\n\ + y = floor(y - tex_size.y) * 2.0 + 0.5;\n\ + float xOffset = mod(x, 2.0);\n\ + float antiAliasMod = xOffset == 0 ? 1.0 : -1.0;\n\ + pix = texture(tex, vec2(x - xOffset, y) / tex_size);\n\ + pix += texture(tex, vec2(x - xOffset + antiAliasMod, y) / tex_size);\n\ + pix += texture(tex, vec2(x - xOffset, y + 1.0) / tex_size);\n\ + pix += texture(tex, vec2(x - xOffset + antiAliasMod, y + 1.0) / tex_size);\n\ + pix /= 4.0;\n\ + pix.a = 1.0;\n\ + if (xOffset < 1.0)\n\ + {\n\ + pix.r = clamp(dot(umath, pix), 0.0, 1.0);\n\ + }\n\ + else\n\ + {\n\ + pix.r = clamp(dot(vmath, pix), 0.0, 1.0);\n\ + }\n\ + }\n\ + FragColor = vec4(pix.r, 0.0, 0.0, 1.0);\n\ +}\n"; /* AUXILIARY VIEW - NV12 @@ -339,6 +340,7 @@ void main(void)\n\ }\n\ }\n"; + /* AUXILIARY VIEW V2 - NV12 @@ -354,57 +356,35 @@ AUXILIARY VIEW V2 - NV12 ... 0F 2F 4F 6F 8F AF CF EF 0F 2F 4F 6F 8F AF CF EF */ + static const GLchar g_fs_rgb_to_yuv420_av_v2[] = "\ +#version 460 core\n\ uniform sampler2D tex;\n\ uniform vec2 tex_size;\n\ uniform vec4 umath;\n\ uniform vec4 vmath;\n\ +out vec4 FragColor;\n\ void main(void)\n\ {\n\ - vec4 pix;\n\ - float x;\n\ - float y;\n\ - float x1;\n\ - x = gl_FragCoord.x;\n\ - y = gl_FragCoord.y;\n\ - x1 = tex_size.x / 2.0;\n\ - if (y < tex_size.y)\n\ - {\n\ - if (x < x1)\n\ - {\n\ - x = floor(x) * 2.0 + 1.5;\n\ - pix = texture2D(tex, vec2(x, y) / tex_size);\n\ - pix.a = 1.0;\n\ - pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - gl_FragColor = pix;\n\ - }\n\ - else\n\ - {\n\ - x = floor(x - x1) * 2.0 + 1.5;\n\ - pix = texture2D(tex, vec2(x, y) / tex_size);\n\ - pix.a = 1.0;\n\ - pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - gl_FragColor = pix;\n\ - }\n\ - }\n\ - else\n\ + vec4 pix[4];\n\ + vec2 coords[4];\n\ + float x = gl_FragCoord.x;\n\ + float y = gl_FragCoord.y;\n\ + float x1 = tex_size.x / 2.0;\n\ + float y1 = tex_size.y / 2.0;\n\ + float adjustedY = floor(y - tex_size.y) * 2.0 + 1.5;\n\ + coords[0] = vec2(floor(x) * 2.0 + 1.5, y);\n\ + coords[1] = vec2(floor(x - x1) * 2.0 + 1.5, y);\n\ + coords[2] = vec2(floor(x) * 2.0 + 0.5, adjustedY);\n\ + coords[3] = vec2(floor(x - x1) * 2.0 + 0.5, adjustedY);\n\ + for (int i = 0; i < 4; i++)\n\ {\n\ - y = floor(y - tex_size.y) * 2.0 + 1.5;\n\ - if (x < x1)\n\ - {\n\ - x = floor(x) * 2.0 + 0.5;\n\ - pix = texture2D(tex, vec2(x, y) / tex_size);\n\ - pix.a = 1.0;\n\ - pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - gl_FragColor = pix;\n\ - }\n\ - else\n\ - {\n\ - x = floor(x - x1) * 2.0 + 0.5;\n\ - pix = texture2D(tex, vec2(x, y) / tex_size);\n\ - pix.a = 1.0;\n\ - pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - gl_FragColor = pix;\n\ - }\n\ + pix[i] = texture(tex, coords[i] / tex_size);\n\ + pix[i].a = 1.0;\n\ }\n\ + pix[0] = vec4(clamp(dot(umath, pix[0]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + pix[1] = vec4(clamp(dot(vmath, pix[1]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + pix[2] = vec4(clamp(dot(umath, pix[2]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + pix[3] = vec4(clamp(dot(vmath, pix[3]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + FragColor = pix[int(y < tex_size.y ? (x < x1 ? 0 : 1) : (x < x1 ? 2 : 3))];\n\ }\n"; From 4dee577b51a8cd61689d0b1416ec175c05faddd3 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 21 Oct 2023 01:36:44 -0400 Subject: [PATCH 03/39] Updating nvenc encoder --- xorgxrdp_helper/xorgxrdp_helper_nvenc.c | 124 ++++++++++++++++-------- 1 file changed, 85 insertions(+), 39 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c index f5fdb450e3..c66dad2d4a 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c @@ -142,8 +142,8 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, g_memset(&createEncodeParams, 0, sizeof(createEncodeParams)); createEncodeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; createEncodeParams.encodeGUID = NV_ENC_CODEC_H264_GUID; - createEncodeParams.presetGUID = NV_ENC_PRESET_P4_GUID; - createEncodeParams.tuningInfo = NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY; + createEncodeParams.presetGUID = NV_ENC_PRESET_P7_GUID; + createEncodeParams.tuningInfo = NV_ENC_TUNING_INFO_LOW_LATENCY; createEncodeParams.encodeWidth = width; createEncodeParams.encodeHeight = height; createEncodeParams.darWidth = width; @@ -155,10 +155,10 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, preset_config.presetCfg.version = NV_ENC_CONFIG_VER; nv_error = g_enc_funcs.nvEncGetEncodePresetConfigEx( - lei->enc, createEncodeParams.encodeGUID, - createEncodeParams.presetGUID, - createEncodeParams.tuningInfo, - &preset_config); + lei->enc, createEncodeParams.encodeGUID, + createEncodeParams.presetGUID, + createEncodeParams.tuningInfo, + &preset_config); LOG(LOG_LEVEL_INFO, "nvEncGetEncodePresetConfig rv %d", nv_error); if (nv_error != NV_ENC_SUCCESS) @@ -167,9 +167,12 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, } g_memset(&encCfg, 0, sizeof(encCfg)); + encCfg.version = NV_ENC_CONFIG_VER; + memcpy(&encCfg, &preset_config.presetCfg, sizeof(NV_ENC_CONFIG)); encCfg.profileGUID = NV_ENC_H264_PROFILE_MAIN_GUID; - encCfg.gopLength = NVENC_INFINITE_GOPLENGTH; + //encCfg.gopLength = NVENC_INFINITE_GOPLENGTH; + encCfg.gopLength = 10; encCfg.frameIntervalP = 1; /* 1 + B_Frame_Count */ encCfg.frameFieldMode = NV_ENC_PARAMS_FRAME_FIELD_MODE_FRAME; encCfg.mvPrecision = NV_ENC_MV_PRECISION_QUARTER_PEL; @@ -236,14 +239,30 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, } encCfg.rcParams.cbQPIndexOffset = -1; encCfg.encodeCodecConfig.h264Config.chromaFormatIDC = 1; - encCfg.encodeCodecConfig.h264Config.idrPeriod = NVENC_INFINITE_GOPLENGTH; + encCfg.encodeCodecConfig.h264Config.idrPeriod = 2; encCfg.encodeCodecConfig.h264Config.repeatSPSPPS = 1; encCfg.encodeCodecConfig.h264Config.disableSPSPPS = 0; + //encCfg.encodeCodecConfig.h264Config.maxNumRefFrames = 1; + encCfg.encodeCodecConfig.h264Config.sliceMode = 0; + encCfg.encodeCodecConfig.h264Config.sliceModeData = 0; encCfg.encodeCodecConfig.h264Config.outputAUD = 1; + encCfg.encodeCodecConfig.h264Config.outputBufferingPeriodSEI = 1; + encCfg.encodeCodecConfig.h264Config.outputPictureTimingSEI = 1; + encCfg.encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_AUTOSELECT; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = 1; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFormat = NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.bitstreamRestrictionFlag = 1; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourDescriptionPresentFlag = 1; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourPrimaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.transferCharacteristics = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; createEncodeParams.encodeConfig = &encCfg; + nv_error = g_enc_funcs.nvEncInitializeEncoder(lei->enc, &createEncodeParams); + LOG(LOG_LEVEL_INFO, "nvEncInitializeEncoder rv %d", nv_error); if (nv_error != NV_ENC_SUCCESS) { @@ -339,58 +358,85 @@ xorgxrdp_helper_nvenc_encode(struct enc_info *ei, int tex, picParams.inputWidth = ei->width; picParams.inputHeight = ei->height; picParams.outputBitstream = ei->bitstreamBuffer; - picParams.inputTimeStamp = ei->frameCount; + picParams.inputTimeStamp = g_time3(); picParams.pictureStruct = NV_ENC_PIC_STRUCT_FRAME; if (xrdp_invalidate > 0 || ei->frameCount == 0) { - picParams.encodePicFlags |= NV_ENC_PIC_FLAG_FORCEIDR; - picParams.encodePicFlags |= NV_ENC_PIC_FLAG_OUTPUT_SPSPPS; - picParams.encodePicFlags |= NV_ENC_PIC_FLAG_FORCEINTRA; + picParams.encodePicFlags = NV_ENC_PIC_FLAG_OUTPUT_SPSPPS | NV_ENC_PIC_FLAG_FORCEIDR | NV_ENC_PIC_FLAG_FORCEINTRA; + picParams.pictureType = NV_ENC_PIC_TYPE_IDR; LOG(LOG_LEVEL_INFO, "Forcing NVENC H264 IDR SPSPPS for frame id: %d," "invalidate is: %d", ei->frameCount, xrdp_invalidate); xrdp_invalidate = MAX(0, xrdp_invalidate - 1); } - nv_error = g_enc_funcs.nvEncEncodePicture(ei->enc, &picParams); - rv = ENCODER_ERROR; - if (nv_error != NV_ENC_SUCCESS) + else { - LOG(LOG_LEVEL_ERROR, "error nvEncEncodePicture %d", nv_error); - return rv; + picParams.pictureType = NV_ENC_PIC_TYPE_NONREF_P; + picParams.encodePicFlags = 0; } - g_memset(&lockBitstream, 0, sizeof(lockBitstream)); - lockBitstream.version = NV_ENC_LOCK_BITSTREAM_VER; - lockBitstream.outputBitstream = ei->bitstreamBuffer; - lockBitstream.doNotWait = 0; - nv_error = g_enc_funcs.nvEncLockBitstream(ei->enc, &lockBitstream); + nv_error = g_enc_funcs.nvEncEncodePicture(ei->enc, &picParams); + rv = ENCODER_ERROR; if (nv_error == NV_ENC_SUCCESS) { - if (*cdata_bytes >= ((int) (lockBitstream.bitstreamSizeInBytes))) + g_memset(&lockBitstream, 0, sizeof(lockBitstream)); + lockBitstream.version = NV_ENC_LOCK_BITSTREAM_VER; + lockBitstream.outputBitstream = ei->bitstreamBuffer; + lockBitstream.doNotWait = 0; + nv_error = g_enc_funcs.nvEncLockBitstream(ei->enc, &lockBitstream); + if (nv_error == NV_ENC_SUCCESS) { - g_memcpy(cdata, lockBitstream.bitstreamBufferPtr, - lockBitstream.bitstreamSizeInBytes); - *cdata_bytes = lockBitstream.bitstreamSizeInBytes; - rv = INCREMENTAL_FRAME_ENCODED; + if (*cdata_bytes >= ((int) (lockBitstream.bitstreamSizeInBytes))) + { + /* for (int i = 0; i < lockBitstream.bitstreamSizeInBytes; ++i) + { + unsigned char* payload = (unsigned char*)lockBitstream.bitstreamBufferPtr + i; + bool b_long_startcode = payload[0] == 0 && payload[1] == 0 + && payload[2] == 0 && payload[3] == 1; + bool b_short_startcode = payload[0] == 0 && payload[1] == 0 + && payload[2] == 1; + int nalUnitType; + // 4-byte start code + if (b_long_startcode) + { + nalUnitType = (payload[4] & 0x1F); + LOG(LOG_LEVEL_INFO, "Frame: %d: Found long start code at index %d. Type is %d", ei->frameCount, i, nalUnitType); + i += 3; + } + // assume 3-byte start code + else if (b_short_startcode) + { + nalUnitType = (payload[3] & 0x1F); + LOG(LOG_LEVEL_INFO, "Frame: %d: Found short start code at index %d. Type is %d", ei->frameCount, i, nalUnitType); + } + } */ + g_memcpy(cdata, lockBitstream.bitstreamBufferPtr, + lockBitstream.bitstreamSizeInBytes); + *cdata_bytes = lockBitstream.bitstreamSizeInBytes; + rv = INCREMENTAL_FRAME_ENCODED; + } + else + { + LOG(LOG_LEVEL_INFO, "error not enough room %d %d", + *cdata_bytes, + (int) (lockBitstream.bitstreamSizeInBytes)); + } + nv_error = g_enc_funcs.nvEncUnlockBitstream(ei->enc, + lockBitstream.outputBitstream); + if (nv_error != NV_ENC_SUCCESS) + { + LOG(LOG_LEVEL_INFO, "unlocking failed"); + } } else { - LOG(LOG_LEVEL_ERROR, "error not enough room %d %d", - *cdata_bytes, - (int) (lockBitstream.bitstreamSizeInBytes)); - } - nv_error = g_enc_funcs.nvEncUnlockBitstream(ei->enc, - lockBitstream.outputBitstream); - if (nv_error != NV_ENC_SUCCESS) - { - LOG(LOG_LEVEL_ERROR, "error nvEncUnlockBitstream %d", + LOG(LOG_LEVEL_INFO, "error nvEncLockBitstream %d", nv_error); } + ++ei->frameCount; } else { - LOG(LOG_LEVEL_ERROR, "error nvEncLockBitstream %d", - nv_error); + LOG(LOG_LEVEL_INFO, "error nvEncEncodePicture %d", nv_error); } - ++ei->frameCount; if (rv == INCREMENTAL_FRAME_ENCODED && (picParams.encodePicFlags & NV_ENC_PIC_FLAG_FORCEIDR)) { From 7a051bf90f55cd12d324dec77d79428e3bb97f0a Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 11 Nov 2023 18:27:57 -0500 Subject: [PATCH 04/39] Slowly introducing AVC444 prototype on top of SHM --- xorgxrdp_helper/xorgxrdp_helper_x11.c | 108 +++++++++++++++----------- xrdp/xrdp_encoder.c | 39 +++++----- xrdp/xrdp_encoder.h | 36 +++++++-- xrdp/xrdp_mm.c | 63 ++++++++++----- 4 files changed, 156 insertions(+), 90 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper_x11.c b/xorgxrdp_helper/xorgxrdp_helper_x11.c index 8bee522ce4..a05dc04685 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_x11.c +++ b/xorgxrdp_helper/xorgxrdp_helper_x11.c @@ -144,7 +144,8 @@ struct mon_info inf_image_t inf_image; GLuint bmp_texture; GLuint enc_texture; - int tex_format; + int tex_format[2]; + int num_tex_formats; GLfloat *(*get_vertices)(GLuint *vertices_bytes, GLuint *vertices_pointes, int num_crects, struct xh_rect *crects, @@ -645,7 +646,14 @@ xorgxrdp_helper_x11_create_pixmap(int width, int height, int magic, if (g_enc == ENC_NVENC) { LOG(LOG_LEVEL_INFO, "using XH_YUV420"); - mi->tex_format = XH_YUV420; +#if AVC444 + mi->tex_format[0] = XH_YUV444_V2_MV; + mi->tex_format[1] = XH_YUV444_V2_AUX; + mi->num_tex_formats = 2; +#else + mi->tex_format[0] = XH_YUV420; + mi->num_tex_formats = 1; +#endif glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height * 3 / 2, 0, GL_RED, GL_UNSIGNED_BYTE, NULL); mi->get_vertices = get_vertices420; @@ -657,7 +665,8 @@ xorgxrdp_helper_x11_create_pixmap(int width, int height, int magic, else if (g_enc == ENC_YAMI) { LOG(LOG_LEVEL_INFO, "using XH_YUV422"); - mi->tex_format = XH_YUV422; + mi->tex_format[0] = XH_YUV422; + mi->num_tex_formats = 1; glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width / 2, height, 0, GL_RGBA, GL_UNSIGNED_INT_8_8_8_8, NULL); mi->get_vertices = get_vertices444; /* same as 444 */ @@ -669,7 +678,8 @@ xorgxrdp_helper_x11_create_pixmap(int width, int height, int magic, else { LOG(LOG_LEVEL_INFO, "using XH_YUV444"); - mi->tex_format = XH_YUV444; + mi->tex_format[0] = XH_YUV444; + mi->num_tex_formats = 1; glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_RGBA, GL_UNSIGNED_INT_8_8_8_8, NULL); mi->get_vertices = get_vertices444; @@ -686,7 +696,7 @@ xorgxrdp_helper_x11_create_pixmap(int width, int height, int magic, glBindTexture(GL_TEXTURE_2D, 0); if (g_enc_funcs[g_enc].create_enc(width, height, - enc_texture, mi->tex_format, + enc_texture, mi->tex_format[0], &(mi->ei)) != 0) { return 1; @@ -703,18 +713,32 @@ xorgxrdp_helper_x11_create_pixmap(int width, int height, int magic, } /*****************************************************************************/ -static void -xorgxrdp_helper_x11_run_shader(int width, int height, - struct mon_info *mi, - struct shader_info *si, - int num_crects, struct xh_rect *crects) +enum encoder_result +xorgxrdp_helper_x11_encode_pixmap(int left, int top, int width, int height, + int mon_id, int num_crects, + struct xh_rect *crects, + void *cdata, int *cdata_bytes) { + struct mon_info *mi; + struct shader_info *si; + enum encoder_result rv; GLuint vao; GLuint vbo; GLfloat *vertices; GLuint vertices_bytes; GLuint vertices_pointes; + mi = g_mons + mon_id % MAX_MON; + int running_size = 0; + if ((width != mi->width) || (height != mi->height)) + { + LOG(LOG_LEVEL_ERROR, "error width %d should be %d " + "height %d should be %d", + width, mi->width, height, mi->height); + return ENCODER_ERROR; + } + void *enc_write_location; + /* rgb to yuv */ glEnable(GL_TEXTURE_2D); glActiveTexture(GL_TEXTURE0); @@ -723,7 +747,7 @@ xorgxrdp_helper_x11_run_shader(int width, int height, glBindFramebuffer(GL_FRAMEBUFFER, g_fb); glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, mi->enc_texture, 0); - glUseProgram(si->program); + /* setup vertices from crects */ vertices = mi->get_vertices(&vertices_bytes, &vertices_pointes, num_crects, crects, width, height); @@ -731,7 +755,7 @@ xorgxrdp_helper_x11_run_shader(int width, int height, { LOG(LOG_LEVEL_ERROR, "error get_vertices failed num_crects %d", num_crects); - return; + return ENCODER_ERROR; } glGenVertexArrays(1, &vao); glGenBuffers(1, &vbo); @@ -740,11 +764,32 @@ xorgxrdp_helper_x11_run_shader(int width, int height, glBufferData(GL_ARRAY_BUFFER, vertices_bytes, vertices, GL_STATIC_DRAW); glEnableVertexAttribArray(0); glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, sizeof(float) * 2, NULL); - /* uniforms */ - glUniform2f(si->tex_size_loc, mi->width, mi->height); - /* viewport and draw */ glViewport(mi->viewport.x, mi->viewport.y, mi->viewport.w, mi->viewport.h); - glDrawArrays(GL_TRIANGLES, 0, vertices_pointes); + XFlush(g_display); + + for (int i = 0; i < mi->num_tex_formats; ++i) { + int index = mi->tex_format[i] % XH_NUM_SHADERS; + si = g_si + index; + LOG(LOG_LEVEL_TRACE, "Running shader ID: %d", index); + glUseProgram(si->program); + /* uniforms */ + glUniform2f(si->tex_size_loc, mi->width, mi->height); + /* draw */ + glDrawArrays(GL_TRIANGLES, 0, vertices_pointes); + /* sync before encoding */ + glFinish(); + /* encode */ + int quad_index = 4 * (i + 1); + enc_write_location = cdata + running_size + quad_index; + rv = g_enc_funcs[g_enc].encode(mi->ei, mi->enc_texture, + enc_write_location, &(cdata_bytes[i])); + if (rv == ENCODER_ERROR) + { + return ENCODER_ERROR; + } + running_size += cdata_bytes[i]; + } + /* cleanup */ glBindBuffer(GL_ARRAY_BUFFER, 0); glBindVertexArray(0); @@ -755,34 +800,9 @@ xorgxrdp_helper_x11_run_shader(int width, int height, g_inf_funcs[g_inf].release_tex_image(mi->inf_image); glBindTexture(GL_TEXTURE_2D, 0); glUseProgram(0); -} - -/*****************************************************************************/ -enum encoder_result -xorgxrdp_helper_x11_encode_pixmap(int left, int top, int width, int height, - int mon_id, int num_crects, - struct xh_rect *crects, - void *cdata, int *cdata_bytes) -{ - struct mon_info *mi; - struct shader_info *si; - enum encoder_result rv; - mi = g_mons + mon_id % MAX_MON; - if ((width != mi->width) || (height != mi->height)) - { - LOG(LOG_LEVEL_ERROR, "error width %d should be %d " - "height %d should be %d", - width, mi->width, height, mi->height); - return ENCODER_ERROR; - } - si = g_si + mi->tex_format % XH_NUM_SHADERS; - xorgxrdp_helper_x11_run_shader(width, height, mi, si, num_crects, crects); - /* sync before encoding */ - XFlush(g_display); + //XFlush(g_display); glFinish(); - /* encode */ - rv = g_enc_funcs[g_enc].encode(mi->ei, mi->enc_texture, - cdata, cdata_bytes); - return rv; + + return INCREMENTAL_FRAME_ENCODED; } diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index c8262a456d..3a2557e0f7 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -82,7 +82,8 @@ static void xrdp_enc_data_done_destructor(void *item, void *closure) { XRDP_ENC_DATA_DONE *enc_done = (XRDP_ENC_DATA_DONE *)item; - g_free(enc_done->comp_pad_data); + g_free(enc_done->comp_pad_data1); + g_free(enc_done->comp_pad_data2); g_free(enc_done); } @@ -361,15 +362,15 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) LOG(LOG_LEVEL_INFO, "process_enc_jpg: error 3"); return 1; } - enc_done->comp_bytes = out_data_bytes + 2; - enc_done->pad_bytes = 256; - enc_done->comp_pad_data = out_data; + enc_done->comp_bytes1 = out_data_bytes + 2; + enc_done->pad_bytes1 = 256; + enc_done->comp_pad_data1 = out_data; enc_done->enc = enc; enc_done->last = index == (enc->num_crects - 1); - enc_done->x = x; - enc_done->y = y; - enc_done->cx = cx; - enc_done->cy = cy; + enc_done->rect.x = x; + enc_done->rect.y = y; + enc_done->rect.cx = cx; + enc_done->rect.cy = cy; /* done with msg */ /* inform main thread done */ tc_mutex_lock(mutex); @@ -488,12 +489,12 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) { return 1; } - enc_done->comp_bytes = tiles_written > 0 ? out_data_bytes : 0; - enc_done->pad_bytes = XRDP_SURCMD_PREFIX_BYTES; - enc_done->comp_pad_data = out_data; + enc_done->comp_bytes1 = tiles_written > 0 ? out_data_bytes : 0; + enc_done->pad_bytes1 = XRDP_SURCMD_PREFIX_BYTES; + enc_done->comp_pad_data1 = out_data; enc_done->enc = enc; - enc_done->cx = self->mm->wm->screen->width; - enc_done->cy = self->mm->wm->screen->height; + enc_done->rect.cx = self->mm->wm->screen->width; + enc_done->rect.cy = self->mm->wm->screen->height; if (self->gfx) { enc_done->flags = (enum xrdp_encoder_flags) @@ -805,19 +806,19 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) return 0; } #if XRDP_AVC444 - enc_done->comp_bytes = 4 + comp_bytes_pre + enc_done->comp_bytes1 = 4 + comp_bytes_pre + out_data_bytes + comp_bytes_pre1 + out_data_bytes1; #else - enc_done->comp_bytes = comp_bytes_pre + out_data_bytes; + enc_done->comp_bytes1 = comp_bytes_pre + out_data_bytes; #endif - enc_done->pad_bytes = XRDP_SURCMD_PREFIX_BYTES; - enc_done->comp_pad_data = out_data; + enc_done->pad_bytes1 = XRDP_SURCMD_PREFIX_BYTES; + enc_done->comp_pad_data1 = out_data; enc_done->enc = enc; enc_done->last = 1; - enc_done->cx = scr_width; - enc_done->cy = scr_height; + enc_done->rect.cx = scr_width; + enc_done->rect.cy = scr_height; enc_done->flags = enc_done_flags; #if 0 diff --git a/xrdp/xrdp_encoder.h b/xrdp/xrdp_encoder.h index d2447560ae..9af86c4686 100644 --- a/xrdp/xrdp_encoder.h +++ b/xrdp/xrdp_encoder.h @@ -60,21 +60,41 @@ struct xrdp_enc_data int shmem_bytes; }; +struct xrdp_enc_rect +{ + short x; + short y; + short cx; + short cy; +}; + typedef struct xrdp_enc_data XRDP_ENC_DATA; /* used when scheduling tasks from xrdp_encoder.c */ struct xrdp_enc_data_done { - int comp_bytes; - int pad_bytes; - char *comp_pad_data; - struct xrdp_enc_data *enc; + /* + Possibility to return more than one stream from a single encode, + mainly due to AVC444v2 streams. + TODO: This is a pretty ugly hack, we should refactor this similarly + to how Ogon or FreeRDP do it. + */ + int out_data_bytes1; + int comp_bytes1; + int pad_bytes1; + char *comp_pad_data1; + + int out_data_bytes2; + int comp_bytes2; + int pad_bytes2; + char *comp_pad_data2; + + int total_comp_bytes; + + struct xrdp_enc_data *enc; /* incoming data */ int last; /* true is this is last message for enc */ int continuation; /* true if this isn't the start of a frame */ - int x; - int y; - int cx; - int cy; + struct xrdp_enc_rect rect; enum xrdp_encoder_flags flags; }; diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index 33b875a831..6bf35231fc 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -3322,11 +3322,11 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) /* do something with msg */ LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: message back bytes %d", enc_done->comp_bytes); - x = enc_done->x; - y = enc_done->y; - cx = enc_done->cx; - cy = enc_done->cy; - if (enc_done->comp_bytes > 0) + x = enc_done->rect.x; + y = enc_done->rect.y; + cx = enc_done->rect.cx; + cy = enc_done->rect.cy; + if (enc_done->comp_bytes1 > 0) { LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_process_enc_done: x %d y %d cx %d cy %d " "frame_id %d use_frame_acks %d", x, y, cx, cy, @@ -3340,17 +3340,41 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) rect.y1 = y; rect.x2 = x + cx; rect.y2 = y + cy; +#if AVC444 + xrdp_egfx_send_frame_start(self->egfx, + enc_done->enc->frame_id, 0); xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, -#if XRDP_AVC444 - XR_RDPGFX_CODECID_AVC444V2, + XR_RDPGFX_CODECID_AVC444V2, + XR_PIXEL_FORMAT_XRGB_8888, + &rect, + enc_done->comp_pad_data1 + enc_done->pad_bytes1, + enc_done->comp_bytes1); + if (enc_done->comp_bytes2 > 0) { + xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); + ++enc_done->enc->frame_id; + xrdp_egfx_send_frame_start(self->egfx, + enc_done->enc->frame_id, 0); + xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, + XR_RDPGFX_CODECID_AVC444V2, + XR_PIXEL_FORMAT_XRGB_8888, + &rect, + enc_done->comp_pad_data2 + enc_done->pad_bytes2, + enc_done->comp_bytes2); + } + xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); #else - XR_RDPGFX_CODECID_AVC420, -#endif - XR_PIXEL_FORMAT_XRGB_8888, - &rect, - enc_done->comp_pad_data + enc_done->pad_bytes, - enc_done->comp_bytes); + xrdp_egfx_send_frame_start(self->egfx, + enc_done->enc->frame_id, 0); + + xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, + XR_RDPGFX_CODECID_AVC420, + XR_PIXEL_FORMAT_XRGB_8888, + &rect, + enc_done->comp_pad_data1 + enc_done->pad_bytes1, + enc_done->comp_bytes1); + xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); +#endif } else if (enc_done->flags & 2) /* gfx progressive rfx */ { @@ -3358,8 +3382,8 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) enc_done->enc->frame_id, 0); xrdp_egfx_send_wire_to_surface2(self->egfx, self->egfx->surface_id, 9, 1, XR_PIXEL_FORMAT_XRGB_8888, - enc_done->comp_pad_data + enc_done->pad_bytes, - enc_done->comp_bytes); + enc_done->comp_pad_data1 + enc_done->pad_bytes1, + enc_done->comp_bytes1); xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); } else @@ -3367,9 +3391,9 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) libxrdp_fastpath_send_frame_marker(self->wm->session, 0, enc_done->enc->frame_id); libxrdp_fastpath_send_surface(self->wm->session, - enc_done->comp_pad_data, - enc_done->pad_bytes, - enc_done->comp_bytes, + enc_done->comp_pad_data1, + enc_done->pad_bytes1, + enc_done->comp_bytes1, x, y, x + cx, y + cy, 32, self->encoder->codec_id, cx, cy); @@ -3420,7 +3444,8 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) } g_free(enc); } - g_free(enc_done->comp_pad_data); + g_free(enc_done->comp_pad_data1); + g_free(enc_done->comp_pad_data2); g_free(enc_done); } return 0; From 9e1a942e8471c27ded5b5a00752fc73da7c16a74 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sun, 12 Nov 2023 19:19:38 -0500 Subject: [PATCH 05/39] Fixing build error --- xorgxrdp_helper/xorgxrdp_helper_x11.c | 4 ++-- xorgxrdp_helper/xorgxrdp_helper_x11.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper_x11.c b/xorgxrdp_helper/xorgxrdp_helper_x11.c index a05dc04685..b05f90af4a 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_x11.c +++ b/xorgxrdp_helper/xorgxrdp_helper_x11.c @@ -717,7 +717,7 @@ enum encoder_result xorgxrdp_helper_x11_encode_pixmap(int left, int top, int width, int height, int mon_id, int num_crects, struct xh_rect *crects, - void *cdata, int *cdata_bytes) + char *cdata, int *cdata_bytes) { struct mon_info *mi; struct shader_info *si; @@ -737,7 +737,7 @@ xorgxrdp_helper_x11_encode_pixmap(int left, int top, int width, int height, width, mi->width, height, mi->height); return ENCODER_ERROR; } - void *enc_write_location; + char *enc_write_location; /* rgb to yuv */ glEnable(GL_TEXTURE_2D); diff --git a/xorgxrdp_helper/xorgxrdp_helper_x11.h b/xorgxrdp_helper/xorgxrdp_helper_x11.h index b6ff1d973b..b89aab108d 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_x11.h +++ b/xorgxrdp_helper/xorgxrdp_helper_x11.h @@ -41,6 +41,6 @@ enum encoder_result xorgxrdp_helper_x11_encode_pixmap(int left, int top, int width, int height, int mon_id, int num_crects, struct xh_rect *crects, - void *cdata, int *cdata_bytes); + char *cdata, int *cdata_bytes); #endif From 0ea79a56f230d345454b3fe20c8048df99bcd8bb Mon Sep 17 00:00:00 2001 From: Nexarian Date: Wed, 15 Nov 2023 22:03:24 -0500 Subject: [PATCH 06/39] Fixing build break errors --- common/xrdp_client_info.h | 13 +++++++------ xrdp/xrdp_mm.c | 8 ++++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/common/xrdp_client_info.h b/common/xrdp_client_info.h index 5508025e9b..7b62e505cc 100644 --- a/common/xrdp_client_info.h +++ b/common/xrdp_client_info.h @@ -221,12 +221,13 @@ enum xrdp_encoder_flags { NONE = 0, ENCODE_COMPLETE = 1 << 0, - GFX_PROGRESSIVE_RFX = 1 << 1, - CONTAINS_DUAL_FRAME_AVC444 = 1 << 2, - CONTAINS_SINGLE_FRAME_AVC444_YUV420 = 1 << 3, - CONTAINS_SINGLE_FRAME_AVC444_CHROMA420 = 1 << 4, - CONTAINS_KEY_FRAME = 1 << 5, - KEY_FRAME_REQUESTED = 1 << 6 + CONTAINS_DUAL_FRAME_AVC444 = 1 << 1, + CONTAINS_SINGLE_FRAME_AVC444_YUV420 = 1 << 2, + CONTAINS_SINGLE_FRAME_AVC444_CHROMA420 = 1 << 3, + CONTAINS_KEY_FRAME = 1 << 4, + KEY_FRAME_REQUESTED = 1 << 5, + GFX_H264 = 1 << 6, + GFX_PROGRESSIVE_RFX = 1 << 7 }; /* yyyymmdd of last incompatible change to xrdp_client_info */ diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index 6bf35231fc..e6b4dec9df 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -3320,8 +3320,12 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) break; } /* do something with msg */ - LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: message back bytes %d", - enc_done->comp_bytes); + LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: " + "message back bytes " + "(comp_bytes1: %d, " + "comp_bytes2: %d)", + enc_done->comp_bytes1, + enc_done->comp_bytes2); x = enc_done->rect.x; y = enc_done->rect.y; cx = enc_done->rect.cx; From 752e4add5846c342bf2173aef94e18d399cf3a48 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Wed, 15 Nov 2023 23:57:48 -0500 Subject: [PATCH 07/39] Fixing build breaks --- xrdp/xrdp_mm.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index e6b4dec9df..b8bddb98a7 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -3348,22 +3348,22 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) xrdp_egfx_send_frame_start(self->egfx, enc_done->enc->frame_id, 0); xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, - XR_RDPGFX_CODECID_AVC444V2, - XR_PIXEL_FORMAT_XRGB_8888, - &rect, - enc_done->comp_pad_data1 + enc_done->pad_bytes1, - enc_done->comp_bytes1); + XR_RDPGFX_CODECID_AVC444V2, + XR_PIXEL_FORMAT_XRGB_8888, + &rect, + enc_done->comp_pad_data1 + enc_done->pad_bytes1, + enc_done->comp_bytes1); if (enc_done->comp_bytes2 > 0) { xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); ++enc_done->enc->frame_id; xrdp_egfx_send_frame_start(self->egfx, - enc_done->enc->frame_id, 0); + enc_done->enc->frame_id, 0); xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, - XR_RDPGFX_CODECID_AVC444V2, - XR_PIXEL_FORMAT_XRGB_8888, - &rect, - enc_done->comp_pad_data2 + enc_done->pad_bytes2, - enc_done->comp_bytes2); + XR_RDPGFX_CODECID_AVC444V2, + XR_PIXEL_FORMAT_XRGB_8888, + &rect, + enc_done->comp_pad_data2 + enc_done->pad_bytes2, + enc_done->comp_bytes2); } xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); #else @@ -3371,11 +3371,11 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) enc_done->enc->frame_id, 0); xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, - XR_RDPGFX_CODECID_AVC420, - XR_PIXEL_FORMAT_XRGB_8888, - &rect, - enc_done->comp_pad_data1 + enc_done->pad_bytes1, - enc_done->comp_bytes1); + XR_RDPGFX_CODECID_AVC420, + XR_PIXEL_FORMAT_XRGB_8888, + &rect, + enc_done->comp_pad_data1 + enc_done->pad_bytes1, + enc_done->comp_bytes1); xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); #endif From 7661972a78b7fb5143da9b411652f5d6aabb5b54 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Thu, 16 Nov 2023 00:10:09 -0500 Subject: [PATCH 08/39] Fix more formatting --- xorgxrdp_helper/xorgxrdp_helper_nvenc.c | 10 +++++----- xorgxrdp_helper/xorgxrdp_helper_x11.c | 3 ++- xrdp/xrdp_encoder.c | 6 +++--- xrdp/xrdp_mm.c | 5 +++-- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c index c66dad2d4a..37854faea9 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c @@ -155,10 +155,10 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, preset_config.presetCfg.version = NV_ENC_CONFIG_VER; nv_error = g_enc_funcs.nvEncGetEncodePresetConfigEx( - lei->enc, createEncodeParams.encodeGUID, - createEncodeParams.presetGUID, - createEncodeParams.tuningInfo, - &preset_config); + lei->enc, createEncodeParams.encodeGUID, + createEncodeParams.presetGUID, + createEncodeParams.tuningInfo, + &preset_config); LOG(LOG_LEVEL_INFO, "nvEncGetEncodePresetConfig rv %d", nv_error); if (nv_error != NV_ENC_SUCCESS) @@ -420,7 +420,7 @@ xorgxrdp_helper_nvenc_encode(struct enc_info *ei, int tex, (int) (lockBitstream.bitstreamSizeInBytes)); } nv_error = g_enc_funcs.nvEncUnlockBitstream(ei->enc, - lockBitstream.outputBitstream); + lockBitstream.outputBitstream); if (nv_error != NV_ENC_SUCCESS) { LOG(LOG_LEVEL_INFO, "unlocking failed"); diff --git a/xorgxrdp_helper/xorgxrdp_helper_x11.c b/xorgxrdp_helper/xorgxrdp_helper_x11.c index b05f90af4a..7e386eaf11 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_x11.c +++ b/xorgxrdp_helper/xorgxrdp_helper_x11.c @@ -767,7 +767,8 @@ xorgxrdp_helper_x11_encode_pixmap(int left, int top, int width, int height, glViewport(mi->viewport.x, mi->viewport.y, mi->viewport.w, mi->viewport.h); XFlush(g_display); - for (int i = 0; i < mi->num_tex_formats; ++i) { + for (int i = 0; i < mi->num_tex_formats; ++i) + { int index = mi->tex_format[i] % XH_NUM_SHADERS; si = g_si + index; LOG(LOG_LEVEL_TRACE, "Running shader ID: %d", index); diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 3a2557e0f7..6464b390da 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -807,9 +807,9 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) } #if XRDP_AVC444 enc_done->comp_bytes1 = 4 + comp_bytes_pre - + out_data_bytes - + comp_bytes_pre1 - + out_data_bytes1; + + out_data_bytes + + comp_bytes_pre1 + + out_data_bytes1; #else enc_done->comp_bytes1 = comp_bytes_pre + out_data_bytes; #endif diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index b8bddb98a7..bf2b02c8c7 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -3346,14 +3346,15 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) rect.y2 = y + cy; #if AVC444 xrdp_egfx_send_frame_start(self->egfx, - enc_done->enc->frame_id, 0); + enc_done->enc->frame_id, 0); xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, XR_RDPGFX_CODECID_AVC444V2, XR_PIXEL_FORMAT_XRGB_8888, &rect, enc_done->comp_pad_data1 + enc_done->pad_bytes1, enc_done->comp_bytes1); - if (enc_done->comp_bytes2 > 0) { + if (enc_done->comp_bytes2 > 0) + { xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); ++enc_done->enc->frame_id; xrdp_egfx_send_frame_start(self->egfx, From 6c2faa2f4b80ba7c0af209ae7bf86f367f445791 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Fri, 17 Nov 2023 01:08:21 -0500 Subject: [PATCH 09/39] Slowly integrating AVC444 --- xorgxrdp_helper/xorgxrdp_helper_x11.c | 5 +++-- xrdp/xrdp_mm.c | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper_x11.c b/xorgxrdp_helper/xorgxrdp_helper_x11.c index 7e386eaf11..eeefdd3863 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_x11.c +++ b/xorgxrdp_helper/xorgxrdp_helper_x11.c @@ -645,14 +645,15 @@ xorgxrdp_helper_x11_create_pixmap(int width, int height, int magic, glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); if (g_enc == ENC_NVENC) { - LOG(LOG_LEVEL_INFO, "using XH_YUV420"); -#if AVC444 +#if XRDP_AVC444 mi->tex_format[0] = XH_YUV444_V2_MV; mi->tex_format[1] = XH_YUV444_V2_AUX; mi->num_tex_formats = 2; + LOG(LOG_LEVEL_INFO, "using XH_YUV444_V2_MV and XH_YUV444_V2_AUX"); #else mi->tex_format[0] = XH_YUV420; mi->num_tex_formats = 1; + LOG(LOG_LEVEL_INFO, "using XH_YUV420"); #endif glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height * 3 / 2, 0, GL_RED, GL_UNSIGNED_BYTE, NULL); diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index bf2b02c8c7..7545a9cab3 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -3344,7 +3344,7 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) rect.y1 = y; rect.x2 = x + cx; rect.y2 = y + cy; -#if AVC444 +#if XRDP_AVC444 xrdp_egfx_send_frame_start(self->egfx, enc_done->enc->frame_id, 0); xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, From 3d0448d4722782441a3120d2d23e7f409699ec9b Mon Sep 17 00:00:00 2001 From: Nexarian Date: Fri, 17 Nov 2023 19:54:17 -0500 Subject: [PATCH 10/39] Working on more AVC444 --- xrdp/xrdp_encoder.c | 123 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 97 insertions(+), 26 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 6464b390da..7e1797df50 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -621,40 +621,31 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) int y; int cx; int cy; - int out_data_bytes; -#if XRDP_AVC444 - int out_data_bytes1; -#endif int rcount; short *rrects; int error; char *out_data; int out_data_alloc_size; XRDP_ENC_DATA_DONE *enc_done; - struct fifo *fifo_processed; - tbus mutex; - tbus event_processed; struct stream ls; struct stream *s; + int enc_done_flags; + int scr_width; + int scr_height; + int out_data_bytes; int comp_bytes_pre; #if XRDP_AVC444 + int out_data_bytes1; int comp_bytes_pre1; #endif - int enc_done_flags; - int scr_width; - int scr_height; - LOG(LOG_LEVEL_DEBUG, "process_enc_x264:"); - LOG(LOG_LEVEL_DEBUG, "process_enc_x264: num_crects %d num_drects %d", + LOG(LOG_LEVEL_DEBUG, "build_enc_h264:"); + LOG(LOG_LEVEL_DEBUG, "build_enc_h264: num_crects %d num_drects %d", enc->num_crects, enc->num_drects); scr_width = self->mm->wm->screen->width; scr_height = self->mm->wm->screen->height; - fifo_processed = self->fifo_processed; - mutex = self->mutex; - event_processed = self->xrdp_encoder_event_processed; - rcount = enc->num_drects; rrects = enc->drects; if (rcount > 15) @@ -761,15 +752,44 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) s->p += out_data_bytes; + uint8_t LC = 0b00; + uint32_t bitstream = + ((uint32_t)(comp_bytes_pre + out_data_bytes) & 0x3FFFFFFFUL) + | ((LC & 0x03UL) << 30UL); + /* chroma 444 */ /* RFX_AVC420_METABLOCK */ comp_bytes_pre1 = build_rfx_avc420_metablock(s, rrects, rcount, scr_width, scr_height); out_data_bytes1 = OUT_DATA_BYTES_DEFAULT_SIZE; - error = xrdp_encoder_x264_encode(self->codec_handle, 0, - enc->width, enc->height, 0, - enc->data, - s->p, &out_data_bytes1); + if (enc->flags & 1) + { + /* already compressed */ + uint8_t *ud = (uint8_t *) (enc->data); + int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24); + if ((cbytes < 1) || (cbytes > out_data_bytes)) + { + LOG(LOG_LEVEL_INFO, "process_enc_h264: bad h264 bytes %d", cbytes); + g_free(out_data); + return 0; + } + LOG(LOG_LEVEL_DEBUG, + "process_enc_h264: already compressed and size is %d", cbytes); + out_data_bytes = cbytes; + g_memcpy(s->p, enc->data + 4, out_data_bytes); + } +#if defined(XRDP_X264) + else + { + error = xrdp_encoder_x264_encode(self->codec_handle, 0, + enc->width, enc->height, 0, + enc->data + + (enc->height * enc->width) + * 3 / 2, + s->p, &out_data_bytes1); + + } +#endif if (error != 0) { LOG_DEVEL(LOG_LEVEL_TRACE, @@ -827,6 +847,61 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) g_hexdump(enc_done->comp_pad_data + enc_done->pad_bytes, enc_done->comp_bytes); #endif + return enc_done; +} + +struct xrdp_enc_rect calculateBoundingBox(short* boxes, int numBoxes) +{ + struct xrdp_enc_rect boundingBox; + boundingBox.x = INT16_MAX; + boundingBox.y = INT16_MAX; + boundingBox.cx = INT16_MIN; + boundingBox.cy = INT16_MIN; + + for (int i = 0; i < numBoxes; ++i) + { + int location = i * 4; + short x = boxes[location + 0]; + short y = boxes[location + 1]; + short cx = boxes[location + 2]; + short cy = boxes[location + 3]; + + boundingBox.x = MIN(boundingBox.x, x); + boundingBox.y = MIN(boundingBox.y, y); + boundingBox.cx = MAX(boundingBox.cx, cx); + boundingBox.cy = MAX(boundingBox.cy, cy); + } + + return boundingBox; +} + +/*****************************************************************************/ +/* called from encoder thread */ +static int +process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) +{ + FIFO *fifo_processed; + tbus mutex; + tbus event_processed; + + fifo_processed = self->fifo_processed; + mutex = self->mutex; + event_processed = self->xrdp_encoder_event_processed; + + XRDP_ENC_DATA_DONE *enc_done; + int mode = 0; + switch (mode) { + case 0: + enc_done = build_enc_h264(self, enc); + break; + // case 1: + // enc_done = build_enc_h264_avc444_yuv420_stream(self, enc); + // enc_done = build_enc_h264_avc444_chroma420_stream(self, enc, enc_done); + // break; + } + + enc_done->rect = calculateBoundingBox(enc->drects, enc->num_drects); + /* done with msg */ /* inform main thread done */ tc_mutex_lock(mutex); @@ -836,13 +911,9 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) g_set_wait_obj(event_processed); return 0; -} -/*****************************************************************************/ -/* called from encoder thread */ -static int -process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) -{ + + return build_enc_h264(self, enc); } From d6a1fbd31f974d07d4a1d6a00a65c4eacac638a6 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Fri, 17 Nov 2023 22:25:18 -0500 Subject: [PATCH 11/39] Fix From c3e3e61848aecf4f7049fb271b6be92be0330bac Mon Sep 17 00:00:00 2001 From: Nexarian Date: Fri, 17 Nov 2023 22:28:25 -0500 Subject: [PATCH 12/39] Fix formatting --- xrdp/xrdp_encoder.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 7e1797df50..090b481681 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -784,8 +784,8 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) error = xrdp_encoder_x264_encode(self->codec_handle, 0, enc->width, enc->height, 0, enc->data - + (enc->height * enc->width) - * 3 / 2, + + (enc->height * enc->width) + * 3 / 2, s->p, &out_data_bytes1); } @@ -850,7 +850,7 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) return enc_done; } -struct xrdp_enc_rect calculateBoundingBox(short* boxes, int numBoxes) +struct xrdp_enc_rect calculateBoundingBox(short *boxes, int numBoxes) { struct xrdp_enc_rect boundingBox; boundingBox.x = INT16_MAX; @@ -890,14 +890,15 @@ process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) XRDP_ENC_DATA_DONE *enc_done; int mode = 0; - switch (mode) { + switch (mode) + { case 0: enc_done = build_enc_h264(self, enc); break; - // case 1: - // enc_done = build_enc_h264_avc444_yuv420_stream(self, enc); - // enc_done = build_enc_h264_avc444_chroma420_stream(self, enc, enc_done); - // break; + // case 1: + // enc_done = build_enc_h264_avc444_yuv420_stream(self, enc); + // enc_done = build_enc_h264_avc444_chroma420_stream(self, enc, enc_done); + // break; } enc_done->rect = calculateBoundingBox(enc->drects, enc->num_drects); From 0d7cc6e76aa3816bc36c698a30b399c3099271db Mon Sep 17 00:00:00 2001 From: Nexarian Date: Fri, 17 Nov 2023 23:28:19 -0500 Subject: [PATCH 13/39] Fixing more build breaks --- xrdp/xrdp_encoder.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 090b481681..dddc7deded 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -613,9 +613,14 @@ build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, return comp_bytes_pre; } -static int +static XRDP_ENC_DATA_DONE * build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) { + if (!self->gfx) + { + return 0; + } + int index; int x; int y; @@ -850,7 +855,8 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) return enc_done; } -struct xrdp_enc_rect calculateBoundingBox(short *boxes, int numBoxes) +struct xrdp_enc_rect +calculateBoundingBox(short *boxes, int numBoxes) { struct xrdp_enc_rect boundingBox; boundingBox.x = INT16_MAX; @@ -880,7 +886,7 @@ struct xrdp_enc_rect calculateBoundingBox(short *boxes, int numBoxes) static int process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) { - FIFO *fifo_processed; + struct fifo *fifo_processed; tbus mutex; tbus event_processed; @@ -912,10 +918,6 @@ process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) g_set_wait_obj(event_processed); return 0; - - - - return build_enc_h264(self, enc); } #else From af1a0b93ba115a8fe4817e817af8fee5cbc6a049 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 18 Nov 2023 03:14:53 -0500 Subject: [PATCH 14/39] Continuing to work on AVC444. Not working yet. --- xorgxrdp_helper/Makefile.am | 4 ++++ xorgxrdp_helper/xorgxrdp_helper.c | 4 ++-- xorgxrdp_helper/xorgxrdp_helper.h | 8 +++++--- xorgxrdp_helper/xorgxrdp_helper_nvenc.c | 4 +++- xorgxrdp_helper/xorgxrdp_helper_x11.c | 2 +- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/xorgxrdp_helper/Makefile.am b/xorgxrdp_helper/Makefile.am index 5f9aeb3216..dddf14d227 100644 --- a/xorgxrdp_helper/Makefile.am +++ b/xorgxrdp_helper/Makefile.am @@ -21,6 +21,10 @@ XRDP_EXTRA_LIBS += $(XRDP_YAMI_LIBS) XRDP_EXTRA_SOURCES += xorgxrdp_helper_yami.c xorgxrdp_helper_yami.h encoder_headers/yami_inf.h endif +if XRDP_AVC444 +AM_CPPFLAGS += -DXRDP_AVC444 +endif + bin_PROGRAMS = \ xorgxrdp_helper diff --git a/xorgxrdp_helper/xorgxrdp_helper.c b/xorgxrdp_helper/xorgxrdp_helper.c index f8117c911c..b0354d3f1a 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.c +++ b/xorgxrdp_helper/xorgxrdp_helper.c @@ -164,7 +164,7 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st bmpdata[1] = cdata_bytes >> 8; bmpdata[2] = cdata_bytes >> 16; bmpdata[3] = cdata_bytes >> 24; - LOG(LOG_LEVEL_INFO, "cdata_bytes %d", cdata_bytes); + LOG(LOG_LEVEL_DEBUG, "cdata_bytes %d", cdata_bytes); } } } @@ -304,7 +304,7 @@ xorg_process_message_61(struct xorgxrdp_info *xi, struct stream *s) bmpdata[1] = cdata_bytes >> 8; bmpdata[2] = cdata_bytes >> 16; bmpdata[3] = cdata_bytes >> 24; - LOG_DEVEL(LOG_LEVEL_INFO, "cdata_bytes %d", cdata_bytes); + LOG_DEVEL(LOG_LEVEL_DEBUG, "cdata_bytes %d", cdata_bytes); } g_free(crects); return 0; diff --git a/xorgxrdp_helper/xorgxrdp_helper.h b/xorgxrdp_helper/xorgxrdp_helper.h index cd9d80e8c7..b7ef6c7042 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.h +++ b/xorgxrdp_helper/xorgxrdp_helper.h @@ -19,9 +19,11 @@ #ifndef _XORGXRDP_HELPER_H #define _XORGXRDP_HELPER_H -#define XH_YUV420 1 -#define XH_YUV422 2 -#define XH_YUV444 3 +#define XH_YUV420 1 +#define XH_YUV422 2 +#define XH_YUV444 3 +#define XH_YUV444_V2_MV 4 +#define XH_YUV444_V2_AUX 6 #define XH_BT601 0 #define XH_BT709FR 1 diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c index 37854faea9..515e55b148 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c @@ -278,7 +278,9 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, reg_res.resourceType = NV_ENC_INPUT_RESOURCE_TYPE_OPENGL_TEX; reg_res.width = width; reg_res.height = height; - if (tex_format == XH_YUV420) + if (tex_format == XH_YUV420 + || tex_format == XH_YUV444_V2_MV + || tex_format == XH_YUV444_V2_AUX) { reg_res.pitch = width; reg_res.bufferFormat = NV_ENC_BUFFER_FORMAT_NV12; diff --git a/xorgxrdp_helper/xorgxrdp_helper_x11.c b/xorgxrdp_helper/xorgxrdp_helper_x11.c index eeefdd3863..5ca3813144 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_x11.c +++ b/xorgxrdp_helper/xorgxrdp_helper_x11.c @@ -645,7 +645,7 @@ xorgxrdp_helper_x11_create_pixmap(int width, int height, int magic, glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); if (g_enc == ENC_NVENC) { -#if XRDP_AVC444 +#if defined(XRDP_AVC444) mi->tex_format[0] = XH_YUV444_V2_MV; mi->tex_format[1] = XH_YUV444_V2_AUX; mi->num_tex_formats = 2; From b69f3627ab4cf5c6ffa425ac9f6104d19ac2f0bc Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 18 Nov 2023 21:07:09 -0500 Subject: [PATCH 15/39] Sorta working AVC444 --- xorgxrdp_helper/xorgxrdp_helper.c | 49 +++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper.c b/xorgxrdp_helper/xorgxrdp_helper.c index b0354d3f1a..153c596074 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.c +++ b/xorgxrdp_helper/xorgxrdp_helper.c @@ -64,7 +64,6 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st int top; int width; int height; - int cdata_bytes; int index; int recv_bytes; enum encoder_result rv; @@ -143,12 +142,14 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st if ((bmpdata != NULL) && (flags & 1)) { - cdata_bytes = 16 * 1024 * 1024; + int cdata_bytes[2]; + cdata_bytes[0] = 128 * 1024 * 1024; + cdata_bytes[1] = cdata_bytes[0]; rv = xorgxrdp_helper_x11_encode_pixmap(left, top, width, height, 0, num_crects, crects, bmpdata + 4, - &cdata_bytes); + cdata_bytes); if (rv == ENCODER_ERROR) { LOG(LOG_LEVEL_ERROR, "error %d", rv); @@ -160,11 +161,19 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st s->p = final_pointer; } - bmpdata[0] = cdata_bytes; - bmpdata[1] = cdata_bytes >> 8; - bmpdata[2] = cdata_bytes >> 16; - bmpdata[3] = cdata_bytes >> 24; - LOG(LOG_LEVEL_DEBUG, "cdata_bytes %d", cdata_bytes); + for (int i = 0; i < 2; ++i) + { + int cdata_value = cdata_bytes[i]; + char *bmpdata_location = i == 0 + ? bmpdata + : bmpdata + cdata_bytes[i - 1] + (4 * i); + bmpdata_location[0] = cdata_value; + bmpdata_location[1] = cdata_value >> 8; + bmpdata_location[2] = cdata_value >> 16; + bmpdata_location[3] = cdata_value >> 24; + LOG_DEVEL(LOG_LEVEL_INFO, + "cdata_bytes %d", cdata_value); + } } } } @@ -200,7 +209,6 @@ xorg_process_message_61(struct xorgxrdp_info *xi, struct stream *s) int frame_id; int width; int height; - int cdata_bytes; int index; enum encoder_result rv; struct xh_rect *crects; @@ -285,10 +293,12 @@ xorg_process_message_61(struct xorgxrdp_info *xi, struct stream *s) } if ((bmpdata != NULL) && (flags & 1)) { - cdata_bytes = 16 * 1024 * 1024; + int cdata_bytes[2]; + cdata_bytes[0] = 128 * 1024 * 1024; + cdata_bytes[1] = cdata_bytes[0]; rv = xorgxrdp_helper_x11_encode_pixmap(0, 0, width, height, 0, num_crects, crects, - bmpdata + 4, &cdata_bytes); + bmpdata, cdata_bytes); if (rv == ENCODER_ERROR) { LOG(LOG_LEVEL_ERROR, "error %d", rv); @@ -300,11 +310,18 @@ xorg_process_message_61(struct xorgxrdp_info *xi, struct stream *s) s->p = final_pointer; } - bmpdata[0] = cdata_bytes; - bmpdata[1] = cdata_bytes >> 8; - bmpdata[2] = cdata_bytes >> 16; - bmpdata[3] = cdata_bytes >> 24; - LOG_DEVEL(LOG_LEVEL_DEBUG, "cdata_bytes %d", cdata_bytes); + for (int i = 0; i < 2; ++i) + { + int cdata_value = cdata_bytes[i]; + char *bmpdata_location = i == 0 + ? bmpdata + : bmpdata + cdata_bytes[i - 1] + (4 * i); + bmpdata_location[0] = cdata_value; + bmpdata_location[1] = cdata_value >> 8; + bmpdata_location[2] = cdata_value >> 16; + bmpdata_location[3] = cdata_value >> 24; + LOG_DEVEL(LOG_LEVEL_INFO, "cdata_bytes %d", cdata_value); + } } g_free(crects); return 0; From 345270394bc62852cc1ba3bd738104ce924cbff6 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Tue, 21 Nov 2023 23:43:41 -0500 Subject: [PATCH 16/39] Tweaks for testing AVC444 --- xorgxrdp_helper/xorgxrdp_helper.c | 6 +-- xorgxrdp_helper/xorgxrdp_helper_nvenc.c | 38 ++++++------- xorgxrdp_helper/xorgxrdp_helper_nvenc.h | 2 +- xorgxrdp_helper/xorgxrdp_helper_shaders.c | 65 +++++++++++++++-------- xorgxrdp_helper/xorgxrdp_helper_x11.c | 2 +- xorgxrdp_helper/xorgxrdp_helper_yami.c | 2 +- xorgxrdp_helper/xorgxrdp_helper_yami.h | 2 +- xrdp/xrdp_encoder.c | 14 ++--- xrdp/xrdp_mm.c | 4 +- 9 files changed, 79 insertions(+), 56 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper.c b/xorgxrdp_helper/xorgxrdp_helper.c index 153c596074..18195b1efb 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.c +++ b/xorgxrdp_helper/xorgxrdp_helper.c @@ -139,16 +139,16 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st { bmpdata = (char *)shmem_ptr; bmpdata += shmem_offset; - if ((bmpdata != NULL) && (flags & 1)) { int cdata_bytes[2]; cdata_bytes[0] = 128 * 1024 * 1024; cdata_bytes[1] = cdata_bytes[0]; + char *bmpdata_base_location = bmpdata + 4; rv = xorgxrdp_helper_x11_encode_pixmap(left, top, width, height, 0, num_crects, crects, - bmpdata + 4, + bmpdata_base_location, cdata_bytes); if (rv == ENCODER_ERROR) { @@ -171,7 +171,7 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st bmpdata_location[1] = cdata_value >> 8; bmpdata_location[2] = cdata_value >> 16; bmpdata_location[3] = cdata_value >> 24; - LOG_DEVEL(LOG_LEVEL_INFO, + LOG(LOG_LEVEL_INFO, "cdata_bytes %d", cdata_value); } } diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c index 515e55b148..4f5e202c9e 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c @@ -142,7 +142,7 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, g_memset(&createEncodeParams, 0, sizeof(createEncodeParams)); createEncodeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; createEncodeParams.encodeGUID = NV_ENC_CODEC_H264_GUID; - createEncodeParams.presetGUID = NV_ENC_PRESET_P7_GUID; + createEncodeParams.presetGUID = NV_ENC_PRESET_P4_GUID; createEncodeParams.tuningInfo = NV_ENC_TUNING_INFO_LOW_LATENCY; createEncodeParams.encodeWidth = width; createEncodeParams.encodeHeight = height; @@ -171,8 +171,8 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, encCfg.version = NV_ENC_CONFIG_VER; memcpy(&encCfg, &preset_config.presetCfg, sizeof(NV_ENC_CONFIG)); encCfg.profileGUID = NV_ENC_H264_PROFILE_MAIN_GUID; - //encCfg.gopLength = NVENC_INFINITE_GOPLENGTH; - encCfg.gopLength = 10; + encCfg.gopLength = NVENC_INFINITE_GOPLENGTH; + //encCfg.gopLength = 10; encCfg.frameIntervalP = 1; /* 1 + B_Frame_Count */ encCfg.frameFieldMode = NV_ENC_PARAMS_FRAME_FIELD_MODE_FRAME; encCfg.mvPrecision = NV_ENC_MV_PRECISION_QUARTER_PEL; @@ -239,24 +239,24 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, } encCfg.rcParams.cbQPIndexOffset = -1; encCfg.encodeCodecConfig.h264Config.chromaFormatIDC = 1; - encCfg.encodeCodecConfig.h264Config.idrPeriod = 2; + //encCfg.encodeCodecConfig.h264Config.idrPeriod = 2; encCfg.encodeCodecConfig.h264Config.repeatSPSPPS = 1; encCfg.encodeCodecConfig.h264Config.disableSPSPPS = 0; //encCfg.encodeCodecConfig.h264Config.maxNumRefFrames = 1; - encCfg.encodeCodecConfig.h264Config.sliceMode = 0; - encCfg.encodeCodecConfig.h264Config.sliceModeData = 0; - encCfg.encodeCodecConfig.h264Config.outputAUD = 1; - encCfg.encodeCodecConfig.h264Config.outputBufferingPeriodSEI = 1; - encCfg.encodeCodecConfig.h264Config.outputPictureTimingSEI = 1; - encCfg.encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_AUTOSELECT; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = 1; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFormat = NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.bitstreamRestrictionFlag = 1; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourDescriptionPresentFlag = 1; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourPrimaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.transferCharacteristics = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; + // encCfg.encodeCodecConfig.h264Config.sliceMode = 0; + // encCfg.encodeCodecConfig.h264Config.sliceModeData = 0; + encCfg.encodeCodecConfig.h264Config.outputAUD = 0; + // encCfg.encodeCodecConfig.h264Config.outputBufferingPeriodSEI = 1; + // encCfg.encodeCodecConfig.h264Config.outputPictureTimingSEI = 1; + // encCfg.encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_AUTOSELECT; + // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = 1; + // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1; + // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFormat = NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED; + // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.bitstreamRestrictionFlag = 1; + // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourDescriptionPresentFlag = 1; + // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourPrimaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709; + // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.transferCharacteristics = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; + // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; createEncodeParams.encodeConfig = &encCfg; @@ -346,7 +346,7 @@ xorgxrdp_helper_nvenc_delete_encoder(struct enc_info *ei) /*****************************************************************************/ enum encoder_result xorgxrdp_helper_nvenc_encode(struct enc_info *ei, int tex, - void *cdata, int *cdata_bytes) + char *cdata, int *cdata_bytes) { NV_ENC_PIC_PARAMS picParams; NV_ENC_LOCK_BITSTREAM lockBitstream; diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.h b/xorgxrdp_helper/xorgxrdp_helper_nvenc.h index 152b69df37..61a54a309c 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_nvenc.h +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.h @@ -28,6 +28,6 @@ int xorgxrdp_helper_nvenc_delete_encoder(struct enc_info *ei); enum encoder_result xorgxrdp_helper_nvenc_encode(struct enc_info *ei, int tex, - void *cdata, int *cdata_bytes); + char *cdata, int *cdata_bytes); #endif diff --git a/xorgxrdp_helper/xorgxrdp_helper_shaders.c b/xorgxrdp_helper/xorgxrdp_helper_shaders.c index 167e0ed958..79d49ab12a 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_shaders.c +++ b/xorgxrdp_helper/xorgxrdp_helper_shaders.c @@ -358,33 +358,56 @@ AUXILIARY VIEW V2 - NV12 */ static const GLchar g_fs_rgb_to_yuv420_av_v2[] = "\ -#version 460 core\n\ uniform sampler2D tex;\n\ uniform vec2 tex_size;\n\ uniform vec4 umath;\n\ uniform vec4 vmath;\n\ -out vec4 FragColor;\n\ void main(void)\n\ {\n\ - vec4 pix[4];\n\ - vec2 coords[4];\n\ - float x = gl_FragCoord.x;\n\ - float y = gl_FragCoord.y;\n\ - float x1 = tex_size.x / 2.0;\n\ - float y1 = tex_size.y / 2.0;\n\ - float adjustedY = floor(y - tex_size.y) * 2.0 + 1.5;\n\ - coords[0] = vec2(floor(x) * 2.0 + 1.5, y);\n\ - coords[1] = vec2(floor(x - x1) * 2.0 + 1.5, y);\n\ - coords[2] = vec2(floor(x) * 2.0 + 0.5, adjustedY);\n\ - coords[3] = vec2(floor(x - x1) * 2.0 + 0.5, adjustedY);\n\ - for (int i = 0; i < 4; i++)\n\ + vec4 pix;\n\ + float x;\n\ + float y;\n\ + float x1;\n\ + x = gl_FragCoord.x;\n\ + y = gl_FragCoord.y;\n\ + x1 = tex_size.x / 2.0;\n\ + if (y < tex_size.y)\n\ {\n\ - pix[i] = texture(tex, coords[i] / tex_size);\n\ - pix[i].a = 1.0;\n\ + if (x < x1)\n\ + {\n\ + x = floor(x) * 2.0 + 1.5;\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + x = floor(x - x1) * 2.0 + 1.5;\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + }\n\ + else\n\ + {\n\ + y = floor(y - tex_size.y) * 2.0 + 1.5;\n\ + if (x < x1)\n\ + {\n\ + x = floor(x) * 2.0 + 0.5;\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + x = floor(x - x1) * 2.0 + 0.5;\n\ + pix = texture2D(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + gl_FragColor = pix;\n\ + }\n\ }\n\ - pix[0] = vec4(clamp(dot(umath, pix[0]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - pix[1] = vec4(clamp(dot(vmath, pix[1]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - pix[2] = vec4(clamp(dot(umath, pix[2]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - pix[3] = vec4(clamp(dot(vmath, pix[3]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - FragColor = pix[int(y < tex_size.y ? (x < x1 ? 0 : 1) : (x < x1 ? 2 : 3))];\n\ }\n"; diff --git a/xorgxrdp_helper/xorgxrdp_helper_x11.c b/xorgxrdp_helper/xorgxrdp_helper_x11.c index 5ca3813144..56d92f606b 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_x11.c +++ b/xorgxrdp_helper/xorgxrdp_helper_x11.c @@ -72,7 +72,7 @@ struct enc_funcs struct enc_info **ei); int (*destroy_enc)(struct enc_info *ei); enum encoder_result (*encode)(struct enc_info *ei, int tex, - void *cdata, int *cdata_bytes); + char *cdata, int *cdata_bytes); }; static struct enc_funcs g_enc_funcs[] = diff --git a/xorgxrdp_helper/xorgxrdp_helper_yami.c b/xorgxrdp_helper/xorgxrdp_helper_yami.c index 5da3205ff8..cdc6f032bd 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_yami.c +++ b/xorgxrdp_helper/xorgxrdp_helper_yami.c @@ -184,7 +184,7 @@ static const EGLint g_create_image_attr[] = /*****************************************************************************/ enum encoder_result xorgxrdp_helper_yami_encode(struct enc_info *ei, int tex, - void *cdata, int *cdata_bytes) + char *cdata, int *cdata_bytes) { EGLClientBuffer cb; EGLImageKHR image; diff --git a/xorgxrdp_helper/xorgxrdp_helper_yami.h b/xorgxrdp_helper/xorgxrdp_helper_yami.h index 26989c7124..5327c2a3b2 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_yami.h +++ b/xorgxrdp_helper/xorgxrdp_helper_yami.h @@ -28,6 +28,6 @@ int xorgxrdp_helper_yami_delete_encoder(struct enc_info *ei); enum encoder_result xorgxrdp_helper_yami_encode(struct enc_info *ei, int tex, - void *cdata, int *cdata_bytes); + char *cdata, int *cdata_bytes); #endif diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index dddc7deded..8a92f50d44 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -524,7 +524,7 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) } #endif -#define SAVE_VIDEO 0 +#define SAVE_VIDEO 1 #if SAVE_VIDEO #include @@ -543,7 +543,7 @@ static int n_save_data(const char *data, int data_size, int width, int height) int bytes_follow; } header; - fd = open("video.bin", O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + fd = open("~/video.bin", O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); lseek(fd, 0, SEEK_END); header.tag[0] = 'B'; header.tag[1] = 'E'; @@ -770,7 +770,8 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) if (enc->flags & 1) { /* already compressed */ - uint8_t *ud = (uint8_t *) (enc->data); + //uint8_t *ud = (uint8_t *) (enc->data); + uint8_t *ud = (uint8_t *) (enc->data + enc_done->out_data_bytes + 4); int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24); if ((cbytes < 1) || (cbytes > out_data_bytes)) { @@ -780,8 +781,9 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) } LOG(LOG_LEVEL_DEBUG, "process_enc_h264: already compressed and size is %d", cbytes); - out_data_bytes = cbytes; - g_memcpy(s->p, enc->data + 4, out_data_bytes); + out_data_bytes1 = cbytes; + g_memcpy(s->p, enc->data + out_data_bytes + 8, out_data_bytes1); + //g_memcpy(s->p, enc->data + 4, out_data_bytes); } #if defined(XRDP_X264) else @@ -806,7 +808,7 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) s->p += out_data_bytes1; s_push_layer(s, sec_hdr, 0); s_pop_layer(s, mcs_hdr); - out_uint32_le(s, comp_bytes_pre + out_data_bytes); + out_uint32_le(s, bitstream); s_pop_layer(s, sec_hdr); s->end = s->p; diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index 7545a9cab3..8f812b0936 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -3320,7 +3320,7 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) break; } /* do something with msg */ - LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: " + LOG(LOG_LEVEL_INFO, "xrdp_mm_process_enc_done: " "message back bytes " "(comp_bytes1: %d, " "comp_bytes2: %d)", @@ -3338,8 +3338,6 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) self->wm->client_info->use_frame_acks); if (enc_done->flags & 1) /* gfx h264 */ { - xrdp_egfx_send_frame_start(self->egfx, - enc_done->enc->frame_id, 0); rect.x1 = x; rect.y1 = y; rect.x2 = x + cx; From e14d03de61edfcff90ea695b4d683811310f342f Mon Sep 17 00:00:00 2001 From: Nexarian Date: Wed, 22 Nov 2023 01:37:13 -0500 Subject: [PATCH 17/39] More fixes --- xorgxrdp_helper/xorgxrdp_helper.c | 2 +- xorgxrdp_helper/xorgxrdp_helper_nvenc.c | 4 ++-- xrdp/xrdp_encoder.c | 18 ++++++++++-------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper.c b/xorgxrdp_helper/xorgxrdp_helper.c index 18195b1efb..302c86dc87 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.c +++ b/xorgxrdp_helper/xorgxrdp_helper.c @@ -161,7 +161,7 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st s->p = final_pointer; } - for (int i = 0; i < 2; ++i) + for (int i = 0; i < 1; ++i) { int cdata_value = cdata_bytes[i]; char *bmpdata_location = i == 0 diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c index 4f5e202c9e..6f525cf155 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c @@ -243,8 +243,8 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, encCfg.encodeCodecConfig.h264Config.repeatSPSPPS = 1; encCfg.encodeCodecConfig.h264Config.disableSPSPPS = 0; //encCfg.encodeCodecConfig.h264Config.maxNumRefFrames = 1; - // encCfg.encodeCodecConfig.h264Config.sliceMode = 0; - // encCfg.encodeCodecConfig.h264Config.sliceModeData = 0; + encCfg.encodeCodecConfig.h264Config.sliceMode = 0; + encCfg.encodeCodecConfig.h264Config.sliceModeData = 0; encCfg.encodeCodecConfig.h264Config.outputAUD = 0; // encCfg.encodeCodecConfig.h264Config.outputBufferingPeriodSEI = 1; // encCfg.encodeCodecConfig.h264Config.outputPictureTimingSEI = 1; diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 8a92f50d44..de96700368 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -752,9 +752,10 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) g_free(out_data); return 0; } - #if XRDP_AVC444 - +#if SAVE_VIDEO + n_save_data(s->p, out_data_bytes, enc->width, enc->height); +#endif s->p += out_data_bytes; uint8_t LC = 0b00; @@ -773,7 +774,7 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) //uint8_t *ud = (uint8_t *) (enc->data); uint8_t *ud = (uint8_t *) (enc->data + enc_done->out_data_bytes + 4); int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24); - if ((cbytes < 1) || (cbytes > out_data_bytes)) + if ((cbytes < 1) || (cbytes > out_data_bytes1)) { LOG(LOG_LEVEL_INFO, "process_enc_h264: bad h264 bytes %d", cbytes); g_free(out_data); @@ -783,7 +784,6 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) "process_enc_h264: already compressed and size is %d", cbytes); out_data_bytes1 = cbytes; g_memcpy(s->p, enc->data + out_data_bytes + 8, out_data_bytes1); - //g_memcpy(s->p, enc->data + 4, out_data_bytes); } #if defined(XRDP_X264) else @@ -805,6 +805,9 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) g_free(out_data); return 0; } +#if SAVE_VIDEO + n_save_data(s->p, out_data_bytes1, enc->width, enc->height); +#endif s->p += out_data_bytes1; s_push_layer(s, sec_hdr, 0); s_pop_layer(s, mcs_hdr); @@ -813,6 +816,9 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) s->end = s->p; #else +#if SAVE_VIDEO + n_save_data(s->p, out_data_bytes, enc->width, enc->height); +#endif s->end = s->p + out_data_bytes; #endif @@ -823,10 +829,6 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) out_uint32_le(s, out_data_bytes); } -#if SAVE_VIDEO - n_save_data(s->p, out_data_bytes, enc->width, enc->height); -#endif - enc_done = g_new0(XRDP_ENC_DATA_DONE, 1); if (enc_done == NULL) { From 67e04ce15c8ea6c2173ada68f8df4ec3de86dbb0 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Fri, 24 Nov 2023 12:13:24 -0500 Subject: [PATCH 18/39] Continuing to work on AVC444. Fix byte offset error. --- xorgxrdp_helper/xorgxrdp_helper.c | 43 +++++++++++++++++-------- xorgxrdp_helper/xorgxrdp_helper_nvenc.c | 34 +++++++++---------- xorgxrdp_helper/xorgxrdp_helper_x11.c | 4 +-- xrdp/xrdp_encoder.c | 13 ++++---- 4 files changed, 55 insertions(+), 39 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper.c b/xorgxrdp_helper/xorgxrdp_helper.c index 302c86dc87..f10926c149 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.c +++ b/xorgxrdp_helper/xorgxrdp_helper.c @@ -52,7 +52,8 @@ int xrdp_invalidate = 0; /*****************************************************************************/ static int -xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct stream *s, int *fd) +xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, + struct stream *s, int *fd) { int num_drects; int num_crects; @@ -72,6 +73,11 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st char msg[4]; unsigned int num_fds; void *shmem_ptr = NULL; +#ifdef XRDP_AVC444 + const int num_encoded_frames = 2; +#else + const int num_encoded_frames = 1; +#endif /* dirty pixels */ in_uint16_le(s, num_drects); @@ -103,12 +109,15 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st if (xrdp_invalidate > 0 && frame_id == 1) { // Let it through. We are no longer resizing. - LOG(LOG_LEVEL_DEBUG, "Invalidate received and processing frame ID 1. Unblocking encoder. Invalidate is %d.", xrdp_invalidate); + LOG(LOG_LEVEL_DEBUG, "Invalidate received and processing " + "frame ID 1. Unblocking encoder. " + "Invalidate is %d.", xrdp_invalidate); xi->resizing = 0; } else { - LOG(LOG_LEVEL_DEBUG, "Blocked Incoming Frame ID %d. Invalidate is %d", frame_id, xrdp_invalidate); + LOG(LOG_LEVEL_DEBUG, "Blocked Incoming Frame ID %d. " + "Invalidate is %d", frame_id, xrdp_invalidate); return 0; } } @@ -130,7 +139,7 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st "g_sck_recv_fd_set rv %d fd %d, num_fds %d", recv_bytes, *fd, num_fds); - bmpdata = NULL; + shmem_ptr = NULL; if (recv_bytes == 4) { if (num_fds == 1) @@ -141,14 +150,19 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st bmpdata += shmem_offset; if ((bmpdata != NULL) && (flags & 1)) { - int cdata_bytes[2]; - cdata_bytes[0] = 128 * 1024 * 1024; - cdata_bytes[1] = cdata_bytes[0]; - char *bmpdata_base_location = bmpdata + 4; + int cdata_bytes[num_encoded_frames]; + for (int i = 0; i < num_encoded_frames; ++i) + { + cdata_bytes[i] = 128 * 1024 * 1024; + } + /* + Not using a 4 byte offset here, + this is handled in encode_pixmap. + */ rv = xorgxrdp_helper_x11_encode_pixmap(left, top, width, height, 0, num_crects, crects, - bmpdata_base_location, + bmpdata, cdata_bytes); if (rv == ENCODER_ERROR) { @@ -157,11 +171,11 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, struct st if (rv == KEY_FRAME_ENCODED) { s->p = flag_pointer; - out_uint32_le(s, 1 | 2); + out_uint32_le(s, flags | 1 | 2); s->p = final_pointer; } - for (int i = 0; i < 1; ++i) + for (int i = 0; i < num_encoded_frames; ++i) { int cdata_value = cdata_bytes[i]; char *bmpdata_location = i == 0 @@ -242,12 +256,15 @@ xorg_process_message_61(struct xorgxrdp_info *xi, struct stream *s) if (xrdp_invalidate > 0 && frame_id == 1) { // Let it through. We are no longer resizing. - LOG(LOG_LEVEL_DEBUG, "Invalidate received and processing frame ID 1. Unblocking encoder. Invalidate is %d.", xrdp_invalidate); + LOG(LOG_LEVEL_DEBUG, "Invalidate received and processing " + "frame ID 1. Unblocking encoder. " + "Invalidate is %d.", xrdp_invalidate); xi->resizing = 0; } else { - LOG(LOG_LEVEL_DEBUG, "Blocked Incoming Frame ID %d. Invalidate is %d", frame_id, xrdp_invalidate); + LOG(LOG_LEVEL_DEBUG, "Blocked Incoming Frame ID %d. " + "Invalidate is %d", frame_id, xrdp_invalidate); return 0; } } diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c index 6f525cf155..74d4aef45e 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c @@ -142,7 +142,7 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, g_memset(&createEncodeParams, 0, sizeof(createEncodeParams)); createEncodeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; createEncodeParams.encodeGUID = NV_ENC_CODEC_H264_GUID; - createEncodeParams.presetGUID = NV_ENC_PRESET_P4_GUID; + createEncodeParams.presetGUID = NV_ENC_PRESET_P2_GUID; createEncodeParams.tuningInfo = NV_ENC_TUNING_INFO_LOW_LATENCY; createEncodeParams.encodeWidth = width; createEncodeParams.encodeHeight = height; @@ -239,24 +239,22 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, } encCfg.rcParams.cbQPIndexOffset = -1; encCfg.encodeCodecConfig.h264Config.chromaFormatIDC = 1; - //encCfg.encodeCodecConfig.h264Config.idrPeriod = 2; encCfg.encodeCodecConfig.h264Config.repeatSPSPPS = 1; encCfg.encodeCodecConfig.h264Config.disableSPSPPS = 0; - //encCfg.encodeCodecConfig.h264Config.maxNumRefFrames = 1; encCfg.encodeCodecConfig.h264Config.sliceMode = 0; encCfg.encodeCodecConfig.h264Config.sliceModeData = 0; - encCfg.encodeCodecConfig.h264Config.outputAUD = 0; - // encCfg.encodeCodecConfig.h264Config.outputBufferingPeriodSEI = 1; - // encCfg.encodeCodecConfig.h264Config.outputPictureTimingSEI = 1; - // encCfg.encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_AUTOSELECT; - // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = 1; - // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1; - // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFormat = NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED; - // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.bitstreamRestrictionFlag = 1; - // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourDescriptionPresentFlag = 1; - // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourPrimaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709; - // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.transferCharacteristics = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; - // encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; + encCfg.encodeCodecConfig.h264Config.outputAUD = 1; + encCfg.encodeCodecConfig.h264Config.outputBufferingPeriodSEI = 1; + encCfg.encodeCodecConfig.h264Config.outputPictureTimingSEI = 1; + encCfg.encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_AUTOSELECT; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = 1; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFormat = NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.bitstreamRestrictionFlag = 1; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourDescriptionPresentFlag = 1; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourPrimaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.transferCharacteristics = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; + encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; createEncodeParams.encodeConfig = &encCfg; @@ -360,11 +358,13 @@ xorgxrdp_helper_nvenc_encode(struct enc_info *ei, int tex, picParams.inputWidth = ei->width; picParams.inputHeight = ei->height; picParams.outputBitstream = ei->bitstreamBuffer; - picParams.inputTimeStamp = g_time3(); + picParams.inputTimeStamp = ei->frameCount; picParams.pictureStruct = NV_ENC_PIC_STRUCT_FRAME; if (xrdp_invalidate > 0 || ei->frameCount == 0) { - picParams.encodePicFlags = NV_ENC_PIC_FLAG_OUTPUT_SPSPPS | NV_ENC_PIC_FLAG_FORCEIDR | NV_ENC_PIC_FLAG_FORCEINTRA; + picParams.encodePicFlags |= NV_ENC_PIC_FLAG_FORCEIDR; + picParams.encodePicFlags |= NV_ENC_PIC_FLAG_OUTPUT_SPSPPS; + picParams.encodePicFlags |= NV_ENC_PIC_FLAG_FORCEINTRA; picParams.pictureType = NV_ENC_PIC_TYPE_IDR; LOG(LOG_LEVEL_INFO, "Forcing NVENC H264 IDR SPSPPS for frame id: %d," "invalidate is: %d", ei->frameCount, xrdp_invalidate); diff --git a/xorgxrdp_helper/xorgxrdp_helper_x11.c b/xorgxrdp_helper/xorgxrdp_helper_x11.c index 56d92f606b..ca92ed594d 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_x11.c +++ b/xorgxrdp_helper/xorgxrdp_helper_x11.c @@ -306,7 +306,7 @@ xorgxrdp_helper_x11_init(void) fsource[XH_SHADERCOPY] = g_fs_copy; /* create rgb2yuv shader */ vsource[XH_SHADERRGB2YUV420] = g_vs; - fsource[XH_SHADERRGB2YUV420] = g_fs_rgb_to_yuv420; + fsource[XH_SHADERRGB2YUV420] = g_fs_rgb_to_yuv420_mv; //g_fs_rgb_to_yuv420; /* create rgb2yuv shader */ vsource[XH_SHADERRGB2YUV422] = g_vs; fsource[XH_SHADERRGB2YUV422] = g_fs_rgb_to_yuv422; @@ -772,7 +772,7 @@ xorgxrdp_helper_x11_encode_pixmap(int left, int top, int width, int height, { int index = mi->tex_format[i] % XH_NUM_SHADERS; si = g_si + index; - LOG(LOG_LEVEL_TRACE, "Running shader ID: %d", index); + LOG(LOG_LEVEL_INFO, "Running shader ID: %d", index); glUseProgram(si->program); /* uniforms */ glUniform2f(si->tex_size_loc, mi->width, mi->height); diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index de96700368..10c9684de2 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -524,7 +524,7 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) } #endif -#define SAVE_VIDEO 1 +#define SAVE_VIDEO 0 #if SAVE_VIDEO #include @@ -726,8 +726,8 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) g_free(out_data); return 0; } - LOG(LOG_LEVEL_DEBUG, - "process_enc_h264: already compressed and size is %d", cbytes); + LOG(LOG_LEVEL_INFO, + "process_enc_h264: Main frame already compressed and size is %d", cbytes); out_data_bytes = cbytes; g_memcpy(s->p, enc->data + 4, out_data_bytes); } @@ -780,8 +780,8 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) g_free(out_data); return 0; } - LOG(LOG_LEVEL_DEBUG, - "process_enc_h264: already compressed and size is %d", cbytes); + LOG(LOG_LEVEL_INFO, + "process_enc_h264: Aux frame already compressed and size is %d", cbytes); out_data_bytes1 = cbytes; g_memcpy(s->p, enc->data + out_data_bytes + 8, out_data_bytes1); } @@ -794,7 +794,6 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) + (enc->height * enc->width) * 3 / 2, s->p, &out_data_bytes1); - } #endif if (error != 0) @@ -911,7 +910,7 @@ process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) // break; } - enc_done->rect = calculateBoundingBox(enc->drects, enc->num_drects); + //enc_done->rect = calculateBoundingBox(enc->drects, enc->num_drects); /* done with msg */ /* inform main thread done */ From 708e30421e26c0ed497a6dfa1e48fccbff52f396 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 25 Nov 2023 02:26:44 -0500 Subject: [PATCH 19/39] Working AVC444v2 again with shm. --- xorgxrdp_helper/xorgxrdp_helper_nvenc.c | 36 ++++++++----- xorgxrdp_helper/xorgxrdp_helper_shaders.c | 65 ++++++++--------------- xorgxrdp_helper/xorgxrdp_helper_x11.c | 6 ++- xrdp/Makefile.am | 4 ++ xrdp/xrdp_encoder.c | 16 +++--- 5 files changed, 61 insertions(+), 66 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c index 74d4aef45e..f6a9d39aa9 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c @@ -79,7 +79,8 @@ xorgxrdp_helper_nvenc_init(void) g_lib = g_load_library(g_lib_name1); if (g_lib == 0) { - LOG(LOG_LEVEL_ERROR, "load library for %s/%s failed", g_lib_name, g_lib_name1); + LOG(LOG_LEVEL_ERROR, "load library for %s/%s failed", + g_lib_name, g_lib_name1); return 1; } } @@ -133,7 +134,8 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, params.deviceType = NV_ENC_DEVICE_TYPE_OPENGL; params.apiVersion = NVENCAPI_VERSION; nv_error = g_enc_funcs.nvEncOpenEncodeSessionEx(¶ms, &(lei->enc)); - LOG(LOG_LEVEL_INFO, "nvEncOpenEncodeSessionEx rv %d enc %p", nv_error, lei->enc); + LOG(LOG_LEVEL_INFO, "nvEncOpenEncodeSessionEx rv %d enc %p", + nv_error, lei->enc); if (nv_error != NV_ENC_SUCCESS) { return 1; @@ -142,7 +144,7 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, g_memset(&createEncodeParams, 0, sizeof(createEncodeParams)); createEncodeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; createEncodeParams.encodeGUID = NV_ENC_CODEC_H264_GUID; - createEncodeParams.presetGUID = NV_ENC_PRESET_P2_GUID; + createEncodeParams.presetGUID = NV_ENC_PRESET_P7_GUID; createEncodeParams.tuningInfo = NV_ENC_TUNING_INFO_LOW_LATENCY; createEncodeParams.encodeWidth = width; createEncodeParams.encodeHeight = height; @@ -172,7 +174,6 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, memcpy(&encCfg, &preset_config.presetCfg, sizeof(NV_ENC_CONFIG)); encCfg.profileGUID = NV_ENC_H264_PROFILE_MAIN_GUID; encCfg.gopLength = NVENC_INFINITE_GOPLENGTH; - //encCfg.gopLength = 10; encCfg.frameIntervalP = 1; /* 1 + B_Frame_Count */ encCfg.frameFieldMode = NV_ENC_PARAMS_FRAME_FIELD_MODE_FRAME; encCfg.mvPrecision = NV_ENC_MV_PRECISION_QUARTER_PEL; @@ -241,20 +242,31 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, encCfg.encodeCodecConfig.h264Config.chromaFormatIDC = 1; encCfg.encodeCodecConfig.h264Config.repeatSPSPPS = 1; encCfg.encodeCodecConfig.h264Config.disableSPSPPS = 0; + + /* + TODO: Fix this! + This is somehow necessary to work with Mac OS... + */ + encCfg.encodeCodecConfig.h264Config.maxNumRefFrames = 1; + encCfg.encodeCodecConfig.h264Config.sliceMode = 0; encCfg.encodeCodecConfig.h264Config.sliceModeData = 0; encCfg.encodeCodecConfig.h264Config.outputAUD = 1; encCfg.encodeCodecConfig.h264Config.outputBufferingPeriodSEI = 1; encCfg.encodeCodecConfig.h264Config.outputPictureTimingSEI = 1; encCfg.encodeCodecConfig.h264Config.level = NV_ENC_LEVEL_AUTOSELECT; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = 1; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.videoFormat = NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.bitstreamRestrictionFlag = 1; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourDescriptionPresentFlag = 1; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourPrimaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.transferCharacteristics = NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; - encCfg.encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; + + NV_ENC_CONFIG_H264_VUI_PARAMETERS *h264VUIParameters = + &encCfg.encodeCodecConfig.h264Config.h264VUIParameters; + h264VUIParameters->videoFullRangeFlag = 1; + h264VUIParameters->videoSignalTypePresentFlag = 1; + h264VUIParameters->videoFormat = NV_ENC_VUI_VIDEO_FORMAT_UNSPECIFIED; + h264VUIParameters->bitstreamRestrictionFlag = 1; + h264VUIParameters->colourDescriptionPresentFlag = 1; + h264VUIParameters->colourPrimaries = NV_ENC_VUI_COLOR_PRIMARIES_BT709; + h264VUIParameters->transferCharacteristics = + NV_ENC_VUI_TRANSFER_CHARACTERISTIC_BT709; + h264VUIParameters->colourMatrix = NV_ENC_VUI_MATRIX_COEFFS_BT709; createEncodeParams.encodeConfig = &encCfg; diff --git a/xorgxrdp_helper/xorgxrdp_helper_shaders.c b/xorgxrdp_helper/xorgxrdp_helper_shaders.c index 79d49ab12a..167e0ed958 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_shaders.c +++ b/xorgxrdp_helper/xorgxrdp_helper_shaders.c @@ -358,56 +358,33 @@ AUXILIARY VIEW V2 - NV12 */ static const GLchar g_fs_rgb_to_yuv420_av_v2[] = "\ +#version 460 core\n\ uniform sampler2D tex;\n\ uniform vec2 tex_size;\n\ uniform vec4 umath;\n\ uniform vec4 vmath;\n\ +out vec4 FragColor;\n\ void main(void)\n\ {\n\ - vec4 pix;\n\ - float x;\n\ - float y;\n\ - float x1;\n\ - x = gl_FragCoord.x;\n\ - y = gl_FragCoord.y;\n\ - x1 = tex_size.x / 2.0;\n\ - if (y < tex_size.y)\n\ - {\n\ - if (x < x1)\n\ - {\n\ - x = floor(x) * 2.0 + 1.5;\n\ - pix = texture2D(tex, vec2(x, y) / tex_size);\n\ - pix.a = 1.0;\n\ - pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - gl_FragColor = pix;\n\ - }\n\ - else\n\ - {\n\ - x = floor(x - x1) * 2.0 + 1.5;\n\ - pix = texture2D(tex, vec2(x, y) / tex_size);\n\ - pix.a = 1.0;\n\ - pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - gl_FragColor = pix;\n\ - }\n\ - }\n\ - else\n\ + vec4 pix[4];\n\ + vec2 coords[4];\n\ + float x = gl_FragCoord.x;\n\ + float y = gl_FragCoord.y;\n\ + float x1 = tex_size.x / 2.0;\n\ + float y1 = tex_size.y / 2.0;\n\ + float adjustedY = floor(y - tex_size.y) * 2.0 + 1.5;\n\ + coords[0] = vec2(floor(x) * 2.0 + 1.5, y);\n\ + coords[1] = vec2(floor(x - x1) * 2.0 + 1.5, y);\n\ + coords[2] = vec2(floor(x) * 2.0 + 0.5, adjustedY);\n\ + coords[3] = vec2(floor(x - x1) * 2.0 + 0.5, adjustedY);\n\ + for (int i = 0; i < 4; i++)\n\ {\n\ - y = floor(y - tex_size.y) * 2.0 + 1.5;\n\ - if (x < x1)\n\ - {\n\ - x = floor(x) * 2.0 + 0.5;\n\ - pix = texture2D(tex, vec2(x, y) / tex_size);\n\ - pix.a = 1.0;\n\ - pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - gl_FragColor = pix;\n\ - }\n\ - else\n\ - {\n\ - x = floor(x - x1) * 2.0 + 0.5;\n\ - pix = texture2D(tex, vec2(x, y) / tex_size);\n\ - pix.a = 1.0;\n\ - pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - gl_FragColor = pix;\n\ - }\n\ + pix[i] = texture(tex, coords[i] / tex_size);\n\ + pix[i].a = 1.0;\n\ }\n\ + pix[0] = vec4(clamp(dot(umath, pix[0]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + pix[1] = vec4(clamp(dot(vmath, pix[1]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + pix[2] = vec4(clamp(dot(umath, pix[2]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + pix[3] = vec4(clamp(dot(vmath, pix[3]), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + FragColor = pix[int(y < tex_size.y ? (x < x1 ? 0 : 1) : (x < x1 ? 2 : 3))];\n\ }\n"; diff --git a/xorgxrdp_helper/xorgxrdp_helper_x11.c b/xorgxrdp_helper/xorgxrdp_helper_x11.c index ca92ed594d..1479a331e0 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_x11.c +++ b/xorgxrdp_helper/xorgxrdp_helper_x11.c @@ -306,7 +306,7 @@ xorgxrdp_helper_x11_init(void) fsource[XH_SHADERCOPY] = g_fs_copy; /* create rgb2yuv shader */ vsource[XH_SHADERRGB2YUV420] = g_vs; - fsource[XH_SHADERRGB2YUV420] = g_fs_rgb_to_yuv420_mv; //g_fs_rgb_to_yuv420; + fsource[XH_SHADERRGB2YUV420] = g_fs_rgb_to_yuv420; /* create rgb2yuv shader */ vsource[XH_SHADERRGB2YUV422] = g_vs; fsource[XH_SHADERRGB2YUV422] = g_fs_rgb_to_yuv422; @@ -783,12 +783,14 @@ xorgxrdp_helper_x11_encode_pixmap(int left, int top, int width, int height, /* encode */ int quad_index = 4 * (i + 1); enc_write_location = cdata + running_size + quad_index; + int encoded_size = cdata_bytes[i]; rv = g_enc_funcs[g_enc].encode(mi->ei, mi->enc_texture, - enc_write_location, &(cdata_bytes[i])); + enc_write_location, &encoded_size); if (rv == ENCODER_ERROR) { return ENCODER_ERROR; } + cdata_bytes[i] = encoded_size; running_size += cdata_bytes[i]; } diff --git a/xrdp/Makefile.am b/xrdp/Makefile.am index d121ca8b6d..25ca2f5925 100644 --- a/xrdp/Makefile.am +++ b/xrdp/Makefile.am @@ -34,6 +34,10 @@ XRDP_EXTRA_LIBS += $(XRDP_X264_LIBS) XRDP_EXTRA_SOURCES += xrdp_encoder_x264.c xrdp_encoder_x264.h endif +if XRDP_AVC444 +AM_CPPFLAGS += -DXRDP_AVC444 +endif + if XRDP_PIXMAN AM_CPPFLAGS += -DXRDP_PIXMAN AM_CPPFLAGS += $(PIXMAN_CFLAGS) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 10c9684de2..57a572caa8 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -722,24 +722,25 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24); if ((cbytes < 1) || (cbytes > out_data_bytes)) { - LOG(LOG_LEVEL_INFO, "process_enc_h264: bad h264 bytes %d", cbytes); + LOG(LOG_LEVEL_DEBUG, "process_enc_h264: bad h264 bytes %d", cbytes); g_free(out_data); return 0; } - LOG(LOG_LEVEL_INFO, - "process_enc_h264: Main frame already compressed and size is %d", cbytes); + LOG(LOG_LEVEL_DEBUG, + "process_enc_h264: Main frame already compressed and size is %d", + cbytes); out_data_bytes = cbytes; g_memcpy(s->p, enc->data + 4, out_data_bytes); } +#if defined(XRDP_X264) else { -#if defined(XRDP_X264) error = xrdp_encoder_x264_encode(self->codec_handle, 0, enc->width, enc->height, 0, enc->data, s->p, &out_data_bytes); -#endif } +#endif LOG_DEVEL(LOG_LEVEL_TRACE, "process_enc_h264: xrdp_encoder_x264_encode rv %d " "out_data_bytes %d width %d height %d", @@ -771,8 +772,7 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) if (enc->flags & 1) { /* already compressed */ - //uint8_t *ud = (uint8_t *) (enc->data); - uint8_t *ud = (uint8_t *) (enc->data + enc_done->out_data_bytes + 4); + uint8_t *ud = (uint8_t *) (enc->data + out_data_bytes + 4); int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24); if ((cbytes < 1) || (cbytes > out_data_bytes1)) { @@ -910,7 +910,7 @@ process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) // break; } - //enc_done->rect = calculateBoundingBox(enc->drects, enc->num_drects); + enc_done->rect = calculateBoundingBox(enc->drects, enc->num_drects); /* done with msg */ /* inform main thread done */ From 4f363002d38f46442a37026df0fcf00faa01727f Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 25 Nov 2023 02:32:20 -0500 Subject: [PATCH 20/39] Fixing style check --- xorgxrdp_helper/xorgxrdp_helper.c | 10 +++---- xorgxrdp_helper/xorgxrdp_helper_nvenc.c | 4 +-- xrdp/xrdp_mm.c | 39 +++++++++++++++---------- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper.c b/xorgxrdp_helper/xorgxrdp_helper.c index f10926c149..4711914143 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.c +++ b/xorgxrdp_helper/xorgxrdp_helper.c @@ -110,14 +110,14 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, { // Let it through. We are no longer resizing. LOG(LOG_LEVEL_DEBUG, "Invalidate received and processing " - "frame ID 1. Unblocking encoder. " - "Invalidate is %d.", xrdp_invalidate); + "frame ID 1. Unblocking encoder. " + "Invalidate is %d.", xrdp_invalidate); xi->resizing = 0; } else { LOG(LOG_LEVEL_DEBUG, "Blocked Incoming Frame ID %d. " - "Invalidate is %d", frame_id, xrdp_invalidate); + "Invalidate is %d", frame_id, xrdp_invalidate); return 0; } } @@ -179,8 +179,8 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, { int cdata_value = cdata_bytes[i]; char *bmpdata_location = i == 0 - ? bmpdata - : bmpdata + cdata_bytes[i - 1] + (4 * i); + ? bmpdata + : bmpdata + cdata_bytes[i - 1] + (4 * i); bmpdata_location[0] = cdata_value; bmpdata_location[1] = cdata_value >> 8; bmpdata_location[2] = cdata_value >> 16; diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c index f6a9d39aa9..7b30609734 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c @@ -289,8 +289,8 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, reg_res.width = width; reg_res.height = height; if (tex_format == XH_YUV420 - || tex_format == XH_YUV444_V2_MV - || tex_format == XH_YUV444_V2_AUX) + || tex_format == XH_YUV444_V2_MV + || tex_format == XH_YUV444_V2_AUX) { reg_res.pitch = width; reg_res.bufferFormat = NV_ENC_BUFFER_FORMAT_NV12; diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index 8f812b0936..ddb2407d5b 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -3320,21 +3320,21 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) break; } /* do something with msg */ - LOG(LOG_LEVEL_INFO, "xrdp_mm_process_enc_done: " - "message back bytes " - "(comp_bytes1: %d, " - "comp_bytes2: %d)", - enc_done->comp_bytes1, - enc_done->comp_bytes2); + LOG(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: " + "message back bytes " + "(comp_bytes1: %d, " + "comp_bytes2: %d)", + enc_done->comp_bytes1, + enc_done->comp_bytes2); x = enc_done->rect.x; y = enc_done->rect.y; cx = enc_done->rect.cx; cy = enc_done->rect.cy; if (enc_done->comp_bytes1 > 0) { - LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_process_enc_done: x %d y %d cx %d cy %d " - "frame_id %d use_frame_acks %d", x, y, cx, cy, - enc_done->enc->frame_id, + LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_mm_process_enc_done: " + "x %d y %d cx %d cy %d frame_id %d use_frame_acks %d", + x, y, cx, cy, enc_done->enc->frame_id, self->wm->client_info->use_frame_acks); if (enc_done->flags & 1) /* gfx h264 */ { @@ -3345,23 +3345,28 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) #if XRDP_AVC444 xrdp_egfx_send_frame_start(self->egfx, enc_done->enc->frame_id, 0); - xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, + xrdp_egfx_send_wire_to_surface1(self->egfx, + self->egfx->surface_id, XR_RDPGFX_CODECID_AVC444V2, XR_PIXEL_FORMAT_XRGB_8888, &rect, - enc_done->comp_pad_data1 + enc_done->pad_bytes1, + enc_done->comp_pad_data1 + + enc_done->pad_bytes1, enc_done->comp_bytes1); if (enc_done->comp_bytes2 > 0) { - xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); + xrdp_egfx_send_frame_end(self->egfx, + enc_done->enc->frame_id); ++enc_done->enc->frame_id; xrdp_egfx_send_frame_start(self->egfx, enc_done->enc->frame_id, 0); - xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, + xrdp_egfx_send_wire_to_surface1(self->egfx, + self->egfx->surface_id, XR_RDPGFX_CODECID_AVC444V2, XR_PIXEL_FORMAT_XRGB_8888, &rect, - enc_done->comp_pad_data2 + enc_done->pad_bytes2, + enc_done->comp_pad_data2 + + enc_done->pad_bytes2, enc_done->comp_bytes2); } xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); @@ -3369,11 +3374,13 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) xrdp_egfx_send_frame_start(self->egfx, enc_done->enc->frame_id, 0); - xrdp_egfx_send_wire_to_surface1(self->egfx, self->egfx->surface_id, + xrdp_egfx_send_wire_to_surface1(self->egfx, + self->egfx->surface_id, XR_RDPGFX_CODECID_AVC420, XR_PIXEL_FORMAT_XRGB_8888, &rect, - enc_done->comp_pad_data1 + enc_done->pad_bytes1, + enc_done->comp_pad_data1 + + enc_done->pad_bytes1, enc_done->comp_bytes1); xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); From 9d35dfb7d4eeb000cf4efa2947e094f54f545400 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 25 Nov 2023 02:37:28 -0500 Subject: [PATCH 21/39] More style fixes --- xorgxrdp_helper/xorgxrdp_helper.c | 18 ++++++++++-------- xrdp/xrdp_mm.c | 12 +++++++----- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper.c b/xorgxrdp_helper/xorgxrdp_helper.c index 4711914143..157d0e6b1f 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.c +++ b/xorgxrdp_helper/xorgxrdp_helper.c @@ -179,14 +179,16 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, { int cdata_value = cdata_bytes[i]; char *bmpdata_location = i == 0 - ? bmpdata - : bmpdata + cdata_bytes[i - 1] + (4 * i); + ? bmpdata + : bmpdata + + cdata_bytes[i - 1] + + (4 * i); bmpdata_location[0] = cdata_value; bmpdata_location[1] = cdata_value >> 8; bmpdata_location[2] = cdata_value >> 16; bmpdata_location[3] = cdata_value >> 24; LOG(LOG_LEVEL_INFO, - "cdata_bytes %d", cdata_value); + "cdata_bytes %d", cdata_value); } } } @@ -257,14 +259,14 @@ xorg_process_message_61(struct xorgxrdp_info *xi, struct stream *s) { // Let it through. We are no longer resizing. LOG(LOG_LEVEL_DEBUG, "Invalidate received and processing " - "frame ID 1. Unblocking encoder. " - "Invalidate is %d.", xrdp_invalidate); + "frame ID 1. Unblocking encoder. " + "Invalidate is %d.", xrdp_invalidate); xi->resizing = 0; } else { LOG(LOG_LEVEL_DEBUG, "Blocked Incoming Frame ID %d. " - "Invalidate is %d", frame_id, xrdp_invalidate); + "Invalidate is %d", frame_id, xrdp_invalidate); return 0; } } @@ -331,8 +333,8 @@ xorg_process_message_61(struct xorgxrdp_info *xi, struct stream *s) { int cdata_value = cdata_bytes[i]; char *bmpdata_location = i == 0 - ? bmpdata - : bmpdata + cdata_bytes[i - 1] + (4 * i); + ? bmpdata + : bmpdata + cdata_bytes[i - 1] + (4 * i); bmpdata_location[0] = cdata_value; bmpdata_location[1] = cdata_value >> 8; bmpdata_location[2] = cdata_value >> 16; diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index ddb2407d5b..849b70e577 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -3351,7 +3351,7 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) XR_PIXEL_FORMAT_XRGB_8888, &rect, enc_done->comp_pad_data1 - + enc_done->pad_bytes1, + + enc_done->pad_bytes1, enc_done->comp_bytes1); if (enc_done->comp_bytes2 > 0) { @@ -3366,7 +3366,7 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) XR_PIXEL_FORMAT_XRGB_8888, &rect, enc_done->comp_pad_data2 - + enc_done->pad_bytes2, + + enc_done->pad_bytes2, enc_done->comp_bytes2); } xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); @@ -3380,7 +3380,7 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) XR_PIXEL_FORMAT_XRGB_8888, &rect, enc_done->comp_pad_data1 - + enc_done->pad_bytes1, + + enc_done->pad_bytes1, enc_done->comp_bytes1); xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); @@ -3390,9 +3390,11 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) { xrdp_egfx_send_frame_start(self->egfx, enc_done->enc->frame_id, 0); - xrdp_egfx_send_wire_to_surface2(self->egfx, self->egfx->surface_id, 9, 1, + xrdp_egfx_send_wire_to_surface2(self->egfx, + self->egfx->surface_id, 9, 1, XR_PIXEL_FORMAT_XRGB_8888, - enc_done->comp_pad_data1 + enc_done->pad_bytes1, + enc_done->comp_pad_data1 + + enc_done->pad_bytes1, enc_done->comp_bytes1); xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); } From 1411683796f15b97db693b62e6360a2e7a10e169 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 25 Nov 2023 02:39:00 -0500 Subject: [PATCH 22/39] More fixes. --- xorgxrdp_helper/xorgxrdp_helper.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper.c b/xorgxrdp_helper/xorgxrdp_helper.c index 157d0e6b1f..25cabbc48f 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.c +++ b/xorgxrdp_helper/xorgxrdp_helper.c @@ -181,8 +181,8 @@ xorg_process_message_64(struct xorgxrdp_info *xi, struct trans *trans, char *bmpdata_location = i == 0 ? bmpdata : bmpdata - + cdata_bytes[i - 1] - + (4 * i); + + cdata_bytes[i - 1] + + (4 * i); bmpdata_location[0] = cdata_value; bmpdata_location[1] = cdata_value >> 8; bmpdata_location[2] = cdata_value >> 16; From 662ed7d383161cbd65d41fe4f4f9926d002887e5 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 25 Nov 2023 02:54:20 -0500 Subject: [PATCH 23/39] Fixing text corruption by switching to normal 420 view. --- xorgxrdp_helper/xorgxrdp_helper.h | 5 ++++- xorgxrdp_helper/xorgxrdp_helper_shaders.c | 8 +++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper.h b/xorgxrdp_helper/xorgxrdp_helper.h index b7ef6c7042..adf4b2ff71 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.h +++ b/xorgxrdp_helper/xorgxrdp_helper.h @@ -22,7 +22,10 @@ #define XH_YUV420 1 #define XH_YUV422 2 #define XH_YUV444 3 -#define XH_YUV444_V2_MV 4 + +// This should 4, but 4 doesn't work. Fix it! +#define XH_YUV444_V2_MV 1 + #define XH_YUV444_V2_AUX 6 #define XH_BT601 0 diff --git a/xorgxrdp_helper/xorgxrdp_helper_shaders.c b/xorgxrdp_helper/xorgxrdp_helper_shaders.c index 167e0ed958..9f6becfa7b 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_shaders.c +++ b/xorgxrdp_helper/xorgxrdp_helper_shaders.c @@ -168,9 +168,10 @@ MAIN VIEW - NV12 TODO: This appears to be identical to g_fs_rgb_to_yuv420. Keeping this now because this is a dev prototype, but we might want to remove this in the future. -*/ -/* + + TODO: These don't work. Fix it! +*/ static const GLchar g_fs_rgb_to_yuv420_mv[] = "\ #version 460 core\n\ uniform sampler2D tex;\n\ @@ -220,8 +221,8 @@ void main(void)\n\ }\n\ }\n\ }\n"; -*/ +/* static const GLchar g_fs_rgb_to_yuv420_mv[] = "\ #version 460 core\n\ uniform sampler2D tex;\n\ @@ -263,6 +264,7 @@ void main()\n\ }\n\ FragColor = vec4(pix.r, 0.0, 0.0, 1.0);\n\ }\n"; +*/ /* AUXILIARY VIEW - NV12 From d32cadfd750c0faa32d84132aa58a9724830abb9 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 25 Nov 2023 03:16:44 -0500 Subject: [PATCH 24/39] Misc fixes --- xorgxrdp_helper/xorgxrdp_helper.h | 2 +- xorgxrdp_helper/xorgxrdp_helper_nvenc.c | 2 +- xorgxrdp_helper/xorgxrdp_helper_shaders.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/xorgxrdp_helper/xorgxrdp_helper.h b/xorgxrdp_helper/xorgxrdp_helper.h index adf4b2ff71..7bd1dfe594 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.h +++ b/xorgxrdp_helper/xorgxrdp_helper.h @@ -23,7 +23,7 @@ #define XH_YUV422 2 #define XH_YUV444 3 -// This should 4, but 4 doesn't work. Fix it! +// This should 4, but 4 doesn't work (graphical glitches). Fix it! #define XH_YUV444_V2_MV 1 #define XH_YUV444_V2_AUX 6 diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c index 7b30609734..b85fbdab9f 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c @@ -145,7 +145,7 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, createEncodeParams.version = NV_ENC_INITIALIZE_PARAMS_VER; createEncodeParams.encodeGUID = NV_ENC_CODEC_H264_GUID; createEncodeParams.presetGUID = NV_ENC_PRESET_P7_GUID; - createEncodeParams.tuningInfo = NV_ENC_TUNING_INFO_LOW_LATENCY; + createEncodeParams.tuningInfo = NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY; createEncodeParams.encodeWidth = width; createEncodeParams.encodeHeight = height; createEncodeParams.darWidth = width; diff --git a/xorgxrdp_helper/xorgxrdp_helper_shaders.c b/xorgxrdp_helper/xorgxrdp_helper_shaders.c index 9f6becfa7b..468d479fdf 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_shaders.c +++ b/xorgxrdp_helper/xorgxrdp_helper_shaders.c @@ -203,7 +203,7 @@ void main(void)\n\ pix += texture(tex, vec2(x + 1.0, y) / tex_size);\n\ pix += texture(tex, vec2(x, y + 1.0) / tex_size);\n\ pix += texture(tex, vec2(x + 1.0, y + 1.0) / tex_size);\n\ - pix *= 0.25;\n\ + pix /= 4;\n\ pix.a = 1.0;\n\ pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ FragColor = pix;\n\ @@ -214,7 +214,7 @@ void main(void)\n\ pix += texture(tex, vec2(x + 1.0, y) / tex_size);\n\ pix += texture(tex, vec2(x, y + 1.0) / tex_size);\n\ pix += texture(tex, vec2(x + 1.0, y + 1.0) / tex_size);\n\ - pix *= 0.25;\n\ + pix /= 4;\n\ pix.a = 1.0;\n\ pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ FragColor = pix;\n\ From e5d243e5af9403f610de43173e255f89a07537b7 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 25 Nov 2023 23:17:01 -0500 Subject: [PATCH 25/39] Improve quality and performance. --- sesman/sesman.ini.in | 2 +- xorgxrdp_helper/xorgxrdp_helper.h | 2 +- xorgxrdp_helper/xorgxrdp_helper_shaders.c | 68 ++--------------------- xrdp/xrdp_encoder.c | 2 +- 4 files changed, 7 insertions(+), 67 deletions(-) diff --git a/sesman/sesman.ini.in b/sesman/sesman.ini.in index 0d89bbc54e..5ec9c80748 100644 --- a/sesman/sesman.ini.in +++ b/sesman/sesman.ini.in @@ -208,6 +208,6 @@ EnableSyslog=true PULSE_SCRIPT=@sesmansysconfdir@/pulse/default.pa XRDP_USE_HELPER=1 XRDP_NVENC_RATE_CONTROL_MODE=NV_ENC_PARAMS_RC_CONSTQP -XRDP_NVENC_QP=22 +XRDP_NVENC_QP=19 @ldlibrarypath@ @libvadriverspath@ diff --git a/xorgxrdp_helper/xorgxrdp_helper.h b/xorgxrdp_helper/xorgxrdp_helper.h index 7bd1dfe594..a062a4ee88 100644 --- a/xorgxrdp_helper/xorgxrdp_helper.h +++ b/xorgxrdp_helper/xorgxrdp_helper.h @@ -24,7 +24,7 @@ #define XH_YUV444 3 // This should 4, but 4 doesn't work (graphical glitches). Fix it! -#define XH_YUV444_V2_MV 1 +#define XH_YUV444_V2_MV 4 #define XH_YUV444_V2_AUX 6 diff --git a/xorgxrdp_helper/xorgxrdp_helper_shaders.c b/xorgxrdp_helper/xorgxrdp_helper_shaders.c index 468d479fdf..0883e95e23 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_shaders.c +++ b/xorgxrdp_helper/xorgxrdp_helper_shaders.c @@ -164,65 +164,6 @@ MAIN VIEW - NV12 ... 0E 0E 2E 2E 4E 4E 6E 6E 8E 8E AE AE CE CE EE EE */ -/* - TODO: This appears to be identical to g_fs_rgb_to_yuv420. - Keeping this now because this is a dev prototype, but we might - want to remove this in the future. - - - TODO: These don't work. Fix it! -*/ -static const GLchar g_fs_rgb_to_yuv420_mv[] = "\ -#version 460 core\n\ -uniform sampler2D tex;\n\ -uniform vec2 tex_size;\n\ -uniform vec4 ymath;\n\ -uniform vec4 umath;\n\ -uniform vec4 vmath;\n\ -out vec4 FragColor;\n\ -void main(void)\n\ -{\n\ - vec4 pix;\n\ - float x;\n\ - float y;\n\ - x = gl_FragCoord.x;\n\ - y = gl_FragCoord.y;\n\ - if (y < tex_size.y)\n\ - {\n\ - pix = texture(tex, vec2(x, y) / tex_size);\n\ - pix.a = 1.0;\n\ - pix = vec4(clamp(dot(ymath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - FragColor = pix;\n\ - }\n\ - else\n\ - {\n\ - y = floor(y - tex_size.y) * 2.0 + 0.5;\n\ - if (mod(x, 2.0) < 1.0)\n\ - {\n\ - pix = texture(tex, vec2(x, y) / tex_size);\n\ - pix += texture(tex, vec2(x + 1.0, y) / tex_size);\n\ - pix += texture(tex, vec2(x, y + 1.0) / tex_size);\n\ - pix += texture(tex, vec2(x + 1.0, y + 1.0) / tex_size);\n\ - pix /= 4;\n\ - pix.a = 1.0;\n\ - pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - FragColor = pix;\n\ - }\n\ - else\n\ - {\n\ - pix = texture(tex, vec2(x, y) / tex_size);\n\ - pix += texture(tex, vec2(x + 1.0, y) / tex_size);\n\ - pix += texture(tex, vec2(x, y + 1.0) / tex_size);\n\ - pix += texture(tex, vec2(x + 1.0, y + 1.0) / tex_size);\n\ - pix /= 4;\n\ - pix.a = 1.0;\n\ - pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ - FragColor = pix;\n\ - }\n\ - }\n\ -}\n"; - -/* static const GLchar g_fs_rgb_to_yuv420_mv[] = "\ #version 460 core\n\ uniform sampler2D tex;\n\ @@ -247,10 +188,10 @@ void main()\n\ y = floor(y - tex_size.y) * 2.0 + 0.5;\n\ float xOffset = mod(x, 2.0);\n\ float antiAliasMod = xOffset == 0 ? 1.0 : -1.0;\n\ - pix = texture(tex, vec2(x - xOffset, y) / tex_size);\n\ - pix += texture(tex, vec2(x - xOffset + antiAliasMod, y) / tex_size);\n\ - pix += texture(tex, vec2(x - xOffset, y + 1.0) / tex_size);\n\ - pix += texture(tex, vec2(x - xOffset + antiAliasMod, y + 1.0) / tex_size);\n\ + pix = texture(tex, vec2(x, y) / tex_size);\n\ + pix += texture(tex, vec2(x + antiAliasMod, y) / tex_size);\n\ + pix += texture(tex, vec2(x, y + 1.0) / tex_size);\n\ + pix += texture(tex, vec2(x + antiAliasMod, y + 1.0) / tex_size);\n\ pix /= 4.0;\n\ pix.a = 1.0;\n\ if (xOffset < 1.0)\n\ @@ -264,7 +205,6 @@ void main()\n\ }\n\ FragColor = vec4(pix.r, 0.0, 0.0, 1.0);\n\ }\n"; -*/ /* AUXILIARY VIEW - NV12 diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 57a572caa8..c06d6fa59b 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -577,7 +577,7 @@ build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, int x, y, cx, cy; int location; struct enc_rect rect; - const uint8_t qp = 22; // Default set by Microsoft. + const uint8_t qp = 19; // Default set by Microsoft. const uint8_t r = 0; // Required to be 0. const uint8_t p = 0; // Progressively encoded flag. int qpVal = 0; From cdc15323a56dc021e78e9fb231ae1d1023d331dc Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 25 Nov 2023 23:19:54 -0500 Subject: [PATCH 26/39] Update comment --- xrdp/xrdp_encoder.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index c06d6fa59b..2a44ec1f9a 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -577,7 +577,11 @@ build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, int x, y, cx, cy; int location; struct enc_rect rect; - const uint8_t qp = 19; // Default set by Microsoft. + /* + Microsoft default QP is 22, but emperical + testing shows 19 is slightly better. + */ + const uint8_t qp = 19; const uint8_t r = 0; // Required to be 0. const uint8_t p = 0; // Progressively encoded flag. int qpVal = 0; From 185a1f488bb5b993d33c920dcfdc879b8cca5899 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 25 Nov 2023 23:20:37 -0500 Subject: [PATCH 27/39] Re-add original 420 shader --- xorgxrdp_helper/xorgxrdp_helper_shaders.c | 61 +++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/xorgxrdp_helper/xorgxrdp_helper_shaders.c b/xorgxrdp_helper/xorgxrdp_helper_shaders.c index 0883e95e23..430a7dbbf2 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_shaders.c +++ b/xorgxrdp_helper/xorgxrdp_helper_shaders.c @@ -164,6 +164,67 @@ MAIN VIEW - NV12 ... 0E 0E 2E 2E 4E 4E 6E 6E 8E 8E AE AE CE CE EE EE */ +/* + TODO: This appears to be identical to g_fs_rgb_to_yuv420. + Keeping this now because this is a dev prototype, but we might + want to remove this in the future. + + + TODO: These don't work. Fix it! +*/ + +/* +static const GLchar g_fs_rgb_to_yuv420_mv[] = "\ +#version 460 core\n\ +uniform sampler2D tex;\n\ +uniform vec2 tex_size;\n\ +uniform vec4 ymath;\n\ +uniform vec4 umath;\n\ +uniform vec4 vmath;\n\ +out vec4 FragColor;\n\ +void main(void)\n\ +{\n\ + vec4 pix;\n\ + float x;\n\ + float y;\n\ + x = gl_FragCoord.x;\n\ + y = gl_FragCoord.y;\n\ + if (y < tex_size.y)\n\ + {\n\ + pix = texture(tex, vec2(x, y) / tex_size);\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(ymath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + y = floor(y - tex_size.y) * 2.0 + 0.5;\n\ + if (mod(x, 2.0) < 1.0)\n\ + {\n\ + pix = texture(tex, vec2(x, y) / tex_size);\n\ + pix += texture(tex, vec2(x + 1.0, y) / tex_size);\n\ + pix += texture(tex, vec2(x, y + 1.0) / tex_size);\n\ + pix += texture(tex, vec2(x + 1.0, y + 1.0) / tex_size);\n\ + pix /= 4.0;\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(umath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + FragColor = pix;\n\ + }\n\ + else\n\ + {\n\ + pix = texture(tex, vec2(x, y) / tex_size);\n\ + pix += texture(tex, vec2(x + 1.0, y) / tex_size);\n\ + pix += texture(tex, vec2(x, y + 1.0) / tex_size);\n\ + pix += texture(tex, vec2(x + 1.0, y + 1.0) / tex_size);\n\ + pix /= 4.0;\n\ + pix.a = 1.0;\n\ + pix = vec4(clamp(dot(vmath, pix), 0.0, 1.0), 0.0, 0.0, 1.0);\n\ + FragColor = pix;\n\ + }\n\ + }\n\ +}\n"; +*/ + static const GLchar g_fs_rgb_to_yuv420_mv[] = "\ #version 460 core\n\ uniform sampler2D tex;\n\ From 1e45d9fb24316f893cafe0e4aeaa8dac694cc8b1 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 25 Nov 2023 23:33:28 -0500 Subject: [PATCH 28/39] Silencing log --- xrdp/xrdp_encoder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 2a44ec1f9a..6bf53070e5 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -784,7 +784,7 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) g_free(out_data); return 0; } - LOG(LOG_LEVEL_INFO, + LOG(LOG_LEVEL_DEBUG, "process_enc_h264: Aux frame already compressed and size is %d", cbytes); out_data_bytes1 = cbytes; g_memcpy(s->p, enc->data + out_data_bytes + 8, out_data_bytes1); From 8ca18063434a18c6f91fc085c65b4ad53dfeba0a Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sun, 26 Nov 2023 02:46:17 -0500 Subject: [PATCH 29/39] Refactor encoder for common function for pre-encoded frames --- xrdp/xrdp_encoder.c | 47 +++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 6bf53070e5..5d9207a0bf 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -617,6 +617,26 @@ build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, return comp_bytes_pre; } +static int load_pre_encoded_frame(const char* data, struct stream *s, + int size_offset, int frame_data_offset, + int *frame_size_bytes) +{ + /* already compressed */ + uint8_t *ud = (uint8_t *)(data + size_offset); + int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24); + if ((cbytes < 1) || (cbytes > *frame_size_bytes)) + { + LOG(LOG_LEVEL_DEBUG, "process_enc_h264: bad h264 bytes %d", cbytes); + return 1; + } + LOG(LOG_LEVEL_DEBUG, + "process_enc_h264: Frame already compressed and size is %d", + cbytes); + *frame_size_bytes = cbytes; + g_memcpy(s->p, data + frame_data_offset, *frame_size_bytes); + return 0; +} + static XRDP_ENC_DATA_DONE * build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) { @@ -719,22 +739,13 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) enc_done_flags = 0; } error = 0; - if (enc->flags & 1) + if (enc->flags & ENCODE_COMPLETE) { - /* already compressed */ - uint8_t *ud = (uint8_t *) (enc->data); - int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24); - if ((cbytes < 1) || (cbytes > out_data_bytes)) + if (load_pre_encoded_frame(enc->data, s, 0, 4, &out_data_bytes)) { - LOG(LOG_LEVEL_DEBUG, "process_enc_h264: bad h264 bytes %d", cbytes); g_free(out_data); return 0; } - LOG(LOG_LEVEL_DEBUG, - "process_enc_h264: Main frame already compressed and size is %d", - cbytes); - out_data_bytes = cbytes; - g_memcpy(s->p, enc->data + 4, out_data_bytes); } #if defined(XRDP_X264) else @@ -773,21 +784,15 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) comp_bytes_pre1 = build_rfx_avc420_metablock(s, rrects, rcount, scr_width, scr_height); out_data_bytes1 = OUT_DATA_BYTES_DEFAULT_SIZE; - if (enc->flags & 1) + if (enc->flags & ENCODE_COMPLETE) { - /* already compressed */ - uint8_t *ud = (uint8_t *) (enc->data + out_data_bytes + 4); - int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24); - if ((cbytes < 1) || (cbytes > out_data_bytes1)) + if (load_pre_encoded_frame(enc->data, s, + out_data_bytes + 4, out_data_bytes + 8, + &out_data_bytes1)) { - LOG(LOG_LEVEL_INFO, "process_enc_h264: bad h264 bytes %d", cbytes); g_free(out_data); return 0; } - LOG(LOG_LEVEL_DEBUG, - "process_enc_h264: Aux frame already compressed and size is %d", cbytes); - out_data_bytes1 = cbytes; - g_memcpy(s->p, enc->data + out_data_bytes + 8, out_data_bytes1); } #if defined(XRDP_X264) else From ad58f1970f6307334607bb475daf57bf02ab4d70 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sun, 26 Nov 2023 02:50:15 -0500 Subject: [PATCH 30/39] Fixing code formatting --- xrdp/xrdp_encoder.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 5d9207a0bf..3a8fc95688 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -578,7 +578,7 @@ build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, int location; struct enc_rect rect; /* - Microsoft default QP is 22, but emperical + Microsoft default QP is 22, but empirical testing shows 19 is slightly better. */ const uint8_t qp = 19; @@ -617,7 +617,7 @@ build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, return comp_bytes_pre; } -static int load_pre_encoded_frame(const char* data, struct stream *s, +static int load_pre_encoded_frame(const char *data, struct stream *s, int size_offset, int frame_data_offset, int *frame_size_bytes) { From 4902c01b0d7a071f1c320090fefa92870178a419 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sun, 26 Nov 2023 22:38:26 -0500 Subject: [PATCH 31/39] Clarifying refactor --- xrdp/xrdp_encoder.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 3a8fc95688..11c3e396a5 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -617,12 +617,13 @@ build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, return comp_bytes_pre; } -static int load_pre_encoded_frame(const char *data, struct stream *s, - int size_offset, int frame_data_offset, +static int load_pre_encoded_frame(const char *source_data, + char *dest_data, int size_offset, + int frame_data_offset, int *frame_size_bytes) { /* already compressed */ - uint8_t *ud = (uint8_t *)(data + size_offset); + uint8_t *ud = (uint8_t *)(source_data + size_offset); int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24); if ((cbytes < 1) || (cbytes > *frame_size_bytes)) { @@ -633,7 +634,7 @@ static int load_pre_encoded_frame(const char *data, struct stream *s, "process_enc_h264: Frame already compressed and size is %d", cbytes); *frame_size_bytes = cbytes; - g_memcpy(s->p, data + frame_data_offset, *frame_size_bytes); + g_memcpy(dest_data, source_data + frame_data_offset, *frame_size_bytes); return 0; } @@ -741,7 +742,7 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) error = 0; if (enc->flags & ENCODE_COMPLETE) { - if (load_pre_encoded_frame(enc->data, s, 0, 4, &out_data_bytes)) + if (load_pre_encoded_frame(enc->data, s->p, 0, 4, &out_data_bytes)) { g_free(out_data); return 0; @@ -773,12 +774,6 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) n_save_data(s->p, out_data_bytes, enc->width, enc->height); #endif s->p += out_data_bytes; - - uint8_t LC = 0b00; - uint32_t bitstream = - ((uint32_t)(comp_bytes_pre + out_data_bytes) & 0x3FFFFFFFUL) - | ((LC & 0x03UL) << 30UL); - /* chroma 444 */ /* RFX_AVC420_METABLOCK */ comp_bytes_pre1 = build_rfx_avc420_metablock(s, rrects, rcount, @@ -786,7 +781,7 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) out_data_bytes1 = OUT_DATA_BYTES_DEFAULT_SIZE; if (enc->flags & ENCODE_COMPLETE) { - if (load_pre_encoded_frame(enc->data, s, + if (load_pre_encoded_frame(enc->data, s->p, out_data_bytes + 4, out_data_bytes + 8, &out_data_bytes1)) { @@ -819,7 +814,13 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) s->p += out_data_bytes1; s_push_layer(s, sec_hdr, 0); s_pop_layer(s, mcs_hdr); + + uint8_t LC = 0b00; + uint32_t bitstream = + ((uint32_t)(comp_bytes_pre + out_data_bytes) & 0x3FFFFFFFUL) + | ((LC & 0x03UL) << 30UL); out_uint32_le(s, bitstream); + s_pop_layer(s, sec_hdr); s->end = s->p; From 7bf5ab95999fab875fa252b08ad8b9e5daf4be8a Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sun, 26 Nov 2023 23:40:59 -0500 Subject: [PATCH 32/39] More refactoring --- xrdp/xrdp_encoder.c | 66 ++++++++++++++++++++++----------------------- xrdp/xrdp_mm.c | 4 +-- 2 files changed, 34 insertions(+), 36 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 11c3e396a5..817a3d2f5e 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -566,31 +566,14 @@ static int n_save_data(const char *data, int data_size, int width, int height) } #endif -#if defined(XRDP_X264) - -/*****************************************************************************/ -static int -build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, - int width, int height) +static void +build_stream_rectangles(struct stream *s, short *rrects, + int rcount, int width, int height) { - int index; + int index, location; int x, y, cx, cy; - int location; struct enc_rect rect; - /* - Microsoft default QP is 22, but empirical - testing shows 19 is slightly better. - */ - const uint8_t qp = 19; - const uint8_t r = 0; // Required to be 0. - const uint8_t p = 0; // Progressively encoded flag. - int qpVal = 0; - qpVal |= qp & 0x3F; - qpVal |= (r & 1) << 6; - qpVal |= (p & 1) << 7; - - out_uint32_le(s, rcount); /* numRegionRects */ - for (index = 0; index < rcount; index++) + for (index = 0; index < rcount; ++index) { location = index * 4; x = rrects[location + 0]; @@ -607,7 +590,29 @@ build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, out_uint16_le(s, rect.x2); out_uint16_le(s, rect.y2); } - for (index = 0; index < rcount; index++) +} + +#if defined(XRDP_X264) + +/*****************************************************************************/ +static int +build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, + int width, int height) +{ + /* + Microsoft default QP is 22, but empirical + testing shows 19 is slightly better. + */ + const uint8_t qp = 19; + const uint8_t r = 0; // Required to be 0. + const uint8_t p = 0; // Progressively encoded flag. + int qpVal = 0; + qpVal |= qp & 0x3F; + qpVal |= (r & 1) << 6; + qpVal |= (p & 1) << 7; + out_uint32_le(s, rcount); /* numRegionRects */ + build_stream_rectangles(s, rrects, rcount, width, height); + for (int index = 0; index < rcount; ++index) { // 2.2.4.4.2 RDPGFX_AVC420_QUANT_QUALITY out_uint8(s, qpVal); /* qp */ @@ -717,6 +722,9 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) } else { + // Header for fastpath??? + // Appears to be for sending H264 over standard bitmap + // commands... Looking for references. out_uint32_le(s, 0); /* flags */ out_uint32_le(s, 0); /* session id */ out_uint16_le(s, enc->width); /* src_width */ @@ -724,17 +732,7 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) out_uint16_le(s, enc->width); /* dst_width */ out_uint16_le(s, enc->height); /* dst_height */ out_uint16_le(s, rcount); - for (index = 0; index < rcount; index++) - { - x = rrects[index * 4 + 0]; - y = rrects[index * 4 + 1]; - cx = rrects[index * 4 + 2]; - cy = rrects[index * 4 + 3]; - out_uint16_le(s, x); - out_uint16_le(s, y); - out_uint16_le(s, cx); - out_uint16_le(s, cy); - } + build_stream_rectangles(s, rrects, rcount, width, height); s_push_layer(s, iso_hdr, 4); comp_bytes_pre = 4 + 4 + 2 + 2 + 2 + 2 + 2 + rcount * 8 + 4; enc_done_flags = 0; diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index 849b70e577..d9ace34844 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -3336,7 +3336,7 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) "x %d y %d cx %d cy %d frame_id %d use_frame_acks %d", x, y, cx, cy, enc_done->enc->frame_id, self->wm->client_info->use_frame_acks); - if (enc_done->flags & 1) /* gfx h264 */ + if (enc_done->flags & ENCODE_COMPLETE) /* gfx h264 */ { rect.x1 = x; rect.y1 = y; @@ -3386,7 +3386,7 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) xrdp_egfx_send_frame_end(self->egfx, enc_done->enc->frame_id); #endif } - else if (enc_done->flags & 2) /* gfx progressive rfx */ + else if (enc_done->flags & GFX_PROGRESSIVE_RFX) /* gfx progressive rfx */ { xrdp_egfx_send_frame_start(self->egfx, enc_done->enc->frame_id, 0); From 1ac962a52efdad8a28648cbbb505031751f1cd94 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Mon, 27 Nov 2023 10:04:21 -0500 Subject: [PATCH 33/39] Fixing build break --- xrdp/xrdp_encoder.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 817a3d2f5e..4ec5cd2d42 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -652,10 +652,6 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) } int index; - int x; - int y; - int cx; - int cy; int rcount; short *rrects; int error; @@ -732,7 +728,7 @@ build_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) out_uint16_le(s, enc->width); /* dst_width */ out_uint16_le(s, enc->height); /* dst_height */ out_uint16_le(s, rcount); - build_stream_rectangles(s, rrects, rcount, width, height); + build_stream_rectangles(s, rrects, rcount, enc->width, enc->height); s_push_layer(s, iso_hdr, 4); comp_bytes_pre = 4 + 4 + 2 + 2 + 2 + 2 + 2 + rcount * 8 + 4; enc_done_flags = 0; From ebe8b0a3d638de37a9b08ea351cdef2d48c0973c Mon Sep 17 00:00:00 2001 From: Nexarian Date: Mon, 27 Nov 2023 10:29:20 -0500 Subject: [PATCH 34/39] Change back to qp 22 --- sesman/sesman.ini.in | 2 +- xrdp/xrdp_encoder.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sesman/sesman.ini.in b/sesman/sesman.ini.in index 5ec9c80748..0d89bbc54e 100644 --- a/sesman/sesman.ini.in +++ b/sesman/sesman.ini.in @@ -208,6 +208,6 @@ EnableSyslog=true PULSE_SCRIPT=@sesmansysconfdir@/pulse/default.pa XRDP_USE_HELPER=1 XRDP_NVENC_RATE_CONTROL_MODE=NV_ENC_PARAMS_RC_CONSTQP -XRDP_NVENC_QP=19 +XRDP_NVENC_QP=22 @ldlibrarypath@ @libvadriverspath@ diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 4ec5cd2d42..f66f255188 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -603,7 +603,7 @@ build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, Microsoft default QP is 22, but empirical testing shows 19 is slightly better. */ - const uint8_t qp = 19; + const uint8_t qp = 22; const uint8_t r = 0; // Required to be 0. const uint8_t p = 0; // Progressively encoded flag. int qpVal = 0; From 9340fe8ce8a8b8b80ed5cf51ed00173e83fcce2e Mon Sep 17 00:00:00 2001 From: Nexarian Date: Mon, 27 Nov 2023 10:54:00 -0500 Subject: [PATCH 35/39] Fix encoder build --- xrdp/xrdp_encoder.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index f66f255188..e192692ea5 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -566,6 +566,8 @@ static int n_save_data(const char *data, int data_size, int width, int height) } #endif +#if defined(XRDP_X264) + static void build_stream_rectangles(struct stream *s, short *rrects, int rcount, int width, int height) @@ -592,8 +594,6 @@ build_stream_rectangles(struct stream *s, short *rrects, } } -#if defined(XRDP_X264) - /*****************************************************************************/ static int build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, From 901a3d03c3f79785a7f814a227950d226d072ff8 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Mon, 27 Nov 2023 21:02:33 -0500 Subject: [PATCH 36/39] Fixing permissions --- instfiles/xrdp.service.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/instfiles/xrdp.service.in b/instfiles/xrdp.service.in index 814d4de590..faacd16d44 100644 --- a/instfiles/xrdp.service.in +++ b/instfiles/xrdp.service.in @@ -10,7 +10,9 @@ EnvironmentFile=-@sysconfdir@/sysconfig/xrdp EnvironmentFile=-@sysconfdir@/default/xrdp ExecStart=@sbindir@/xrdp $XRDP_OPTIONS --nodaemon SystemCallArchitectures=native -SystemCallFilter=@basic-io @file-system @io-event @ipc @network-io @process @signal ioctl madvise sysinfo uname +SystemCallFilter=@basic-io @file-system @io-event @ipc @network-io @process +SystemCallFilter=@signal ioctl madvise sysinfo uname @system-service brk +SystemCallErrorNumber=EPERM [Install] WantedBy=multi-user.target From 4c09100d759ea120ad470045b755fdaa4e55602e Mon Sep 17 00:00:00 2001 From: Nexarian Date: Mon, 27 Nov 2023 21:08:40 -0500 Subject: [PATCH 37/39] Organize calls --- instfiles/xrdp.service.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instfiles/xrdp.service.in b/instfiles/xrdp.service.in index faacd16d44..de2e2ddcb3 100644 --- a/instfiles/xrdp.service.in +++ b/instfiles/xrdp.service.in @@ -11,7 +11,7 @@ EnvironmentFile=-@sysconfdir@/default/xrdp ExecStart=@sbindir@/xrdp $XRDP_OPTIONS --nodaemon SystemCallArchitectures=native SystemCallFilter=@basic-io @file-system @io-event @ipc @network-io @process -SystemCallFilter=@signal ioctl madvise sysinfo uname @system-service brk +SystemCallFilter=@signal @system-service brk ioctl madvise sysinfo uname SystemCallErrorNumber=EPERM [Install] From 4235e6f07dfd5f9fccb2de16cdddb6873c646f33 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 16 Dec 2023 00:59:19 -0500 Subject: [PATCH 38/39] Fixing memory leak. --- xorgxrdp_helper/xorgxrdp_helper_nvenc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c index b85fbdab9f..8b5f4ffd4c 100644 --- a/xorgxrdp_helper/xorgxrdp_helper_nvenc.c +++ b/xorgxrdp_helper/xorgxrdp_helper_nvenc.c @@ -138,6 +138,7 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, nv_error, lei->enc); if (nv_error != NV_ENC_SUCCESS) { + g_free(lei); return 1; } @@ -165,6 +166,7 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, LOG(LOG_LEVEL_INFO, "nvEncGetEncodePresetConfig rv %d", nv_error); if (nv_error != NV_ENC_SUCCESS) { + g_free(lei); return 1; } @@ -276,6 +278,7 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, LOG(LOG_LEVEL_INFO, "nvEncInitializeEncoder rv %d", nv_error); if (nv_error != NV_ENC_SUCCESS) { + g_free(lei); return 1; } @@ -306,6 +309,7 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, LOG(LOG_LEVEL_INFO, "nvEncRegisterResource rv %d", nv_error); if (nv_error != NV_ENC_SUCCESS) { + g_free(lei); return 1; } @@ -316,6 +320,7 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, LOG(LOG_LEVEL_INFO, "nvEncMapInputResource rv %d", nv_error); if (nv_error != NV_ENC_SUCCESS) { + g_free(lei); return 1; } @@ -326,6 +331,7 @@ xorgxrdp_helper_nvenc_create_encoder(int width, int height, int tex, LOG(LOG_LEVEL_INFO, "nvEncCreateBitstreamBuffer rv %d", nv_error); if (nv_error != NV_ENC_SUCCESS) { + g_free(lei); return 1; } From 483e7467c29c9d63462d57ec467d3dc7b4481061 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Thu, 21 Dec 2023 23:43:38 -0500 Subject: [PATCH 39/39] Fixing build break --- xrdp/xrdp_painter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/xrdp/xrdp_painter.c b/xrdp/xrdp_painter.c index 91db83416b..ec50e8a6cd 100644 --- a/xrdp/xrdp_painter.c +++ b/xrdp/xrdp_painter.c @@ -133,6 +133,7 @@ xrdp_painter_send_dirty(struct xrdp_painter *self) } LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_painter_send_dirty:" " x %d y %d cx %d cy %d", + rect.left, rect.top, cx, cy); libxrdp_send_bitmap(self->session, cx, cy, bpp, ldata, rect.left, rect.top, cx, cy); g_free(ldata);