add support for nvenc and accel_assist

.gitignore

@@ -63,3 +63,4 @@ xrdp/xrdp.ini
 xrdp_configure_options.h
 xrdpapi/xrdp-xrdpapi-simple
 .vscode/*
+xrdp_accel_assist/xrdp-accel-assist

Makefile.am

@@ -70,7 +70,8 @@ SUBDIRS = \
   $(XRDPVRDIR) \
   $(ULALACADIR) \
   tests \
-  tools
+  tools \
+  xrdp_accel_assist
 
 distclean-local:
 	-rm -f xrdp_configure_options.h

common/Makefile.am

@@ -30,6 +30,10 @@ AM_CPPFLAGS = \
   -DXRDP_PID_PATH=\"${localstatedir}/run\" \
   -DXRDP_LOG_PATH=\"${localstatedir}/log\"
 
+if XRDP_NVENC
+AM_CPPFLAGS += -DXRDP_NVENC
+endif
+
 # -no-suppress is an automake-specific flag which is needed
 # to prevent us missing compiler errors in some circumstances
 # (see https://github.com/neutrinolabs/xrdp/pull/1843 )

common/os_calls.c

@@ -154,6 +154,36 @@ g_init(const char *app_name)
 
     WSAStartup(2, &wsadata);
 #endif
+#if defined(XRDP_NVENC)
+    if (g_strcmp(app_name, "xrdp") == 0)
+    {
+        /* call cuInit() to initalize the nvidia drivers */
+        /* TODO create an issue on nvidia forums to figure out why we need to
+        *  do this */
+        if (g_fork() == 0)
+        {
+            typedef int (*cu_init_proc)(int flags);
+            cu_init_proc cu_init;
+            long lib;
+            char cuda_lib_name[] = "libcuda.so";
+            char cuda_func_name[] = "cuInit";
+
+            lib = g_load_library(cuda_lib_name);
+            if (lib != 0)
+            {
+                cu_init = (cu_init_proc)
+                          g_get_proc_address(lib, cuda_func_name);
+                if (cu_init != NULL)
+                {
+                    cu_init(0);
+                }
+            }
+            log_end();
+            g_deinit();
+            g_exit(0);
+        }
+    }
+#endif
 }
 
 /*****************************************************************************/
@@ -1530,6 +1560,39 @@ g_sck_send_fd_set(int sck, const void *ptr, unsigned int len,
     return rv;
 }
 
+/******************************************************************************/
+int
+g_alloc_shm_map_fd(void **addr, int *fd, size_t size)
+{
+    int lfd = -1;
+    void *laddr;
+    char name[128];
+    static unsigned int autoinc;
+
+    snprintf(name, 128, "/%8.8X%8.8X", getpid(), autoinc++);
+    lfd = shm_open(name, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
+    if (lfd == -1)
+    {
+        return 1;
+    }
+    shm_unlink(name);
+    if (ftruncate(lfd, size) == -1)
+    {
+        close(lfd);
+        return 2;
+    }
+    /* map fd to address space */
+    laddr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, lfd, 0);
+    if (laddr == MAP_FAILED)
+    {
+        close(lfd);
+        return 3;
+    }
+    *addr = laddr;
+    *fd = lfd;
+    return 0;
+}
+
 /*****************************************************************************/
 /* returns boolean */
 int

common/os_calls.h

@@ -124,6 +124,7 @@ int      g_sck_recv_fd_set(int sck, void *ptr, unsigned int len,
  */
 int      g_sck_send_fd_set(int sck, const void *ptr, unsigned int len,
                            int fds[], unsigned int fdcount);
+int      g_alloc_shm_map_fd(void **addr, int *fd, size_t size);
 int      g_sck_last_error_would_block(int sck);
 int      g_sck_socket_ok(int sck);
 /**

common/trans.h

@@ -57,6 +57,8 @@ enum xrdp_source
     XRDP_SOURCE_SESMAN,
     XRDP_SOURCE_CHANSRV,
     XRDP_SOURCE_MOD,
+    XORGXRDP_SOURCE_XORG,
+    XORGXRDP_SOURCE_XRDP,
 
     XRDP_SOURCE_MAX_COUNT
 };

configure.ac

@@ -175,6 +175,12 @@ AC_ARG_ENABLE(x264, AS_HELP_STRING([--enable-x264],
               [Use x264 library (default: no)]),
               [], [enable_x264=no])
 AM_CONDITIONAL(XRDP_X264, [test x$enable_x264 = xyes])
+AC_ARG_ENABLE(nvenc, AS_HELP_STRING([--enable-nvenc],
+              [Use nvenc library (default: no), env vars XRDP_NVENC_CFLAGS and
+               XRDP_NVENC_LIBS should be set if used]),
+              [], [enable_nvenc=no])
+AM_CONDITIONAL(XRDP_NVENC, [test x$enable_nvenc = xyes])
+AM_CONDITIONAL(XRDP_YAMI, [test x$enable_yami = xyes])
 AC_ARG_ENABLE(painter, AS_HELP_STRING([--disable-painter],
               [Do not use included painter library (default: no)]),
               [], [enable_painter=yes])
@@ -259,6 +265,9 @@ AC_CHECK_HEADER([security/_pam_types.h],
 AC_CHECK_HEADER([security/pam_constants.h],
                 [AC_DEFINE([HAVE_PAM_CONSTANTS_H], 1, [Using OpenPAM], [])])
 
+# shm_open may not be in the C library
+AC_SEARCH_LIBS([shm_open], [rt])
+
 # Find imlib2
 case "$with_imlib2" in
     '' | no) AC_MSG_NOTICE([imlib2 will not be supported])
@@ -493,6 +502,18 @@ AS_IF( [test "x$enable_pixman" = "xyes"] , [PKG_CHECK_MODULES(PIXMAN, pixman-1 >
 
 AS_IF( [test "x$enable_x264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_X264, x264 >= 0.3.0)] )
 
+if test "x$enable_nvenc" = "xyes"
+then
+if test ! -z "$XRDP_NVENC_CFLAGS"
+then
+  AC_SUBST(XRDP_NVENC_CFLAGS, ["$XRDP_NVENC_CFLAGS"])
+fi
+if test ! -z "$XRDP_NVENC_LIBS"
+then
+  AC_SUBST(XRDP_NVENC_LIBS, ["$XRDP_NVENC_LIBS"])
+fi
+fi
+
 # checking for TurboJPEG
 if test "x$enable_tjpeg" = "xyes"
 then
@@ -653,6 +674,8 @@ AC_CONFIG_FILES([
   xup/Makefile
   third_party/Makefile
   third_party/tomlc99/Makefile
+  xrdp_accel_assist/Makefile
+
 ])
 
 AC_REQUIRE_AUX_FILE([tap-driver.sh])
@@ -668,6 +691,7 @@ echo "  jpeg                    $enable_jpeg"
 echo "  turbo jpeg              $enable_tjpeg"
 echo "  rfxcodec                $enable_rfxcodec"
 echo "  x264                    $enable_x264"
+echo "  nvenc                   $enable_nvenc"
 echo "  painter                 $enable_painter"
 echo "  pixman                  $enable_pixman"
 echo "  fuse                    $enable_fuse"

sesman/sesman.ini.in

@@ -147,6 +147,7 @@ param=Xorg
 ; Leave the rest parameters as-is unless you understand what will happen.
 param=-config
 param=xrdp/xorg.conf
+;param=xrdp/xorg_nvidia.conf
 param=-noreset
 param=-nolisten
 param=tcp
@@ -213,3 +214,4 @@ EnableSyslog=true
 
 [SessionVariables]
 PULSE_SCRIPT=@sesmansysconfdir@/pulse/default.pa
+;XRDP_USE_ACCEL_ASSIST=1

xrdp_accel_assist/Makefile.am

@@ -0,0 +1,41 @@
+AM_CPPFLAGS = \
+  -I$(top_srcdir)/common
+
+EXTRA_DIST = xrdp_accel_assist_shaders.c
+
+XRDP_EXTRA_LIBS =
+XRDP_EXTRA_SOURCES =
+
+if XRDP_NVENC
+AM_CPPFLAGS += -DXRDP_NVENC
+AM_CPPFLAGS += $(XRDP_NVENC_CFLAGS)
+XRDP_EXTRA_LIBS += $(XRDP_NVENC_LIBS)
+XRDP_EXTRA_SOURCES += xrdp_accel_assist_nvenc.c xrdp_accel_assist_nvenc.h encoder_headers/nvEncodeAPI_11_1.h
+endif
+
+if XRDP_YAMI
+AM_CPPFLAGS += -DXRDP_YAMI
+AM_CPPFLAGS += $(XRDP_YAMI_CFLAGS)
+XRDP_EXTRA_LIBS += $(XRDP_YAMI_LIBS)
+XRDP_EXTRA_SOURCES += xrdp_accel_assist_yami.c xrdp_accel_assist_yami.h encoder_headers/yami_inf.h
+endif
+
+pkglibexec_PROGRAMS = \
+  xrdp-accel-assist
+
+xrdp_accel_assist_SOURCES = \
+  xrdp_accel_assist.c \
+  xrdp_accel_assist.h \
+  xrdp_accel_assist_x11.c \
+  xrdp_accel_assist_x11.h \
+  xrdp_accel_assist_egl.c \
+  xrdp_accel_assist_egl.h \
+  xrdp_accel_assist_glx.c \
+  xrdp_accel_assist_glx.h \
+  $(XRDP_EXTRA_SOURCES)
+
+xrdp_accel_assist_LDADD = \
+  $(top_builddir)/common/libcommon.la \
+  $(XRDP_EXTRA_LIBS) \
+  -lX11 -lepoxy
+