Merge pull request #349 from GoSecure/encoding-speed-improvements

Massive MP4 conversion speed improvement

.github/workflows/ci.yml

@@ -80,6 +80,14 @@ jobs:
         working-directory: ./
         run: coverage run --append bin/pyrdp-player.py --headless test/files/test_session.replay
 
+      - name: pyrdp-convert.py to MP4
+        working-directory: ./
+        run: coverage run --append bin/pyrdp-convert.py test/files/test_convert.pyrdp -f mp4
+
+      - name: Verify the MP4 file
+        working-directory: ./
+        run: file test_convert.mp4 | grep "MP4 Base Media"
+
       - name: Run unit tests
         working-directory: ./
         run: coverage run --append -m unittest discover -v
@@ -132,6 +140,14 @@ jobs:
         working-directory: ./
         run: coverage run --append bin/pyrdp-player.py --headless test/files/test_session.replay
 
+      - name: pyrdp-convert.py to MP4
+        working-directory: ./
+        run: coverage run --append bin/pyrdp-convert.py test/files/test_convert.pyrdp -f mp4
+
+      - name: Verify the MP4 file
+        working-directory: ./
+        run: file test_convert.mp4 | grep "MP4 Base Media"
+
       - name: Run unit tests
         working-directory: ./
         run: coverage run --append -m unittest discover -v

CHANGELOG.adoc

@@ -10,6 +10,7 @@ For a detailed view of what has changed, refer to the {uri-repo}/commits/master[
 
 === Enhancements
 
+* `pyrdp-convert` video conversion is now 6x faster! (See {uri-issue}349[#349])
 * Minor CLI improvements
 * Improved type hints
 * Updated instructions to extract the RDP certificate and private key ({uri-issue}345[#345])
@@ -23,6 +24,7 @@ For a detailed view of what has changed, refer to the {uri-repo}/commits/master[
 
 * The slim flavor of our Docker image is now provided for the ARM and ARM64 platforms ({uri-issue}346[#346])
 * Docker images are now built and pushed via GitHub Actions ({uri-issue}334[#334], {uri-issue}341[#341])
+* Added an automated video conversion test to CI configuration ({uri-issue}349[#349])
 
 
 == v1.1.0 - 2021-08-05

pyrdp/convert/MP4EventHandler.py

@@ -4,16 +4,16 @@
 # Licensed under the GPLv3 or later.
 #
 
-from pyrdp.enum import BitmapFlags, CapabilityType
-from pyrdp.pdu import BitmapUpdateData, PlayerPDU
+from pyrdp.enum import CapabilityType
+from pyrdp.pdu import PlayerPDU
 from pyrdp.player.ImageHandler import ImageHandler
 from pyrdp.player.RenderingEventHandler import RenderingEventHandler
 from pyrdp.ui import RDPBitmapToQtImage
 
 import logging
 
 import av
-from PIL import ImageQt
+import qimage2ndarray
 from PySide2.QtGui import QImage, QPainter, QColor
 
 
@@ -50,6 +50,11 @@ def __init__(self, filename: str, fps=30, progress=None):
         self.filename = filename
         self.mp4 = f = av.open(filename, 'w')
         self.stream = f.add_stream('h264', rate=fps)
+        # TODO: this undocumented PyAV stream feature needs to be properly investigated
+        #       we could probably batch the encoding of several frames and benefit from threads
+        #       but trying this as-is lead to no gains
+        #       (actually a degradation but that could be statistically irrelevant)
+        #self.stream.thread_count = 4
         self.stream.pix_fmt = 'yuv420p'
         self.progress = progress
         self.scale = False
@@ -135,7 +140,7 @@ def writeFrame(self):
         p.end()
 
         # Output frame.
-        frame = av.VideoFrame.from_image(ImageQt.fromqimage(surface))
+        frame = av.VideoFrame.from_ndarray(qimage2ndarray.rgb_view(surface))
         for packet in self.stream.encode(frame):
             if self.progress:
                 self.progress()

pyrdp/player/BaseEventHandler.py

@@ -3,7 +3,7 @@
 # Copyright (C) 2020 GoSecure Inc.
 # Licensed under the GPLv3 or later.
 #
-from typing import Optional
+from typing import Optional, Tuple
 from pyrdp.core import decodeUTF16LE, Observer
 from pyrdp.enum import PlayerPDUType, CapabilityType, ParserMode, \
     FastPathFragmentation, \
@@ -176,7 +176,7 @@ def onMouse(self, event: FastPathMouseEvent):
                 3: event.pointerFlags & PointerFlag.PTRFLAGS_BUTTON3
             }, (event.mouseX, event.mouseY))
 
-    def onMouseButton(self, buttons: dict, pos: (int, int)):
+    def onMouseButton(self, buttons: dict, pos: Tuple[int, int]):
         """
         Called when mouse buttons have been pressed.
 

requirements.txt

@@ -11,8 +11,6 @@ idna==3.2
 incremental==21.3.0
 names==0.3.0
 notify2==0.3.1
-Pillow==8.3.1
-pillowcase==2.0.0
 progressbar2==3.53.1
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
@@ -22,6 +20,7 @@ pyOpenSSL==19.1.0
 PySide2==5.15.2
 python-utils==2.5.6
 pytz==2021.1
+qimage2ndarray==1.8.3
 rsa==4.7.2
 scapy==2.4.5
 service-identity==21.1.0

setup.py

@@ -48,8 +48,8 @@
         "full": [
             'wheel>=0.34.2',
             'av>=8',
-            'pillowcase>=2',
             'PySide2>=5.12,<6',
+            'qimage2ndarray>=1.6',
             'dbus-python>=1.2<1.3;platform_system!="Windows"',
             'notify2>=0.3,<1;platform_system!="Windows"'
         ]