feat(#313): Add support for conversion of multiple exported sessions.

.github/workflows/ci.yml

@@ -76,41 +76,9 @@ jobs:
         working-directory: ./
         run: coverage run --append test/test_mitm_initialization.py dummy_value
 
-      - name: pyrdp-player.py read a replay in headless mode test
-        working-directory: ./
-        run: coverage run --append bin/pyrdp-player.py --headless test/files/test_session.replay
-
-      - name: pyrdp-convert.py to MP4
-        working-directory: ./
-        run: coverage run --append bin/pyrdp-convert.py test/files/test_convert.pyrdp -f mp4
-
-      - name: Verify the MP4 file
-        working-directory: ./
-        run: file test_convert.mp4 | grep "MP4 Base Media"
-
-      - name: pyrdp-convert.py replay to JSON
-        working-directory: ./
-        run: coverage run --append bin/pyrdp-convert.py test/files/test_convert.pyrdp -f json
-
-      - name: Verify the replay to JSON file
-        working-directory: ./
-        run: ./test/validate_json.sh test_convert.json
-
-      - name: pyrdp-convert.py PCAP to JSON
-        working-directory: ./
-        run: coverage run --append bin/pyrdp-convert.py test/files/test_session.pcap -f json
-
-      - name: Verify the PCAP to JSON file
-        working-directory: ./
-        run: ./test/validate_json.sh "20200319000716_192.168.38.1:20989-192.168.38.1:3389.json"
-
-      - name: pyrdp-convert.py PCAP to replay
-        working-directory: ./
-        run: coverage run --append bin/pyrdp-convert.py test/files/test_session.pcap -f replay
-
-      - name: Verify that the replay file exists
+      - name: Running pyrdp-player and pyrdp-convert integration tests with verifications
         working-directory: ./
-        run: file -E "20200319000716_192.168.38.1:20989-192.168.38.1:3389.pyrdp"
+        run: ./test/integration.sh
 
       - name: Run unit tests
         working-directory: ./

CHANGELOG.adoc

@@ -21,6 +21,7 @@ For a detailed view of what has changed, refer to the {uri-repo}/commits/master[
 * Capture and log NetNTLMv2 hash if the server enforces NLA and we don't have the NLA redirection attack activated ({uri-issue}367[#367], {uri-issue}358[#358])
 * `pyrdp-convert` video conversion is now 6x faster! (See {uri-issue}349[#349])
 * `pyrdp-convert` video format can be viewed during encoding and will play even if the conversion process crashes or is halted ({uri-issue}352[#352], {uri-issue}353[#353])
+* `pyrdp-convert` can now handle exported PDUs (decrypted pcaps) with multiple sessions in them ({uri-issue}313[#313], {uri-issue}368[#368])
 * `pyrdp-convert` can now extract session information including keyboard and mouse movement information in JSON from pcap and PDUs ({uri-issue}331[#331], {uri-issues}366[#366])
 * `pyrdp-convert` has better success messages, error reporting and exit status ({uri-issue}361[#361], {uri-issue}369[#369])
 * Minor CLI improvements
@@ -46,6 +47,7 @@ For a detailed view of what has changed, refer to the {uri-repo}/commits/master[
 * Added an automated video conversion test to CI configuration ({uri-issue}349[#349])
 * Added an automated JSON conversion test to CI configuration with some validation ({uri-issue}369[#369])
 * Added an automated replay conversion test to CI configuration ({uri-issue}369[#369])
+* Test refactoring to allow running most GitHub CI tests locally when developing ({uri-issue}368[#368])
 
 
 == v1.1.0 - 2021-08-05

pyrdp/convert/ExportedPDUStream.py

@@ -1,6 +1,6 @@
 #
 # This file is part of the PyRDP project.
-# Copyright (C) 2021 GoSecure Inc.
+# Copyright (C) 2021, 2022 GoSecure Inc.
 # Licensed under the GPLv3 or later.
 #
 from pyrdp.convert.PCAPStream import PCAPStream
@@ -25,7 +25,6 @@ def __iter__(self):
         return self
 
     def __next__(self):
-
         while True:
             if self.n >= len(self):
                 raise StopIteration
@@ -35,7 +34,4 @@ def __next__(self):
             data = packet.load[60:]
             self.n += 1
 
-            if any(ip not in self.ips for ip in [src.ip, dst.ip]):
-                continue  # Skip packets not meant for this stream.
-
-            return PCAPStream.output(data, packet.time, src, dst)
+            return PCAPStream.output(data, packet.time, src, dst)

pyrdp/convert/utils.py

@@ -1,6 +1,6 @@
 #
 # This file is part of the PyRDP project.
-# Copyright (C) 2021 GoSecure Inc.
+# Copyright (C) 2021, 2022 GoSecure Inc.
 # Licensed under the GPLv3 or later.
 #
 import enum
@@ -10,7 +10,6 @@
 from scapy.layers.l2 import Ether
 
 from pyrdp.convert.JSONEventHandler import JSONEventHandler
-from pyrdp.core import Uint32BE
 from pyrdp.player import HAS_GUI
 
 from pyrdp.convert.pyrdp_scapy import *
@@ -71,10 +70,8 @@ def __str__(self):
 
 def extractInetAddressesFromPDUPacket(packet) -> Tuple[InetAddress, InetAddress]:
     """Returns the src and dst InetAddress (IP, port) from a PDU packet"""
-    return (InetAddress(".".join(str(b) for b in packet.load[12:16]),
-                        Uint32BE.unpack(packet.load[36:40])),
-            InetAddress(".".join(str(b) for b in packet.load[20:24]),
-                        Uint32BE.unpack(packet.load[44:48])))
+    x = ExportedPDU(packet.load)
+    return (InetAddress(x.src, x.sport), InetAddress(x.dst, x.dport))
 
 
 def createHandler(format: str, outputFileBase: str, progress=None) -> Tuple[str, str]:
@@ -93,6 +90,30 @@ def createHandler(format: str, outputFileBase: str, progress=None) -> Tuple[str,
     return HandlerClass(outputFileBase, progress=progress) if HandlerClass else None, outputFileBase
 
 
+class ExportedPDU(Packet):
+    """60 byte EXPORTED_PDU header."""
+    # We could properly parse the EXPORTED_PDU struct, but we are mostly dealing with IP exported PDUs
+    # so let's just wing it.
+    name = "ExportedPDU"
+    fields_desc = [ 
+                    IntField("tag1Num", None),  # 4
+                    StrFixedLenField("proto", None, length=4),  # 8
+                    IntField("tag2Num", None),  # 12
+                    IPField("src", None),  # 16
+                    IntField("tag3Num", None),  # 20
+                    IPField("dst", None),  # 24
+                    IntField("tag4Num", None),  # 28
+                    IntField("portType", None),  # 32
+                    IntField("tag5Num", None),  # 36
+                    IntField("sport", None),  # 40
+                    IntField("tag6Num", None),  # 44
+                    IntField("dport", None),  # 48
+                    IntField("tag7Num", None),  # 52
+                    IntField("frame", None),   # 56
+                    IntField("endOfTags", None),  # 60
+    ]
+
+
 # noinspection PyUnresolvedReferences
 def tcp_both(p) -> str:
     """Session extractor which merges both sides of a TCP channel."""
@@ -101,7 +122,15 @@ def tcp_both(p) -> str:
         return str(
             sorted(["TCP", p[IP].src, p[TCP].sport, p[IP].dst, p[TCP].dport], key=str)
         )
-    return "Other"
+
+    # Need to make sure this is OK when non-TCP, non-exported data is present.
+    if Ether not in p:
+        x = ExportedPDU(p.load)
+        return str(
+                sorted([x.proto.upper(), x.src, x.sport, x.dst, x.dport], key=str)
+        )
+
+    return "Unsupported"
 
 
 # noinspection PyUnresolvedReferences
@@ -142,18 +171,19 @@ def loadSecrets(filename: str) -> dict:
 
 def canExtractSessionInfo(session: PacketList) -> bool:
     packet = session[0]
+    # TODO: Eventually we should be able to wrap the session as an ExportedSession
+    # and check for the presence of exported.
     return IP in packet or Ether not in packet
 
 def getSessionInfo(session: PacketList) -> Tuple[InetAddress, InetAddress, float, bool]:
     """Attempt to retrieve an (src, dst, ts, isPlaintext) tuple for a data stream."""
     packet = session[0]
 
-    if IP in packet:
-        # This is a plaintext stream.
-        #
-        # FIXME: This relies on the fact that decrypted traces are using EXPORTED_PDU and
-        #        thus have no `IP` layer, but it is technically possible to have a true
-        #        plaintext capture with very old implementations of RDP.
+    # FIXME: This relies on the fact that decrypted traces are using EXPORTED_PDU and
+    #        thus have no `Ether` layer, but it is technically possible to have a true
+    #        plaintext capture with very old implementations of RDP.
+    if TCP in packet:
+        # Assume an encrypted stream...
         return (InetAddress(packet[IP].src, packet[IP][TCP].sport),
                 InetAddress(packet[IP].dst, packet[IP][TCP].dport),
                 packet.time, False)

test/integration.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+#
+# This file is part of the PyRDP project.
+# Copyright (C) 2022 GoSecure Inc.
+# Licensed under the GPLv3 or later.
+#
+# We extracted a set of important tests that were run as part of a GitHub
+# workflow before. Having them all here makes them easy to run from a
+# development environment. The GitHub worfklows can still run them.
+#
+# NOTE: Running these locally requires the test/files/test_files.zip to be
+#       extracted in test/files/.
+
+# Any non-zero exit code becomes an error now
+set -e
+
+# Sets how to launch commands. GitHub workflows sets the CI environment variable
+if [[ -z "${CI}" ]]; then
+	PREPEND=""
+else
+	PREPEND="coverage run --append bin/"
+fi
+
+echo ===================================================
+echo pyrdp-player.py read a replay in headless mode test
+${PREPEND}pyrdp-player.py --headless test/files/test_session.replay
+echo
+
+echo ===================================================
+echo pyrdp-convert.py to MP4
+${PREPEND}pyrdp-convert.py test/files/test_convert.pyrdp -f mp4
+echo
+
+echo ===================================================
+echo Verify the MP4 file
+file test_convert.mp4 | grep "MP4 Base Media"
+rm test_convert.mp4
+echo
+
+echo ===================================================
+echo pyrdp-convert.py replay to JSON
+${PREPEND}pyrdp-convert.py test/files/test_convert.pyrdp -f json
+echo
+
+echo ===================================================
+echo Verify the replay to JSON file
+./test/validate_json.sh test_convert.json
+rm test_convert.json
+echo
+
+echo ===================================================
+echo pyrdp-convert.py PCAP to JSON
+${PREPEND}pyrdp-convert.py test/files/test_session.pcap -f json
+echo
+
+echo ===================================================
+echo Verify the PCAP to JSON file
+./test/validate_json.sh "20200319000716_192.168.38.1:20989-192.168.38.1:3389.json"
+rm "20200319000716_192.168.38.1:20989-192.168.38.1:3389.json"
+echo
+
+echo ===================================================
+echo pyrdp-convert.py PCAP to replay
+${PREPEND}pyrdp-convert.py test/files/test_session.pcap -f replay
+echo
+
+echo ===================================================
+echo Verify that the replay file exists
+file -E "20200319000716_192.168.38.1:20989-192.168.38.1:3389.pyrdp"
+rm "20200319000716_192.168.38.1:20989-192.168.38.1:3389.pyrdp"