From 2023ac61772a0d8c2953b5b6c512f20009a6b2e0 Mon Sep 17 00:00:00 2001 From: Alexandre Beaulieu Date: Wed, 27 Oct 2021 07:23:29 -0400 Subject: [PATCH 1/4] feat(#313): Add support for conversion of multiple exported sessions. Rebased on master by @obilodeau. Some problems likely remain. --- pyrdp/convert/ExportedPDUStream.py | 9 ++---- pyrdp/convert/utils.py | 47 +++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/pyrdp/convert/ExportedPDUStream.py b/pyrdp/convert/ExportedPDUStream.py index cd81f8b7f..7d6c65674 100644 --- a/pyrdp/convert/ExportedPDUStream.py +++ b/pyrdp/convert/ExportedPDUStream.py @@ -5,7 +5,7 @@ # from pyrdp.convert.PCAPStream import PCAPStream from pyrdp.convert.pyrdp_scapy import * -from pyrdp.convert.utils import extractInetAddressesFromPDUPacket, InetAddress +from pyrdp.convert.utils import Exported, extractInetAddressesFromPDUPacket, InetAddress class ExportedPDUStream(PCAPStream): @@ -25,17 +25,14 @@ def __iter__(self): return self def __next__(self): - while True: if self.n >= len(self): raise StopIteration + #packet = Exported(self.packets[self.n].load) packet = self.packets[self.n] src, dst = extractInetAddressesFromPDUPacket(packet) data = packet.load[60:] self.n += 1 - if any(ip not in self.ips for ip in [src.ip, dst.ip]): - continue # Skip packets not meant for this stream. - - return PCAPStream.output(data, packet.time, src, dst) \ No newline at end of file + return PCAPStream.output(data, packet.time, src, dst) diff --git a/pyrdp/convert/utils.py b/pyrdp/convert/utils.py index e313314d1..25538a2b6 100644 --- a/pyrdp/convert/utils.py +++ b/pyrdp/convert/utils.py @@ -93,6 +93,30 @@ def createHandler(format: str, outputFileBase: str, progress=None) -> Tuple[str, return HandlerClass(outputFileBase, progress=progress) if HandlerClass else None, outputFileBase +class Exported(Packet): + """60 byte EXPORTED_PDU header.""" + # We could properly parse the EXPORTED_PDU struct, but we are mostly dealing with IP exported PDUs + # so let's just wing it. + name = "Exported" + fields_desc = [ + IntField("tag1Num", None), # 4 + StrFixedLenField("proto", None, length=4), # 8 + IntField("tag2Num", None), # 12 + IPField("src", None), # 16 + IntField("tag3Num", None), # 20 + IPField("dst", None), # 24 + IntField("tag4Num", None), # 28 + IntField("portType", None), # 32 + IntField("tag5Num", None), # 36 + IntField("sport", None), # 40 + IntField("tag6Num", None), # 44 + IntField("dport", None), # 48 + IntField("tag7Num", None), # 52 + IntField("frame", None), # 56 + IntField("endOfTags", None), # 60 + ] + + # noinspection PyUnresolvedReferences def tcp_both(p) -> str: """Session extractor which merges both sides of a TCP channel.""" @@ -101,7 +125,15 @@ def tcp_both(p) -> str: return str( sorted(["TCP", p[IP].src, p[TCP].sport, p[IP].dst, p[TCP].dport], key=str) ) - return "Other" + + # Need to make sure this is OK when non-TCP, non-exported data is present. + if Ether not in p: + x = Exported(p.load) + return str( + sorted([x.proto.upper(), x.src, x.sport, x.dst, x.dport], key=str) + ) + + return "Unsupported" # noinspection PyUnresolvedReferences @@ -142,18 +174,19 @@ def loadSecrets(filename: str) -> dict: def canExtractSessionInfo(session: PacketList) -> bool: packet = session[0] + # TODO: Eventually we should be able to wrap the session as an ExportedSession + # and check for the presence of exported. return IP in packet or Ether not in packet def getSessionInfo(session: PacketList) -> Tuple[InetAddress, InetAddress, float, bool]: """Attempt to retrieve an (src, dst, ts, isPlaintext) tuple for a data stream.""" packet = session[0] - if IP in packet: - # This is a plaintext stream. - # - # FIXME: This relies on the fact that decrypted traces are using EXPORTED_PDU and - # thus have no `IP` layer, but it is technically possible to have a true - # plaintext capture with very old implementations of RDP. + # FIXME: This relies on the fact that decrypted traces are using EXPORTED_PDU and + # thus have no `Ether` layer, but it is technically possible to have a true + # plaintext capture with very old implementations of RDP. + if TCP in packet: + # Assume an encrypted stream... return (InetAddress(packet[IP].src, packet[IP][TCP].sport), InetAddress(packet[IP].dst, packet[IP][TCP].dport), packet.time, False) From 7459e4589c1054bf4d4dc82c37a1e781867483f9 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Thu, 6 Jan 2022 22:54:11 -0500 Subject: [PATCH 2/4] Moving GitHub workflow tests out to a shell script Allows us to run them locally easily --- .github/workflows/ci.yml | 36 ++------------------- test/integration.sh | 70 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 34 deletions(-) create mode 100755 test/integration.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af4ff96e7..d1d5a4d7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,41 +76,9 @@ jobs: working-directory: ./ run: coverage run --append test/test_mitm_initialization.py dummy_value - - name: pyrdp-player.py read a replay in headless mode test - working-directory: ./ - run: coverage run --append bin/pyrdp-player.py --headless test/files/test_session.replay - - - name: pyrdp-convert.py to MP4 - working-directory: ./ - run: coverage run --append bin/pyrdp-convert.py test/files/test_convert.pyrdp -f mp4 - - - name: Verify the MP4 file - working-directory: ./ - run: file test_convert.mp4 | grep "MP4 Base Media" - - - name: pyrdp-convert.py replay to JSON - working-directory: ./ - run: coverage run --append bin/pyrdp-convert.py test/files/test_convert.pyrdp -f json - - - name: Verify the replay to JSON file - working-directory: ./ - run: ./test/validate_json.sh test_convert.json - - - name: pyrdp-convert.py PCAP to JSON - working-directory: ./ - run: coverage run --append bin/pyrdp-convert.py test/files/test_session.pcap -f json - - - name: Verify the PCAP to JSON file - working-directory: ./ - run: ./test/validate_json.sh "20200319000716_192.168.38.1:20989-192.168.38.1:3389.json" - - - name: pyrdp-convert.py PCAP to replay - working-directory: ./ - run: coverage run --append bin/pyrdp-convert.py test/files/test_session.pcap -f replay - - - name: Verify that the replay file exists + - name: Running pyrdp-player and pyrdp-convert integration tests with verifications working-directory: ./ - run: file -E "20200319000716_192.168.38.1:20989-192.168.38.1:3389.pyrdp" + run: ./test/integration.sh - name: Run unit tests working-directory: ./ diff --git a/test/integration.sh b/test/integration.sh new file mode 100755 index 000000000..647ac2640 --- /dev/null +++ b/test/integration.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# This file is part of the PyRDP project. +# Copyright (C) 2022 GoSecure Inc. +# Licensed under the GPLv3 or later. +# +# We extracted a set of important tests that were run as part of a GitHub +# workflow before. Having them all here makes them easy to run from a +# development environment. The GitHub worfklows can still run them. +# +# NOTE: Running these locally requires the test/files/test_files.zip to be +# extracted in test/files/. + +# Any non-zero exit code becomes an error now +set -e + +# Sets how to launch commands. GitHub workflows sets the CI environment variable +if [[ -z "${CI}" ]]; then + PREPEND="" +else + PREPEND="coverage run --append bin/" +fi + +echo =================================================== +echo pyrdp-player.py read a replay in headless mode test +${PREPEND}pyrdp-player.py --headless test/files/test_session.replay +echo + +echo =================================================== +echo pyrdp-convert.py to MP4 +${PREPEND}pyrdp-convert.py test/files/test_convert.pyrdp -f mp4 +echo + +echo =================================================== +echo Verify the MP4 file +file test_convert.mp4 | grep "MP4 Base Media" +rm test_convert.mp4 +echo + +echo =================================================== +echo pyrdp-convert.py replay to JSON +${PREPEND}pyrdp-convert.py test/files/test_convert.pyrdp -f json +echo + +echo =================================================== +echo Verify the replay to JSON file +./test/validate_json.sh test_convert.json +rm test_convert.json +echo + +echo =================================================== +echo pyrdp-convert.py PCAP to JSON +${PREPEND}pyrdp-convert.py test/files/test_session.pcap -f json +echo + +echo =================================================== +echo Verify the PCAP to JSON file +./test/validate_json.sh "20200319000716_192.168.38.1:20989-192.168.38.1:3389.json" +rm "20200319000716_192.168.38.1:20989-192.168.38.1:3389.json" +echo + +echo =================================================== +echo pyrdp-convert.py PCAP to replay +${PREPEND}pyrdp-convert.py test/files/test_session.pcap -f replay +echo + +echo =================================================== +echo Verify that the replay file exists +file -E "20200319000716_192.168.38.1:20989-192.168.38.1:3389.pyrdp" +rm "20200319000716_192.168.38.1:20989-192.168.38.1:3389.pyrdp" From 7dc5b2ff2ec0c45bc098cdf5c22da935a772917f Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Thu, 6 Jan 2022 23:57:38 -0500 Subject: [PATCH 3/4] refactor to use Exported PDU struct (now called ExportedPDU) --- pyrdp/convert/ExportedPDUStream.py | 5 ++--- pyrdp/convert/utils.py | 15 ++++++--------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/pyrdp/convert/ExportedPDUStream.py b/pyrdp/convert/ExportedPDUStream.py index 7d6c65674..dfc4362fc 100644 --- a/pyrdp/convert/ExportedPDUStream.py +++ b/pyrdp/convert/ExportedPDUStream.py @@ -1,11 +1,11 @@ # # This file is part of the PyRDP project. -# Copyright (C) 2021 GoSecure Inc. +# Copyright (C) 2021, 2022 GoSecure Inc. # Licensed under the GPLv3 or later. # from pyrdp.convert.PCAPStream import PCAPStream from pyrdp.convert.pyrdp_scapy import * -from pyrdp.convert.utils import Exported, extractInetAddressesFromPDUPacket, InetAddress +from pyrdp.convert.utils import extractInetAddressesFromPDUPacket, InetAddress class ExportedPDUStream(PCAPStream): @@ -29,7 +29,6 @@ def __next__(self): if self.n >= len(self): raise StopIteration - #packet = Exported(self.packets[self.n].load) packet = self.packets[self.n] src, dst = extractInetAddressesFromPDUPacket(packet) data = packet.load[60:] diff --git a/pyrdp/convert/utils.py b/pyrdp/convert/utils.py index 25538a2b6..0a75e1a86 100644 --- a/pyrdp/convert/utils.py +++ b/pyrdp/convert/utils.py @@ -1,6 +1,6 @@ # # This file is part of the PyRDP project. -# Copyright (C) 2021 GoSecure Inc. +# Copyright (C) 2021, 2022 GoSecure Inc. # Licensed under the GPLv3 or later. # import enum @@ -10,7 +10,6 @@ from scapy.layers.l2 import Ether from pyrdp.convert.JSONEventHandler import JSONEventHandler -from pyrdp.core import Uint32BE from pyrdp.player import HAS_GUI from pyrdp.convert.pyrdp_scapy import * @@ -71,10 +70,8 @@ def __str__(self): def extractInetAddressesFromPDUPacket(packet) -> Tuple[InetAddress, InetAddress]: """Returns the src and dst InetAddress (IP, port) from a PDU packet""" - return (InetAddress(".".join(str(b) for b in packet.load[12:16]), - Uint32BE.unpack(packet.load[36:40])), - InetAddress(".".join(str(b) for b in packet.load[20:24]), - Uint32BE.unpack(packet.load[44:48]))) + x = ExportedPDU(packet.load) + return (InetAddress(x.src, x.sport), InetAddress(x.dst, x.dport)) def createHandler(format: str, outputFileBase: str, progress=None) -> Tuple[str, str]: @@ -93,11 +90,11 @@ def createHandler(format: str, outputFileBase: str, progress=None) -> Tuple[str, return HandlerClass(outputFileBase, progress=progress) if HandlerClass else None, outputFileBase -class Exported(Packet): +class ExportedPDU(Packet): """60 byte EXPORTED_PDU header.""" # We could properly parse the EXPORTED_PDU struct, but we are mostly dealing with IP exported PDUs # so let's just wing it. - name = "Exported" + name = "ExportedPDU" fields_desc = [ IntField("tag1Num", None), # 4 StrFixedLenField("proto", None, length=4), # 8 @@ -128,7 +125,7 @@ def tcp_both(p) -> str: # Need to make sure this is OK when non-TCP, non-exported data is present. if Ether not in p: - x = Exported(p.load) + x = ExportedPDU(p.load) return str( sorted([x.proto.upper(), x.src, x.sport, x.dst, x.dport], key=str) ) From 4cd1af7857b93444dd3533132f49ae8c92d61cf1 Mon Sep 17 00:00:00 2001 From: Olivier Bilodeau Date: Fri, 7 Jan 2022 00:58:39 -0500 Subject: [PATCH 4/4] Updated CHANGELOG --- CHANGELOG.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index d4e7f6b64..a6e3dd61f 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -21,6 +21,7 @@ For a detailed view of what has changed, refer to the {uri-repo}/commits/master[ * Capture and log NetNTLMv2 hash if the server enforces NLA and we don't have the NLA redirection attack activated ({uri-issue}367[#367], {uri-issue}358[#358]) * `pyrdp-convert` video conversion is now 6x faster! (See {uri-issue}349[#349]) * `pyrdp-convert` video format can be viewed during encoding and will play even if the conversion process crashes or is halted ({uri-issue}352[#352], {uri-issue}353[#353]) +* `pyrdp-convert` can now handle exported PDUs (decrypted pcaps) with multiple sessions in them ({uri-issue}313[#313], {uri-issue}368[#368]) * `pyrdp-convert` can now extract session information including keyboard and mouse movement information in JSON from pcap and PDUs ({uri-issue}331[#331], {uri-issues}366[#366]) * `pyrdp-convert` has better success messages, error reporting and exit status ({uri-issue}361[#361], {uri-issue}369[#369]) * Minor CLI improvements @@ -46,6 +47,7 @@ For a detailed view of what has changed, refer to the {uri-repo}/commits/master[ * Added an automated video conversion test to CI configuration ({uri-issue}349[#349]) * Added an automated JSON conversion test to CI configuration with some validation ({uri-issue}369[#369]) * Added an automated replay conversion test to CI configuration ({uri-issue}369[#369]) +* Test refactoring to allow running most GitHub CI tests locally when developing ({uri-issue}368[#368]) == v1.1.0 - 2021-08-05