diff --git a/README.md b/README.md index 3eef9cc..33e78bf 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,60 @@ # Flowsynth # -Flowsynth is a tool for rapidly modelling network traffic. Flowsynth can be used to generate text-based hexdumps of packets as well as native libpcap format packet captures. +Flowsynth is a tool for rapidly modeling network traffic. Flowsynth can be used to generate text-based hexdumps of packets as well as native libpcap format packet captures. -## Installation ## +## Installation and Usage Overview ## + +Flowsynth has been tested on Python 2.7 and Python 3. + +### Python Script ### The following python modules are required to run Flowsynth: + argparse + scapy -Flowsynth has been tested on python 2.7 and python 3. +To install requirements with pip: + + pip install -r requirements.txt + +Usage: + + usage: flowsynth.py [-h] [-f OUTPUT_FORMAT] [-w OUTPUT_FILE] [-q] [-d] + [--display {text,json}] [--no-filecontent] + input + + positional arguments: + input input files + + optional arguments: + -h, --help show this help message and exit + -f OUTPUT_FORMAT Output format. Valid output formats include: hex, pcap + -w OUTPUT_FILE Output file. + -q Run silently + -d Run in debug mode + --display {text,json} + Display format + --no-filecontent Disable support for the filecontent attribute + +### Python Module ### + +Flowsynth can also be installed and used as a Python module: + + pip install flowsynth + +Example usage: + + import flowsynth + fsmodel = flowsynth.Model(input="my.synth", output_file="out.pcap", output_format="pcap") + fsmodel.build() + +The Model class function `build()` executes flowsynth and the class constructor takes the same arguments as the script (see above): + + class Model(): + def __init__(self, input, output_format="pcap", output_file="", quiet=False, debug=False, display="text", no_filecontent=False): + ... + +*Note:* Because of the current less-than-ideal use of global variables instead of class variables, if more than one Model object is used concurrently, there will be issues. Hopefully this limitation will be remedied in a future release. ## How it works ## @@ -20,8 +65,8 @@ These three phases are referred to as the *parsing phase*, *rendering phase*, an Take the following synfile as an example: flow default tcp myhost.corp.acme.net:12323 > google.com:80 ( tcp.initialize; ); - default > ( content:"GET / HTTP/1.1\x0d\x0a"; content:"Host: google.com\x0d\x0a\x0d\x0a"; ); - default < ( content:"HTTP/1.1 200 OK"; ); + default > ( content:"GET / HTTP/1.1\x0d\x0a"; content:"Host: google.com\x0d\x0a\x0d\x0a"; ); + default < ( content:"HTTP/1.1 200 OK"; ); This sample contains two types of instructions: Flow declarations and event declarations. The first line (*flow default tcp...*) declares to Flowsynth that a flow is being tracked between myhost.corp.acme.net and google.com. The flow name is *default*. All events that apply to this flow will use this name (*default*) to identify which flow they apply to. The third argument specifies which protocol the flow will use. In this case it's *tcp*. Next we specify the source and destination addresses and ports. Finally, an optional attributes section is included at the end. The *tcp.initialize* attribute is included, which tells Flowsynth to automatically generate a three-way handshake for this flow. It's worth nothing that each attribute and line should be closed with a semicolon (;), as shown above. When this flow declaration instruction is parsed by Flowsynth the application will automatically generate event entries in the compiler timeline to establish a three way handshake. @@ -74,15 +119,15 @@ You can declare a flow using the following syntax: The following flow declaration would describe a flow going from a computer to google.com: - flow my_connection tcp mydesktop.corp.acme.com:44123 > google.com:80 (tcp.initialize;); + flow my_connection tcp mydesktop.corp.acme.com:44123 > google.com:80 (tcp.initialize;); The following flow declaration would describe a flow going from a computer to a DNS server: - flow dns_request udp mydesktop.corp.acme.com:11234 > 8.8.8.8:53; + flow dns_request udp mydesktop.corp.acme.com:11234 > 8.8.8.8:53; The following flow declaration would describe a flow using IPv6 addresses: - flow default tcp [2600:1337:2800:1:248:1893:25c8:d1]:31337 > [2600:1337:2800::f1]:80 (tcp.initialize;); + flow default tcp [2600:1337:2800:1:248:1893:25c8:d1]:31337 > [2600:1337:2800::f1]:80 (tcp.initialize;); For the interim, directionality should always be specified as to server: > @@ -117,18 +162,18 @@ usage: Data can be transferred between hosts using two methods. The example below outlines a data exchange between a client and a webserver: my_connection > (content:"GET / HTTP/1.1\x0d\x0aHost:google.com\x0d\x0aUser-Agent: DogBot\x0d\x0a\x0d\x0a";); - my_connection < (content:"HTTP/1.1 200 OK\x0d\x0aContent-Length: 300\x0d\x0a\x0d\x0aWelcome to Google.com!";); + my_connection < (content:"HTTP/1.1 200 OK\x0d\x0aContent-Length: 300\x0d\x0a\x0d\x0aWelcome to Google.com!";); In this example, the flow *my_connection* must have been previously declared. A single packet with the content specified will be transmitted from the client to the server. The following method is also accepted, however, this may change in the future as the syntax is formalized.: my_connection.to_server (content:"GET / HTTP/1.1\x0d\x0aHost:google.com\x0d\x0aUser-Agent: DogBot\x0d\x0a\x0d\x0a";); - my_connection.to_client (content:"HTTP/1.1 200 OK\x0d\x0aContent-Length: 300\x0d\x0a\x0d\x0aWelcome to Google.com!";); + my_connection.to_client (content:"HTTP/1.1 200 OK\x0d\x0aContent-Length: 300\x0d\x0a\x0d\x0aWelcome to Google.com!";); Each content keyword within the () should be closed by a semicolon. Each line should also be closed with a semicolon. Failure to do so will generate a lexer error. Multiple content matches can also be used to logically seperate parts of the response, for example: - # the commands below describe a simple HTTP request - my_connection > (content:"GET / HTTP/1.1\x0d\x0aHost:google.com\x0d\x0a\x0d\x0a";); - my_connection < (content:"HTTP/1.1 200 OK\x0d\x0aContent-Type: text/html\x0d\x0a\x0d\x0a"; content:"This is my response body.";); + # the commands below describe a simple HTTP request + my_connection > (content:"GET / HTTP/1.1\x0d\x0aHost:google.com\x0d\x0a\x0d\x0a";); + my_connection < (content:"HTTP/1.1 200 OK\x0d\x0aContent-Type: text/html\x0d\x0a\x0d\x0a"; content:"This is my response body.";); #### Event Attributes #### The following event attributes are currently supported: diff --git a/scripts/make_pcap_poc.py b/scripts/make_pcap_poc.py index d4d5677..c98741e 100644 --- a/scripts/make_pcap_poc.py +++ b/scripts/make_pcap_poc.py @@ -1,9 +1,9 @@ -#!/usr/bin/python +#!/usr/bin/env python """ make_pcap_poc.py - A tool that takes a file and creates a pcap of that file being downloaded over HTTP. Originally created to make pcaps from proof of concept exploit files related to particular CVEs. -This uses flowsynth to make the pcap. +This uses flowsynth to make the pcap (pip install flowsynth). """ # Copyright 2017 Secureworks # @@ -115,10 +115,9 @@ def usage(): # important - reset file pointer so we can read from the top fs_fh.seek(0) -fs_args = "flowsynth.py \"%s\" -f pcap -w \"%s\"" % (fs_fh.name, os.path.abspath(pcap_file)) -sys.argv = shlex.split(fs_args) +model = flowsynth.Model(input=fs_fh.name, output_format="pcap", output_file=os.path.abspath(pcap_file)) -flowsynth.main() +model.build() fs_fh.close() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1c13d12 --- /dev/null +++ b/setup.py @@ -0,0 +1,38 @@ +from setuptools import setup, find_packages + +with open("README.md", "r") as fh: + long_description = fh.read() + +setup( + name="flowsynth", + version="1.3.0", + author="Will Urbanski", + maintainer="David Wharton", + maintainer_email="counterthreatunit@users.noreply.github.com", + description="Flowsynth is a tool for rapidly modeling network traffic. Flowsynth can be used to generate text-based hexdumps of packets as well as native libpcap format packet captures.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/secureworks/flowsynth", + package_dir={"flowsynth": "src"}, + packages=["flowsynth"], + install_requires=[ + "scapy>=2.4.0", + "argparse", + ], + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Intended Audience :: Information Technology", + "Intended Audience :: Developers", + "Topic :: System :: Networking", + ], + python_requires='>=2.7', + keywords='pcap, pcaps, packet capture, libpcap, IDS, IPS, packets, scapy', + project_urls={ + 'Documentation': 'https://github.com/secureworks/flowsynth/blob/master/README.md', + 'Source': 'https://github.com/secureworks/flowsynth', + }, +) diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..637ea29 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,6 @@ +name = "flowsynth" + +try: + from flowsynth import Model +except ImportError: + from flowsynth.flowsynth import Model diff --git a/src/flowsynth.py b/src/flowsynth.py index 3e655ab..31d1ad1 100755 --- a/src/flowsynth.py +++ b/src/flowsynth.py @@ -38,7 +38,7 @@ from scapy.all import Ether, IP, IPv6, TCP, UDP, RandMAC, hexdump, wrpcap #global variables -APP_VERSION_STRING = "1.0.6" +APP_VERSION_STRING = "1.3.0" LOGGING_LEVEL = logging.INFO ARGS = None @@ -80,7 +80,7 @@ def __str__(self): class FSLexer: """a lexer for the synfile format""" - + LEX_NEW = 0 LEX_EXISTING = 1 @@ -99,7 +99,7 @@ class FSLexer: ipv6regex = r"^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$" def __init__(self, synfiledata): - + #init self.instructions = [] self.dnscache = {} @@ -135,7 +135,7 @@ def resolve_dns(self, shost): except socket.gaierror: compiler_bailout("Cannot resolve %s" % shost) return shost - + def lex_flow(self, tokens): """ lex flow declarations""" logging.debug("lex_flow() called with %s", tokens) @@ -288,7 +288,7 @@ def lex_event(self, tokens): if (tokens[1] == '.'): idx_flowdir = 2 else: - idx_flowdir = 1 + idx_flowdir = 1 except IndexError: parser_bailout("Invalid Syntax. Unexpected flow directionality.") @@ -473,7 +473,7 @@ def parse_content(self, content): mo_text = re.match(pcre_text, content) if (mo_text != None): logging.debug("Content: %s", mo_text.group(1)) - + content_text = mo_text.group(1) replacements = re.findall(r"\\x[a-fA-F0-9]{2}", content_text) for replacement in replacements: @@ -535,7 +535,7 @@ def format_port(port): def render(self, eventid): """ render a specific eventid """ - + event = self.timeline[eventid] pkts = [] @@ -578,7 +578,7 @@ def render(self, eventid): tcp_ack = self.to_server_ack logging.debug("*** Flow %s --> S:%s A:%s B:%s", self.name, tcp_seq, tcp_ack, self.tcp_server_bytes) logging.debug("*** %s", self.timeline[eventid]) - + #nooooooooooo if (len(payload) > 0): #set tcp ack to last ack @@ -621,7 +621,7 @@ def render(self, eventid): else: #generate tcp packet logging.debug("TCP Packet") - + #handle SEQ if 'tcp.seq' in event['attributes']: logging.debug("tcp.seq has been set manually") @@ -671,7 +671,7 @@ def render(self, eventid): tcp_seq = tcp_ack tcp_ack = self.to_client_seq + len(payload) - + self.to_client_ack = self.to_client_seq + len(payload) self.to_client_seq = self.to_client_ack @@ -683,7 +683,7 @@ def render(self, eventid): tcp_seq = tcp_ack tcp_ack = tmp_ack + payload_size - + self.to_server_ack = self.to_server_seq + payload_size self.to_server_seq = self.to_server_ack @@ -748,7 +748,47 @@ def main(): run(ARGS.input) - +class Model(): + """main class.""" + + def __init__(self, input, output_format="pcap", output_file="", quiet=False, debug=False, display="text", no_filecontent=False): + """constructor""" + global ARGS, LOGGING_LEVEL, COMPILER_FLOWS, COMPILER_OUTPUT, COMPILER_TIMELINE, START_TIME, END_TIME, BUILD_STATUS + + # reset globals. A dirty hack for when this is used as a module ... these really should be class variables + # but I don't feel like updating all the code at the moment. If more than one Model object is used concurrently, + # there will be issues.... + LOGGING_LEVEL = logging.INFO + ARGS = None + COMPILER_FLOWS = {} + COMPILER_OUTPUT = [] + COMPILER_TIMELINE = [] + START_TIME = 0 + END_TIME = 0 + BUILD_STATUS = {} + + ARGS = argparse.Namespace() + ARGS.input = input + ARGS.output_format = output_format + ARGS.output_file = output_file + ARGS.quiet = quiet + ARGS.debug = debug + ARGS.display = display + ARGS.no_filecontent = no_filecontent + + if (ARGS.debug == True): + LOGGING_LEVEL = logging.DEBUG + elif (ARGS.quiet == True): + LOGGING_LEVEL = logging.CRITICAL + + logging.basicConfig(format='%(levelname)s: %(message)s', level=LOGGING_LEVEL) + + def build(self): + global START_TIME + + START_TIME = time.time() + run(ARGS.input) + def run(sFile): """ executes the compiler """ global BUILD_STATUS @@ -983,7 +1023,7 @@ def load_syn_file(filename): filedata = fptr.read() fptr.close() except IOError: - compiler_bailout("Cannot open file ('%s')", filename) + compiler_bailout("Cannot open file ('%s')" % filename) return filedata @@ -1016,7 +1056,7 @@ def parser_bailout(msg): sys.exit(-1) except AttributeError: raise SynSyntaxError(msg) - + def show_build_status(): """print the build status to screen""" print(json.dumps(BUILD_STATUS))