Merge release 3.4.2 from pilot branch to master branch

.github/workflows/python-app.yml

@@ -0,0 +1,52 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python application
+
+on:
+  push:
+    branches: [ "master", "pilot" ]
+  pull_request:
+    branches: [ "master", "pilot" ]
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        make venv
+    - name: Run the lint checker
+      run: |
+        make lint
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          make venv
+      - name: Run the tests
+        run: |
+          make test
+
+  docker-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run the hadolint tool on the Dockerfile files
+        run: |
+          make hadolint

CHANGELOG.rst

@@ -149,3 +149,35 @@ Zeek Enhancements (#177)
 
 Author: Nikhileswar Reddy <nreddy@octolabs.io>
 
+3.4.0 (2024-11-12)
+##################
+
+Version 3.4.x is available initially on the pilot branch,
+in a sort of pre-release mode.
+
+* Use pyproject.toml (#184) (#189)
+* Use ruff format to format the code (#183) (#190)
+* Use ruff check --fix to make style changes (#183) (#192)
+* Add github actions CI (#191) (#193)
+* Be able to run unit tests on dalton and flowsynth (#182) (#194)
+* Update nginx from 1.19 to 1.27 (#200) (#202)
+* Update redis from 3.2 to 7.4 (#201)
+* Add unit tests for flowsynth (#204)
+* Use ruff to sort and format imports (#207)
+* Use ruff to detect flake8 bugbears (B) (#209)
+* Use pre-built zeek images (#181)
+* Use bump-my-version to update the version and tag (#197)
+  * Also, use bump-my-version to update the dalton-agent version
+  * Also, show the dalton controller version on the About page
+
+3.4.1 (2024-11-14)
+##################
+
+* Fixed bug with zeek processing. (#213) (#214) (#216)
+* Added some unit tests. (#203) (#215)
+
+3.4.2 (2024-11-15)
+##################
+
+* Updated flask dependencies (#180) (#222)
+  * Configure flask maximum content length

Dockerfile-dalton

@@ -1,24 +1,32 @@
-FROM python:3.9.0
-MAINTAINER David Wharton
+FROM python:3.10.15
 
 # wireshark needed for mergecap; statically compiled
 #  mergecap would be smaller but doing this for now
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get -y install wireshark-common \
-    p7zip-full
+
+# hadolint ignore=DL3008
+RUN apt-get update -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    wireshark-common \
+    p7zip-full \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # for development; not needed by the app
 #RUN apt-get install -y less nano net-tools
 
 WORKDIR /opt/dalton
 
-COPY requirements.txt /opt/dalton/requirements.txt
-RUN pip install -r requirements.txt
 
+COPY pyproject.toml /opt/dalton
 COPY app /opt/dalton/app
+RUN pip install --no-cache-dir -e .
 COPY run.py /opt/dalton/run.py
 COPY dalton.conf /opt/dalton/dalton.conf
 COPY rulesets /opt/dalton/rulesets
 COPY engine-configs /opt/dalton/engine-configs
 
-CMD python /opt/dalton/run.py -c /opt/dalton/dalton.conf
+STOPSIGNAL SIGINT
+EXPOSE 8080
+
+# Note: if changing the next line, also look to change the command in docker-compose.yml
+CMD ["flask", "--app", "app", "run", "--port=8080", "--host=0.0.0.0"]

Dockerfile-nginx

@@ -1,9 +1,8 @@
 # spin up nginx with custom conf
-FROM nginx:1.19.4
-MAINTAINER David Wharton
+FROM nginx:1.27.2
 
-ARG DALTON_EXTERNAL_PORT
-ARG DALTON_EXTERNAL_PORT_SSL
+ARG DALTON_EXTERNAL_PORT=80
+ARG DALTON_EXTERNAL_PORT_SSL=443
 
 RUN rm /etc/nginx/nginx.conf && rm -rf /etc/nginx/conf.d
 COPY nginx-conf/nginx.conf /etc/nginx/nginx.conf
@@ -13,6 +12,5 @@ COPY nginx-conf/tls /etc/nginx/tls
 # adjust nginx config so redirects point to external port(s).
 # order of sed operations matters since one replaced string is a subset of the other.
 RUN sed -i 's/REPLACE_AT_DOCKER_BUILD_SSL/'"${DALTON_EXTERNAL_PORT_SSL}"'/' /etc/nginx/conf.d/dalton.conf
+# hadolint ignore=DL3059
 RUN sed -i 's/REPLACE_AT_DOCKER_BUILD/'"${DALTON_EXTERNAL_PORT}"'/' /etc/nginx/conf.d/dalton.conf
-
-CMD nginx

Makefile

@@ -0,0 +1,39 @@
+
+VENV := $(or ${VENV},${VENV},$(CURDIR)/.venv)
+PIP=$(VENV)/bin/pip
+PYTHON=$(VENV)/bin/python
+PYTEST=$(VENV)/bin/pytest
+COVERAGE=$(VENV)/bin/coverage
+RUFF=$(VENV)/bin/ruff
+ACTIVATE=$(VENV)/bin/activate
+BUMPVERSION=$(VENV)/bin/bump-my-version
+BUMPPART ?= patch
+
+venv $(VENV):
+	python3 -m venv $(VENV)
+	$(PIP) install --upgrade pip wheel
+	$(PIP) install -e . -e ".[testing]" -e ".[devtools]"
+
+test: $(VENV)
+	. $(ACTIVATE) && $(PYTEST) tests
+
+coverage: $(VENV)
+	. $(ACTIVATE) && $(COVERAGE) run -m pytest tests
+	$(COVERAGE) report
+
+lint: $(VENV)
+	$(RUFF) format --check
+	$(RUFF) check
+
+fix: $(VENV)
+	$(RUFF) format
+	$(RUFF) check --fix
+
+hadolint: Dockerfile-dalton Dockerfile-nginx dalton-agent/Dockerfiles/Dockerfile_*
+	docker run -t --rm -v `pwd`:/app -w /app hadolint/hadolint /bin/hadolint $^
+
+bumpversion: $(VENV) pyproject.toml
+	$(BUMPVERSION) bump $(BUMPPART)
+
+bumpagent: $(VENV) pyproject.toml
+	$(BUMPVERSION) bump --config-file dalton-agent/.bumpversion.toml $(BUMPPART)

api/dalton.py

@@ -1,10 +1,10 @@
 """Dalton API client."""
-import requests
+
 import time
 
+import requests
 from requests.exceptions import HTTPError
 
-
 RETRIES = 3
 SLEEP_TIME = 60
 
@@ -142,6 +142,7 @@ def get_current_sensors(self) -> dict:
             'tech': 'suricata/5.0.7',
             'agent_version': '3.1.1'}}
         """
-        response = self._dalton_get("dalton/controller_api/get-current-sensors-json-full")
+        response = self._dalton_get(
+            "dalton/controller_api/get-current-sensors-json-full"
+        )
         return response.json()
-

api/examples/job_submission.py

@@ -1,4 +1,5 @@
 """Example on how to submit a job using the Dalton Client API. Mock data is in mocks directory."""
+
 import os
 
 from api.dalton import DaltonAPI

app/__init__.py

@@ -0,0 +1,50 @@
+import logging
+import os
+
+from flask import Flask
+
+from app.dalton import dalton_blueprint, ensure_rulesets_exist, setup_dalton_logging
+from app.flowsynth import flowsynth_blueprint, setup_flowsynth_logging
+
+__version__ = "3.4.2"
+
+
+def create_app(test_config=None):
+    """Create the flask app."""
+    curdir = os.path.dirname(os.path.abspath(__file__))
+    static_folder = os.path.join(curdir, "static")
+    daltonfs = Flask("app", static_folder=static_folder)
+    if test_config:
+        # load the test config if passed in
+        daltonfs.config.from_mapping(test_config)
+
+    if not daltonfs.testing:
+        setup_dalton_logging()
+        setup_flowsynth_logging()
+        ensure_rulesets_exist()
+
+    # register modules
+    #
+    # dalton
+    daltonfs.register_blueprint(dalton_blueprint)
+
+    # flowsynth
+    daltonfs.register_blueprint(flowsynth_blueprint, url_prefix="/flowsynth")
+
+    daltonfs.debug = True
+
+    # Apparently the werkzeug default logger logs every HTTP request
+    #  which bubbles up to the root logger and gets output to the
+    #  console which ends up in the docker logs.  Since each agent
+    #  checks in every second (by default), this can be voluminous
+    #  and is superfluous for my current needs.
+    try:
+        logging.getLogger("werkzeug").setLevel(logging.ERROR)
+    except Exception:
+        pass
+
+    # Allow the user or the agent to upload large files
+    daltonfs.config["MAX_CONTENT_LENGTH"] = 1024 * 1024 * 1024
+    daltonfs.config["MAX_FORM_MEMORY_SIZE"] = None
+
+    return daltonfs

app/certsynth.py

@@ -1,8 +1,8 @@
-import struct
 import binascii
 import re
+import struct
 
-SYNTH_START = '''
+SYNTH_START = """
 ## Client Hello
 default > (
 content:"\\x80\\x80"; #Length
@@ -16,15 +16,15 @@
 ## Server Hello and Certificate
 default < (
 ### Server Hello
-content:"\\x16\\x03\\x01\\x00Q\\x02\\x00\\x00M\\x03\\x01U\\xe8\\x83\\x0f\\xa1\\xe8\\xcd\\xc6^k\\xae`\\xf4\\xbe\\x0er\\xab~w,\\xce\\xf6L\\x89#\\xc9\\xbaU\\x8b\\xae\\x0ef\\x20\\xcf\\xb4\\xc1\\xb9\\xb0-\\x1e\\xe6Zl\\x0f\\xff\\xfd\\xe3)\\x97\\x89cy\\xa9\\xdbNV\\x83\\xa5\\x97:\\x12Td\\x09\\xac\\x009\\x00\\x00\\x05\\xff\\x01\\x00\\x01\\x00";'''
+content:"\\x16\\x03\\x01\\x00Q\\x02\\x00\\x00M\\x03\\x01U\\xe8\\x83\\x0f\\xa1\\xe8\\xcd\\xc6^k\\xae`\\xf4\\xbe\\x0er\\xab~w,\\xce\\xf6L\\x89#\\xc9\\xbaU\\x8b\\xae\\x0ef\\x20\\xcf\\xb4\\xc1\\xb9\\xb0-\\x1e\\xe6Zl\\x0f\\xff\\xfd\\xe3)\\x97\\x89cy\\xa9\\xdbNV\\x83\\xa5\\x97:\\x12Td\\x09\\xac\\x009\\x00\\x00\\x05\\xff\\x01\\x00\\x01\\x00";"""
 
-SYNTH_END = '''
+SYNTH_END = """
 ### Server Key Exchange
 content:"\\x16\\x03\\x01\\x01\\x8d\\x0c\\x00\\x01\\x89\\x00\\x80\\xbb\\xbc-\\xca\\xd8Ft\\x90|C\\xfc\\xf5\\x80\\xe9\\xcf\\xdb\\xd9X\\xa3\\xf5h\\xb4-K\\x08\\xee\\xd4\\xeb\\x0f\\xb3PLl\\x03\\x02v\\xe7\\x10\\x80\\x0c\\\\xcb\\xba\\xa8\\x92&\\x14\\xc5\\xbe\\xec\\xa5e\\xa5\\xfd\\xf1\\xd2\\x87\\xa2\\xbc\\x04\\x9b\\xe6w\\x80`\\xe9\\x1a\\x92\\xa7W\\xe3\\x04\\x8fh\\xb0v\\xf7\\xd3l\\xc8\\xf2\\x9b\\xa5\\xdf\\x81\\xdc,\\xa7%\\xec\\xe6bp\\xcc\\x9aP5\\xd8\\xce\\xce\\xef\\x9e\\xa0'Jc\\xab\\x1eX\\xfa\\xfdI\\x88\\xd0\\xf6]\\x14gW\\xda\\x07\\x1d\\xf0E\\xcf\\xe1k\\x9b\\x00\\x01\\x02\\x00\\x80\\xa9\\xc5y)\\U\\x00\\x1f\\xa30\\x9b\\x8e\\xd6.\\xed\\x01\\xe9VY0\\x9e\\x03\\x95\\x1b\\x88[q\\xdd\\xfd\\x16\\x0e\\x1a\\xc3\\xbd\\xd3\\x1c\\xbc\\x92\\xa1o\\xed\\xa5T\\xea\\xaa\\xf7\\xdd\\xcd\\xd7\\xb8\\x20E\\x9b\\x1a\\xd4H}[\\xf46\\x98dL\\x0d\\xb6<A\\x98[\\x8c\\x95\\xc5\\\\x0a\\xef>\\xfc\\xb2\\x0d\\xf7\\x94\\xecv\\xd5\\x1f\\xf2\\x85;\\xa6\\xf6\\xf2U\\xcb\\x16\\xc4z\\xa1/\\xdeq\\xf3\\xb0\\x20\\x19\\xef\\xc8\\xc9\\xa5\\x15\\xae\\x9f\\xe9\\x07:\\x0d\\x10\\xbe\\xc8\\xb3\\x98Zh\\xe6k\\x7f5\\x1d\\x8f\\x00\\x80\\x19\\xbb\\x17\\xd6e\\x00\\xc8Y\\x95L\\xde\\xdb\\x9b\\xc7I\\x20F\\x96Po\\xf8\\xedV\\x92\\x85\\xe5V\\xf2zC\\x06\\xcb\\xcc\\xfe(\\x82\\x1c\\x11\\x9d\\xb8\\xd3wT\\x9c\\x08\\xe6\\x0aA\\x06\\xbax\\xb8\\x85\\x94p+\\x88/\\xb4\\x20%\\x1bhx\\xc462\\xa4;\\x9e\\xe7\\x98`\\x01]<q)!\\x9c\\xe7\\x8a6\\xd4\\xd9\\xb6\\x0f\\xbc*\\x0aV\\xf6\\x1cp\\xb7\\xf6[P\\x85\\xfc\\x9f.\\xc3\\x14k\\x0c\\x80\\xf7\\x20\\xf3\\xf4\\xac}\\x14\\xe66lz\\xf5\\xe9Yf\\x07\\x1a\\x00f\\x98\\x90";
 ### Server Hello Done
-content:"\\x16\\x03\\x01\\x00\\x04\\x0e\\x00\\x00\\x00"; );'''
+content:"\\x16\\x03\\x01\\x00\\x04\\x0e\\x00\\x00\\x00"; );"""
 
-SYNTH_CERTIFICATES = '''
+SYNTH_CERTIFICATES = """
 ### Certificates Handshake
 content:"\\x16"; # Type Handshake
 content:"\\x03\\x01"; # Version (TLS 1.0)
@@ -34,20 +34,20 @@
 content:"{1}"; # Certificate Handshake length: Certificate length + 6 (3 bytes)
 content:"{2}"; # Certificates length: Certificate length + 3 (3 bytes)
 content:"{3}"; # Certificate Length (3 bytes)
-content:"{4}"; # DER certificate bytes'''
+content:"{4}"; # DER certificate bytes"""
 
 
 def pem_cert_validate(pem_str):
     """Validate a Certificate PEM string
-       Returns
-       True if valid form False otherwise
+    Returns
+    True if valid form False otherwise
     """
     lines = pem_str.splitlines()
-    if lines[0] != '-----BEGIN CERTIFICATE-----':
+    if lines[0] != "-----BEGIN CERTIFICATE-----":
         return False
-    elif lines[-1] != '-----END CERTIFICATE-----':
+    elif lines[-1] != "-----END CERTIFICATE-----":
         return False
-    if not re.match(r'[a-zA-Z0-9+/]={0,2}', "".join(lines[1:-1])):
+    if not re.match(r"[a-zA-Z0-9+/]={0,2}", "".join(lines[1:-1])):
         return False
     else:
         return True
@@ -56,43 +56,47 @@ def pem_cert_validate(pem_str):
 def pem_to_der(pem_str):
     """Convert a PEM string to DER format"""
     lines = pem_str.strip().splitlines()
-    der = binascii.a2b_base64(''.join(lines[1:-1]))
+    der = binascii.a2b_base64("".join(lines[1:-1]))
     return der
 
 
 def to_synth_bytes(some_str):
     """Convert a string to flowsynth formatted hex"""
     str_hex = binascii.hexlify(some_str).decode("utf-8")
-    out = "\\x" + "\\x".join(
-        [str_hex[i:i + 2] for i in range(0, len(str_hex), 2)])
+    out = "\\x" + "\\x".join([str_hex[i : i + 2] for i in range(0, len(str_hex), 2)])
     return out
 
 
 def cert_to_synth(cert_str, format):
     """Generates flowsynth of a TLS handshake for the given certificate
-        cert_str -- A string of a PEM or DER encoded certificate
-        format -- "PEM or "DER"
+    cert_str -- A string of a PEM or DER encoded certificate
+    format -- "PEM or "DER"
     """
 
     cert_bytes = ""
-    if format == 'PEM':
+    if format == "PEM":
         # validate?
         cert_bytes = pem_to_der(cert_str)
-    elif format == 'DER':
+    elif format == "DER":
         cert_bytes = cert_str
-    handshake_len_bytes = struct.pack('>H', len(cert_bytes) + 10)
+    handshake_len_bytes = struct.pack(">H", len(cert_bytes) + 10)
     handshake_len_synth_bytes = to_synth_bytes(handshake_len_bytes)
-    cert_handshake_len_bytes = struct.pack('>I', len(cert_bytes) + 6)[1:]
+    cert_handshake_len_bytes = struct.pack(">I", len(cert_bytes) + 6)[1:]
     cert_handshake_len_synth_bytes = to_synth_bytes(cert_handshake_len_bytes)
-    certs_len_bytes = struct.pack('>I', len(cert_bytes) + 3)[1:]
+    certs_len_bytes = struct.pack(">I", len(cert_bytes) + 3)[1:]
     certs_len_synth_bytes = to_synth_bytes(certs_len_bytes)
-    cert_len_bytes = struct.pack('>I', len(cert_bytes))[1:]
+    cert_len_bytes = struct.pack(">I", len(cert_bytes))[1:]
     cert_len_synth_bytes = to_synth_bytes(cert_len_bytes)
-    return "".join([SYNTH_START,
-                    SYNTH_CERTIFICATES.format(
-                        handshake_len_synth_bytes,
-                        cert_handshake_len_synth_bytes,
-                        certs_len_synth_bytes,
-                        cert_len_synth_bytes,
-                        to_synth_bytes(cert_bytes)),
-                    SYNTH_END])
+    return "".join(
+        [
+            SYNTH_START,
+            SYNTH_CERTIFICATES.format(
+                handshake_len_synth_bytes,
+                cert_handshake_len_synth_bytes,
+                certs_len_synth_bytes,
+                cert_len_synth_bytes,
+                to_synth_bytes(cert_bytes),
+            ),
+            SYNTH_END,
+        ]
+    )